amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "errors"
6 "os"
7 "path/filepath"
8 "strings"
9 "testing"
10 "time"
11
12 "github.com/carlosframework/platform/internal/activator"
13 "github.com/carlosframework/platform/internal/registry"
14 "github.com/carlosframework/platform/internal/release"
15 "github.com/carlosframework/platform/internal/rootparticipation"
16)
17
18func (f *r2Live) productionPass(t *testing.T) error {
19 t.Helper()
20 start := f.c
21 start.legacyStart = f.u.Start
22 observer := f.e.root.obs
23 observer.resolve, observer.sys = f.c.resolve, f.u
24 observer.procExe = func(int) string { return f.u.exe }
25 pr := f.e.root
26 pr.obs = observer
27 return pr.around(t.Context(), func() error {
28 restart := rootRestartRouteWith(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, observer.recordSettled(f.e.bake), f.c)
29 return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, restart,
30 release.WithRunningVersion(f.c.runningVersion), release.WithRestartAttemptSettled(f.c.finish),
31 release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, start)),
32 release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)))
33 })
34}
35
36func r3RestartCount(u *r2Units) int {
37 n := 0
38 for _, effect := range u.effects {
39 if strings.HasPrefix(effect, "restart ") {
40 n++
41 }
42 }
43 return n
44}
45
46func TestR3StickyBindingErrorMasksCrash(t *testing.T) {
47 if !plan14RootFixture(t) {
48 return
49 }
50 for _, kind := range []string{"none", "list"} {
51 for _, state := range []string{"failed", "auto-restart", "healthy"} {
52 t.Run(kind+"/"+state, func(t *testing.T) {
53 f := r2Fixture(t, kind)
54 original := f.c.host
55 f.c.host = func(ctx context.Context, host string) (proofBinding, error) {
56 if r3RestartCount(f.u) > 0 && strings.HasSuffix(f.u.exe, "/v2/console") {
57 return proofBinding{}, os.ErrPermission
58 }
59 return original(ctx, host)
60 }
61 ready := func(registry.Route) func() error {
62 return func() error {
63 if f.u.exe == release.VersionPath(f.u.opt, "", "v2", f.rt.App) && state != "healthy" {
64 return os.ErrDeadlineExceeded
65 }
66 return nil
67 }
68 }
69 f.c.ready, f.c.legacyReady = ready, ready
70 sleep := f.c.sleep
71 f.c.sleep = func(d time.Duration) {
72 sleep(d)
73 if f.u.exe != release.VersionPath(f.u.opt, "", "v2", f.rt.App) || state == "healthy" {
74 return
75 }
76 f.u.props.MainPID = 0
77 if state == "failed" {
78 f.u.props.ActiveState, f.u.props.SubState = "failed", "failed"
79 } else {
80 f.u.props.ActiveState, f.u.props.SubState = "activating", "auto-restart"
81 }
82 }
83 if err := f.productionPass(t); err != nil {
84 t.Fatal("per-host failure changed main's pass status", err)
85 }
86 want, restarts := "v1", 2
87 if state == "healthy" {
88 want, restarts = "v2", 1
89 }
90 f.wantLive(t, want)
91 f.wantRunning(t, want)
92 if got := r3RestartCount(f.u); got != restarts {
93 t.Fatalf("restarts=%d want=%d: %v", got, restarts, f.u.effects)
94 }
95 })
96 }
97 }
98}
99
100func TestR3FallbackDoubleRestart(t *testing.T) {
101 if !plan14RootFixture(t) {
102 return
103 }
104 for _, rollback := range []bool{false, true} {
105 for _, journalFailure := range []bool{false, true} {
106 t.Run(map[bool]string{false: "restart", true: "rollback"}[rollback]+map[bool]string{false: "/binding", true: "/journal"}[journalFailure], func(t *testing.T) {
107 f := r2Fixture(t, "list")
108 initial, err := f.c.resolve(t.Context(), f.rt)
109 if err != nil {
110 t.Fatal(err)
111 }
112 if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil {
113 t.Fatal(err)
114 }
115 if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil {
116 t.Fatal(err)
117 }
118 if rollback {
119 f.u.crashNew = true
120 }
121 original := f.c.host
122 f.c.host = func(ctx context.Context, host string) (proofBinding, error) {
123 if r3RestartCount(f.u) > 0 {
124 return proofBinding{}, os.ErrPermission
125 }
126 return original(ctx, host)
127 }
128 if journalFailure {
129 f.c.host = original
130 store, err := f.c.attempts()
131 if err != nil {
132 t.Fatal(err)
133 }
134 memory := store.(*rollbackMemory)
135 f.u.afterEffect = func() { memory.failWrite = true }
136 }
137 err = f.restart()(f.rt.Host)
138 if !rollback {
139 if err != nil {
140 t.Fatal(err)
141 }
142 f.wantRunning(t, "v2")
143 if r3RestartCount(f.u) != 1 {
144 t.Fatal(f.u.effects)
145 }
146 return
147 }
148 var notUp *release.UnitNotUpError
149 if !errors.As(err, &notUp) {
150 t.Fatal("missing crash verdict", err)
151 }
152 if err := release.PointLive(f.u.opt, f.rt.Host, "", "v1", f.rt.App); err != nil {
153 t.Fatal(err)
154 }
155 if journalFailure {
156 store, _ := f.c.attempts()
157 store.(*rollbackMemory).failWrite = false
158 }
159 f.c.host = original
160 f.c.rollback = func(ctx context.Context, b proofBinding) (proofBinding, error) {
161 return f.source.rollback(ctx, initial)
162 }
163 f.u.afterEffect = nil
164 if journalFailure {
165 store, _ := f.c.attempts()
166 f.u.afterEffect = func() { store.(*rollbackMemory).failWrite = true }
167 }
168 // Refuse identity after the rollback effect, inside the callback's copy.
169 host := f.c.host
170 f.c.host = func(ctx context.Context, h string) (proofBinding, error) {
171 if r3RestartCount(f.u) >= 2 {
172 return proofBinding{}, os.ErrPermission
173 }
174 return host(ctx, h)
175 }
176 before := r3RestartCount(f.u)
177 if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil {
178 t.Fatal(err)
179 }
180 if r3RestartCount(f.u)-before != 1 {
181 t.Fatal(f.u.effects)
182 }
183 f.wantLive(t, "v1")
184 f.wantRunning(t, "v1")
185 })
186 }
187 }
188}
189
190type r3ExeUnits struct {
191 *r2Units
192 denied bool
193}
194
195func (u *r3ExeUnits) Exe(pid int) (string, error) {
196 if u.denied {
197 return "", os.ErrPermission
198 }
199 return u.r2Units.Exe(pid)
200}
201
202func TestPlan14Fix4ExeDeniedAfterRestart(t *testing.T) {
203 if !plan14RootFixture(t) {
204 return
205 }
206 f := r2Fixture(t, "list")
207 u := &r3ExeUnits{r2Units: f.u}
208 f.source.sys = u
209 f.u.afterEffect = func() { u.denied = true }
210 if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil {
211 t.Fatal(err)
212 }
213 if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil {
214 t.Fatal(err)
215 }
216 if err := rootRestartRouteWith(f.e.root.reg, u, f.u.opt, func(string) string { return "5s" }, nil, f.c)(f.rt.Host); err != nil {
217 t.Fatal(err)
218 }
219 f.wantLive(t, "v2")
220 f.wantRunning(t, "v2")
221 if r3RestartCount(f.u) != 1 {
222 t.Fatal(f.u.effects)
223 }
224}
225
226func TestR3CensusProbeErrorBlocksOrdinary(t *testing.T) {
227 if !plan14RootFixture(t) {
228 return
229 }
230 if err := os.MkdirAll(rootparticipation.Namespace, 0770); err != nil {
231 t.Fatal(err)
232 }
233 if err := os.Chmod(rootparticipation.Namespace, 0770); err != nil {
234 t.Fatal(err)
235 }
236 if _, err := legacyRootRestart(); err == nil {
237 t.Fatal("unreadable census fixture did not refuse")
238 }
239 for _, stamped := range []bool{false, true} {
240 t.Run(map[bool]string{false: "unstamped", true: "derived"}[stamped], func(t *testing.T) {
241 f := r2Fixture(t, "none")
242 f.rt.Unit = ""
243 if stamped {
244 f.rt.Unit = activator.UnitName(f.rt.Host)
245 }
246 if err := f.e.root.reg.Delete(f.rt.Host); err != nil {
247 t.Fatal(err)
248 }
249 if err := f.e.root.reg.Upsert(f.rt); err != nil {
250 t.Fatal(err)
251 }
252 f.c.legacy = func() (bool, error) { t.Fatal("ordinary route probed census"); return false, os.ErrPermission }
253 if err := f.productionPass(t); err != nil {
254 t.Fatal(err)
255 }
256 f.wantLive(t, "v2")
257 f.wantRunning(t, "v2")
258 if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil {
259 t.Fatal(err)
260 }
261 })
262 }
263}
264
265func TestR3PreviousPins(t *testing.T) {
266 if !plan14RootFixture(t) {
267 return
268 }
269 t.Run("relative live link", func(t *testing.T) {
270 f := r2Fixture(t, "list")
271 link := release.LiveLink(f.u.opt, f.rt.Host)
272 relative, err := filepath.Rel(filepath.Dir(link), f.u.exe)
273 if err != nil {
274 t.Fatal(err)
275 }
276 if err := os.Remove(link); err != nil {
277 t.Fatal(err)
278 }
279 if err := os.Symlink(relative, link); err != nil {
280 t.Fatal(err)
281 }
282 if err := f.productionPass(t); err != nil {
283 t.Fatal(err)
284 }
285 f.wantLive(t, "v2")
286 f.wantRunning(t, "v2")
287 prior, err := filepath.EvalSymlinks(filepath.Join(f.u.opt, "previous", f.rt.Host))
288 if err != nil || prior != release.VersionPath(f.u.opt, "", "v1", f.rt.App) {
289 t.Fatal(prior, err)
290 }
291 })
292}
293
294type r3DiscoveryStore struct {
295 release.Store
296 lists, gets int
297 blocked <-chan struct{}
298}
299
300func (s *r3DiscoveryStore) List(ctx context.Context, prefix string) ([]string, error) {
301 s.lists++
302 if s.blocked != nil {
303 <-s.blocked
304 }
305 return nil, os.ErrPermission
306}
307func (s *r3DiscoveryStore) Get(ctx context.Context, key string) ([]byte, string, error) {
308 s.gets++
309 return s.Store.Get(ctx, key)
310}
311
312func TestPlan14Fix4OrdinaryProductionWrapper(t *testing.T) {
313 if !plan14RootFixture(t) {
314 return
315 }
316 for _, kind := range []string{"unit", "exec", "derived", "hibernate"} {
317 for _, stall := range []bool{false, true} {
318 t.Run(kind+map[bool]string{false: "/error", true: "/hang"}[stall], func(t *testing.T) {
319 f := r2Fixture(t, "none")
320 f.rt.Unit = ""
321 f.rt.Backing = registry.BackingUnit
322 f.rt.Socket = "/run/ordinary.sock"
323 f.rt.Addr, f.rt.Health = "", ""
324 if kind == "exec" {
325 f.rt.Backing = "exec"
326 }
327 if kind == "derived" {
328 f.rt.Unit = activator.UnitName(f.rt.Host)
329 }
330 if kind == "hibernate" {
331 f.rt.Hibernate = true
332 }
333 if err := f.e.root.reg.Delete(f.rt.Host); err != nil {
334 t.Fatal(err)
335 }
336 if err := f.e.root.reg.Upsert(f.rt); err != nil {
337 t.Fatal(err)
338 }
339 store := &r3DiscoveryStore{Store: f.e.store}
340 if stall {
341 unblock := make(chan struct{})
342 store.blocked = unblock
343 timer := time.AfterFunc(time.Second, func() { close(unblock) })
344 defer timer.Stop()
345 }
346 f.source.store = store
347 if err := f.productionPass(t); err != nil {
348 t.Fatal(err)
349 }
350 if store.lists != 0 || store.gets != 0 {
351 t.Fatalf("ordinary proof wrapper read records: lists=%d gets=%d", store.lists, store.gets)
352 }
353 f.wantLive(t, "v2")
354 if kind == "hibernate" {
355 if r3RestartCount(f.u) != 0 {
356 t.Fatal("hibernating route restarted", f.u.effects)
357 }
358 } else {
359 f.wantRunning(t, "v2")
360 }
361 })
362 }
363 }
364}
365
366func TestPlan14Fix4IndexFailureCachedForPass(t *testing.T) {
367 if !plan14RootFixture(t) {
368 return
369 }
370 f := r2Fixture(t, "none")
371 store := &r3DiscoveryStore{Store: f.e.store}
372 f.source.store = store
373 other := f.rt
374 other.Host = "second.example.com"
375 if err := f.e.root.reg.Upsert(other); err != nil {
376 t.Fatal(err)
377 }
378 // Observation, adoption, readiness recording and minting share one resolver.
379 if err := f.productionPass(t); err != nil {
380 t.Fatal(err)
381 }
382 if store.lists != 1 {
383 t.Fatalf("index attempts=%d want=1", store.lists)
384 }
385 next := productionProofBindingSource(store, f.u, f.u.opt)
386 if _, err := next.resolve(t.Context(), f.rt); err == nil {
387 t.Fatal("failed discovery accepted")
388 }
389 if store.lists != 2 {
390 t.Fatalf("new pass did not retry: %d", store.lists)
391 }
392}
393
394func TestPlan14Fix4HibernateStartGuard(t *testing.T) {
395 t.Parallel()
396 e, rt, u, c, _ := plan14AdoptionFixture(t)
397 rt.Hibernate = true
398 if err := e.root.reg.Upsert(rt); err != nil {
399 t.Fatal(err)
400 }
401 c.legacyStart = func(string) error { t.Fatal("woke hibernating route"); return nil }
402 c.legacy = func() (bool, error) { t.Fatal("hibernating start read census"); return false, os.ErrPermission }
403 if err := rollbackUnitRestart(e.root.reg, u, u.opt, nil, c)(rt.Host); err != nil {
404 t.Fatal(err)
405 }
406 if len(u.effects) != 0 {
407 t.Fatal(u.effects)
408 }
409}
410
411func TestPlan14Fix4NotUpWinsOverBinding(t *testing.T) {
412 if !restartEffectFailed(errors.Join(errRestartBinding, &release.UnitNotUpError{Unit: "console.service"})) {
413 t.Fatal("binding masked D1 failure")
414 }
415}
416
417func TestPlan14Fix4IncidentProductionWiring(t *testing.T) {
418 if !plan14RootFixture(t) {
419 return
420 }
421 for _, kind := range []string{"none", "guard", "list"} {
422 for _, order := range []string{"reset before adoption", "reset after adoption", "never failed"} {
423 t.Run(kind+"/"+order, func(t *testing.T) {
424 f := r2Fixture(t, kind)
425 if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil {
426 t.Fatal(err)
427 }
428 if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil {
429 t.Fatal(err)
430 }
431 if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil {
432 t.Fatal(err)
433 }
434 f.u.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App)
435 f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0
436 if order == "never failed" {
437 f.u.props.ActiveState, f.u.props.SubState = "inactive", "dead"
438 }
439 if order == "reset before adoption" {
440 if err := f.u.ResetFailed(f.rt.Unit); err != nil {
441 t.Fatal(err)
442 }
443 }
444 before := len(f.u.effects)
445 if kind != "none" {
446 if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil {
447 t.Fatal("PID-zero binding rejected", err)
448 }
449 }
450 f.promote(t, "v1")
451 if err := f.productionPass(t); err != nil {
452 t.Fatal(err)
453 }
454 f.wantLive(t, "v1")
455 if len(f.u.starts) != 0 {
456 t.Fatal("adoption called start", f.u.starts)
457 }
458 if order != "reset after adoption" && (f.u.props.MainPID != 0 || len(f.u.effects) != before) {
459 t.Fatal("woke inactive unit", f.u.effects)
460 }
461 if order == "reset after adoption" {
462 if err := f.u.ResetFailed(f.rt.Unit); err != nil {
463 t.Fatal(err)
464 }
465 }
466 if err := f.u.Restart(f.rt.Unit); err != nil {
467 t.Fatal(err)
468 }
469 f.wantRunning(t, "v1")
470 })
471 }
472 }
473}
474
475func TestPlan14Fix4RepointFailureStillLogged(t *testing.T) {
476 if !plan14RootFixture(t) {
477 return
478 }
479 f := r2Fixture(t, "list")
480 if err := os.MkdirAll(filepath.Join(f.u.opt, "previous", f.rt.Host), 0755); err != nil {
481 t.Fatal(err)
482 }
483 output, err := os.CreateTemp(t.TempDir(), "pass-output")
484 if err != nil {
485 t.Fatal(err)
486 }
487 original := os.Stdout
488 os.Stdout = output
489 defer func() { os.Stdout = original; output.Close() }()
490 err = f.productionPass(t)
491 os.Stdout = original
492 if err != nil {
493 t.Fatal("repoint failure changed pass status", err)
494 }
495 if err := output.Close(); err != nil {
496 t.Fatal(err)
497 }
498 raw, err := os.ReadFile(output.Name())
499 if err != nil {
500 t.Fatal(err)
501 }
502 if !strings.Contains(string(raw), "repoint failed:") {
503 t.Fatal("missing repoint diagnostic", string(raw))
504 }
505 f.wantLive(t, "v1")
506 f.wantRunning(t, "v1")
507 if r3RestartCount(f.u) != 0 {
508 t.Fatal(f.u.effects)
509 }
510}
511
512type r3ResetErrorUnits struct {
513 *r2Units
514 resets int
515}
516
517func (u *r3ResetErrorUnits) ResetFailed(string) error { u.resets++; return os.ErrPermission }
518
519func TestPlan14Fix4RollbackResetFailureMatchesMain(t *testing.T) {
520 if !plan14RootFixture(t) {
521 return
522 }
523 f := r2Fixture(t, "list")
524 u := &r3ResetErrorUnits{r2Units: f.u}
525 f.source.sys = u
526 if err := release.StageArtifact(u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil {
527 t.Fatal(err)
528 }
529 if err := release.PointLive(u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil {
530 t.Fatal(err)
531 }
532 u.crashNew = true
533 err := rootRestartRouteWith(f.e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, f.c)(f.rt.Host)
534 var notUp *release.UnitNotUpError
535 if !errors.As(err, &notUp) {
536 t.Fatal("missing crash verdict", err)
537 }
538 if err := release.PointLive(u.opt, f.rt.Host, "", "v1", f.rt.App); err != nil {
539 t.Fatal(err)
540 }
541 if err := rollbackUnitRestart(f.e.root.reg, u, u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil {
542 t.Fatal(err)
543 }
544 if u.resets != 1 || r3RestartCount(f.u) != 2 {
545 t.Fatal("reset failure repeated or prevented restart", u.resets, u.effects)
546 }
547 f.wantLive(t, "v1")
548 f.wantRunning(t, "v1")
549}
550