| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "slices" |
| 7 | "testing" |
| 8 | ) |
| 9 | |
| 10 | // Merge of zdd-flag-readiness (main's Task 9): on a box that declined |
| 11 | // CARLOS_SWAP=overlap and honours failure records or hold-ran markers, the |
| 12 | // live exec gate is asked about the build a live-link start is about to |
| 13 | // run. On this branch a cold wake does not exec the live link: step 2b |
| 14 | // captures the link once, records that build and runs the captured file by |
| 15 | // path (StartGenExe, SeedExe, a worker's onto), and a sidecar on an |
| 16 | // unstaged generation does the same. Main's backend checks sit on the |
| 17 | // live-link arms only (exe == "", onto == nil), so the capture itself must |
| 18 | // ask the gate, and before the record: a build the box will not start is |
| 19 | // never named as the one about to run (settings spec §4.2), so a refusal |
| 20 | // completes no request. |
| 21 | |
| 22 | // A cold instance wake whose live link names a held build runs nothing and |
| 23 | // records nothing; one whose link names a build the gate passes is asked |
| 24 | // about exactly that build and runs it. |
| 25 | func TestTheDeclinedGateIsAskedAboutTheCapturedWakeBeforeItsRecord(t *testing.T) { |
| 26 | a, b := identityActivator(t, "v2") |
| 27 | b.restoreFresh = true |
| 28 | l := &execLog{} |
| 29 | a.BeforeExec = recordInto(b, l) |
| 30 | g := &liveGate{refuse: map[string]bool{"v2": true}} |
| 31 | a.SetLiveExecGate(g.gate) |
| 32 | _, err := a.beginRequest(context.Background(), seeded(identityRoute(t, "one.example.com"))) |
| 33 | if !errors.Is(err, errLiveHeld) { |
| 34 | t.Fatalf("err = %v, want the gate's refusal", err) |
| 35 | } |
| 36 | if calls, _, _ := l.seen(); len(calls) != 0 { |
| 37 | t.Fatalf("records %v: a build the box will not start was recorded", calls) |
| 38 | } |
| 39 | if b.startCount() != 0 || b.seedCount() != 0 { |
| 40 | t.Fatalf("started %d, seeded %d: held v2 ran", b.startCount(), b.seedCount()) |
| 41 | } |
| 42 | if got := g.builds(); !slices.Equal(got, []string{"v2"}) { |
| 43 | t.Fatalf("gate asked %v, want [v2]", got) |
| 44 | } |
| 45 | waitFor(t, "the lease to go back", func() bool { return b.releaseCount() == 1 }) |
| 46 | |
| 47 | b.setLive("v1") |
| 48 | done, err := a.beginRequest(context.Background(), identityRoute(t, "one.example.com")) |
| 49 | if err != nil { |
| 50 | t.Fatalf("wake onto v1 = %v", err) |
| 51 | } |
| 52 | done() |
| 53 | if got := g.builds(); !slices.Equal(got, []string{"v2", "v1"}) { |
| 54 | t.Fatalf("gate asked %v, want [v2 v1]", got) |
| 55 | } |
| 56 | if got := identityEvents(b); !slices.Equal(got, []string{"record v1", "start v1"}) { |
| 57 | t.Fatalf("events %v, want v1 recorded and started", got) |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | // A worker wake execs its captured identity (onto), which the backend's |
| 62 | // gate never asks about: step 2b asks it, before the record. |
| 63 | func TestTheDeclinedGateIsAskedAboutTheCapturedWorkerWake(t *testing.T) { |
| 64 | a, b, _, _, _ := stagedActivator(t) |
| 65 | b.fakeBackend.setIdentity("v2") |
| 66 | l := &execLog{} |
| 67 | a.BeforeExec = l.hook |
| 68 | g := &liveGate{refuse: map[string]bool{"v2": true}} |
| 69 | a.SetLiveExecGate(g.gate) |
| 70 | host := "bot.x.internal" |
| 71 | if err := a.WakeWorker(alwaysOnWorkerRoute(host)); !errors.Is(err, errLiveHeld) { |
| 72 | t.Fatalf("WakeWorker = %v, want the gate's refusal", err) |
| 73 | } |
| 74 | if got := b.calls(); len(got) != 0 { |
| 75 | t.Fatalf("staged spawns %v: held v2 ran", got) |
| 76 | } |
| 77 | if calls, _, _ := l.seen(); len(calls) != 0 { |
| 78 | t.Fatalf("records %v: a build the box will not start was recorded", calls) |
| 79 | } |
| 80 | if got := g.builds(); !slices.Equal(got, []string{"v2"}) { |
| 81 | t.Fatalf("gate asked %v, want [v2]", got) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | // A sidecar whose generation names no staged file resolves the live link |
| 86 | // and runs that file by path: the gate is asked about it first. When the |
| 87 | // gate refuses the link's build (root repointed it at held v2 while the |
| 88 | // instance kept serving v1), the sidecar runs the instance's own build |
| 89 | // instead, once that build's staged copy is named (BuildIdentity) and the |
| 90 | // gate passes it too (readiness 2 Task 5, recon A §3a): otherwise nothing |
| 91 | // ever restarts it. With the instance's build refused as well, or its copy |
| 92 | // unreadable, the refusal stands: nothing is recorded or started, and the |
| 93 | // instance keeps serving. |
| 94 | func TestTheDeclinedGateIsAskedAboutAnUnstagedSidecarsCapture(t *testing.T) { |
| 95 | t.Parallel() |
| 96 | host := "one.example.com" |
| 97 | setup := func(t *testing.T, refuse map[string]bool, ownErr error) (*Service, *planGenBackend, *liveGate, *execLog) { |
| 98 | t.Helper() |
| 99 | a, b := sidecarActivator(t, "") |
| 100 | done, err := a.beginRequest(context.Background(), identityRoute(t, host)) |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | done() |
| 105 | if gen, _ := a.entryGen(host); gen.StagedPath != "" || b.sidecarStartCount() != 1 { |
| 106 | t.Fatalf("setup: generation %+v, %d live-link sidecars", gen, b.sidecarStartCount()) |
| 107 | } |
| 108 | // The instance serves v1 from a generation that names no staged |
| 109 | // file (woken before this binary, or the recovery probe's). |
| 110 | a.mu.Lock() |
| 111 | a.entries[host].gen.Build = "v1" |
| 112 | a.mu.Unlock() |
| 113 | b.fakeBackend.setIdentity("v2") |
| 114 | b.fakeBackend.mu.Lock() |
| 115 | b.fakeBackend.ident.buildIDErr = ownErr |
| 116 | b.fakeBackend.mu.Unlock() |
| 117 | l := &execLog{} |
| 118 | a.BeforeExec = l.hook |
| 119 | g := &liveGate{refuse: refuse} |
| 120 | a.SetLiveExecGate(g.gate) |
| 121 | b.lastSidecar().kill() |
| 122 | a.sweepOnce() |
| 123 | return a, b, g, l |
| 124 | } |
| 125 | stranded := func(t *testing.T, a *Service, b *planGenBackend, g *liveGate, l *execLog, asked []string) { |
| 126 | t.Helper() |
| 127 | waitFor(t, "the refused restart", func() bool { return len(g.builds()) >= len(asked) && a.sidecarSettled(host) }) |
| 128 | if sc := b.sidecarsOn(); len(sc) != 0 || b.sidecarStartCount() != 1 { |
| 129 | t.Fatalf("sidecars %+v, %d live-link: a held build ran beside the instance", sc, b.sidecarStartCount()) |
| 130 | } |
| 131 | if calls, _, _ := l.seen(); len(calls) != 0 { |
| 132 | t.Fatalf("records %v: a build the box will not start was recorded", calls) |
| 133 | } |
| 134 | if got := g.builds(); !slices.Equal(got, asked) { |
| 135 | t.Fatalf("gate asked %v, want %v", got, asked) |
| 136 | } |
| 137 | if _, ok := a.entryGen(host); !ok { |
| 138 | t.Fatal("the instance stopped serving over its sidecar's refusal") |
| 139 | } |
| 140 | } |
| 141 | t.Run("the instance's own build runs", func(t *testing.T) { |
| 142 | a, b, g, l := setup(t, map[string]bool{"v2": true}, nil) |
| 143 | waitFor(t, "the sidecar on v1", func() bool { return len(b.sidecarsOn()) == 1 && a.sidecarSettled(host) }) |
| 144 | own := fakeIdentity("v1") |
| 145 | if sc := b.sidecarsOn()[0]; sc.Build != "v1" || sc.StagedPath != own.StagedPath || sc.Digest != own.Digest { |
| 146 | t.Fatalf("sidecar on %+v, want the instance's own v1 (%+v)", sc, own) |
| 147 | } |
| 148 | if got := g.builds(); !slices.Equal(got, []string{"v2", "v1"}) { |
| 149 | t.Fatalf("gate asked %v, want [v2 v1]", got) |
| 150 | } |
| 151 | if got := b.buildIdentities(); !slices.Equal(got, []string{"v1"}) { |
| 152 | t.Fatalf("BuildIdentity asked %v, want [v1]", got) |
| 153 | } |
| 154 | if calls, _, _ := l.seen(); !slices.Equal(calls, []string{host + "@v1"}) { |
| 155 | t.Fatalf("records %v, want v1 recorded before its sidecar ran", calls) |
| 156 | } |
| 157 | if b.sidecarStartCount() != 1 { |
| 158 | t.Fatalf("%d live-link sidecars: the recovery used the live link", b.sidecarStartCount()) |
| 159 | } |
| 160 | }) |
| 161 | t.Run("the instance's own build is held too", func(t *testing.T) { |
| 162 | a, b, g, l := setup(t, map[string]bool{"v2": true, "v1": true}, nil) |
| 163 | stranded(t, a, b, g, l, []string{"v2", "v1"}) |
| 164 | }) |
| 165 | t.Run("the instance's own copy cannot be named", func(t *testing.T) { |
| 166 | a, b, g, l := setup(t, map[string]bool{"v2": true}, errors.New("staged copy unreadable")) |
| 167 | stranded(t, a, b, g, l, []string{"v2"}) |
| 168 | }) |
| 169 | } |
| 170 | |
amadan