amadan
zdd-readiness: a declined box gates the actual launch, workers included, and honours hold-ran markers
Task 9 (Astra R1 #6, the R2/R3 declined residuals, rulings 194/206; plan review #6). ExecBackend.LiveExecGate, set only through Service.SetLiveExecGate by a declined box that honours failure records or hold-ran markers: the flat live-link spawn resolves the link once, gates that build at the launch and execs the resolved staged path; the nested live-link start gates the build the link names before root's start; a worker's spawn from the live link gates its resolved build before publication. The activator asks the same gate before a worker's converge or a recycle's move onto the live link, so a refusal stops nothing: a live worker keeps its build, a crashed one recycles its captured build or stays down with none. A claims-backed wake is gated after its snapshot resolves the link. failureHolds honours hold-ran markers as the armed planner does (decideHoldRan/planCheck), and the book is honoured on records or markers (FailureBook.HoldRanCount). With the gate nil every spawn is today's.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10 files changed,
+945
−69
internal/activator/activator.go+60 −7internal/activator/backend_exec.go+69 −5internal/activator/exec_gate_test.go+304 −0internal/activator/genbackend.go+58 −2internal/activator/workerbuild.go+41 −4internal/host/claimproduction.go+30 −0internal/host/rollout_test.go+7 −1internal/host/rolloutdeclined.go+109 −50internal/host/rolloutdeclined_test.go+258 −0internal/rollout/failures.go+9 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index 1abf6561e..b9d14e9d6 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -962,6 +962,16 @@ type Service struct { |
| // a.mu. |
| uncapturedSaid map[string]bool |
| + // liveExecGate is the backend's LiveExecGate as SetLiveExecGate set it: |
| + // nil on every box but one that declined CARLOS_SWAP=overlap and honours |
| + // failure records or hold-ran markers. The worker converge and recycle |
| + // ask it under a.mu before they stop anything (convergeWorkerLocked, |
| + // recycleWorkerLocked). liveGateSaid is the (host, build) a refusal there |
| + // was last logged for, so a drift the sweep sees every pass is said once. |
| + // Under a.mu. |
| + liveExecGate func(rt registry.Route, build string) error |
| + liveGateSaid map[string]string |
| + |
| fences map[string]uint64 |
| tokenPreparations map[string]chan struct{} |
| stopHistory map[string]*stopObservation |
| @@ -2644,8 +2654,45 @@ func (a *Service) restartWorkerLocked(e *actEntry, now time.Time) { |
| // nil: a converge execs what live/<host> names now. Until plan 5 queues it as |
| // a paced release attempt, this is the one path that moves a worker to a |
| // different installed build. |
| -func (a *Service) convergeWorkerLocked(e *actEntry) { |
| +// |
| +// build is the build live/<host> names, as the sweep just read it. On a box |
| +// that declined CARLOS_SWAP=overlap and honours failure records or hold-ran |
| +// markers, the live exec gate is asked about it first (Task 9): the old |
| +// child is retired before the spawn, so a refusal at the spawn would leave |
| +// the worker down. Refused, nothing is stopped, the worker keeps running its |
| +// current build, and it reports false: no converge happened, and none of the |
| +// sweep's budget is spent. |
| +func (a *Service) convergeWorkerLocked(e *actEntry, build string) bool { |
| + if a.liveGateRefusesLocked(e, build, "it keeps running "+e.spawnVersion) { |
| + return false |
| + } |
| + log.Printf("activator: worker %s serving %s but live symlink is now %s — respawning onto the new build", e.host, e.spawnVersion, build) |
| a.respawnWorkerLocked(e, e.proc, "converging onto a new build", nil) |
| + return true |
| +} |
| + |
| +// liveGateRefusesLocked asks the live exec gate (SetLiveExecGate) whether |
| +// e's worker may move onto build, the build live/<host> names, and reports |
| +// a refusal, which it logs once per host and build. No gate (every box but |
| +// a declined one that honours something) refuses nothing. The gate reads |
| +// only memory, the row and the live link. The caller holds a.mu. |
| +func (a *Service) liveGateRefusesLocked(e *actEntry, build, left string) bool { |
| + if a.liveExecGate == nil { |
| + return false |
| + } |
| + err := a.liveExecGate(e.route, build) |
| + if err == nil { |
| + delete(a.liveGateSaid, e.host) |
| + return false |
| + } |
| + if a.liveGateSaid[e.host] != build { |
| + if a.liveGateSaid == nil { |
| + a.liveGateSaid = map[string]string{} |
| + } |
| + a.liveGateSaid[e.host] = build |
| + log.Printf("activator: worker %s: live/%s names %s, which this box will not start (%v), so it does not move onto it: %s", e.host, e.host, build, err, left) |
| + } |
| + return true |
| } |
| // workerSpawnTimeout bounds a supervised respawn's exec. It is generous |
| @@ -3841,16 +3888,22 @@ func (a *Service) sweepOnce() { |
| // runs once the drift has persisted past ConvergeGrace, and it |
| // consumes the same one-per-sweep budget, so a bad build cannot |
| // respawn the fleet at once. |
| + // |
| + // A converge the live exec gate refuses (a declined box that |
| + // honours a record or marker holding cur, Task 9) stops nothing |
| + // and spends no budget: the worker carries on to supervision |
| + // below, running its current build. |
| if e.worker && !e.route.Hibernate { |
| - log.Printf("activator: worker %s serving %s but live symlink is now %s — respawning onto the new build", e.host, e.spawnVersion, cur) |
| - a.convergeWorkerLocked(e) |
| + if a.convergeWorkerLocked(e, cur) { |
| + a.convergeBudget-- |
| + continue |
| + } |
| + } else { |
| + log.Printf("activator: %s serving %s but live symlink is now %s — cycling onto the new build", e.host, e.spawnVersion, cur) |
| + a.beginStopLocked(e) |
| a.convergeBudget-- |
| continue |
| } |
| - log.Printf("activator: %s serving %s but live symlink is now %s — cycling onto the new build", e.host, e.spawnVersion, cur) |
| - a.beginStopLocked(e) |
| - a.convergeBudget-- |
| - continue |
| } |
| // Sidecar supervision — Restart=always, sweep-driven, no supervisor |
| // goroutine: a sidecar that died (or failed to spawn at wake) while |
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go| index 2da660a85..f15b054e0 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -174,6 +174,21 @@ type ExecBackend struct { |
| // the refusal; this backend only honours the answer. Nil: no gate (every |
| // unit test, and a box with no deploy store to read config from). |
| ConfigGate func(ctx context.Context, rt registry.Route) error |
| + // LiveExecGate answers whether rt may start build, the build a start |
| + // from the live link is about to run: nil means yes, an error refuses |
| + // the start and comes back as it is. Only a box that declined |
| + // CARLOS_SWAP=overlap and honours failure records or hold-ran markers |
| + // from an earlier armed run sets it (internal/host, through |
| + // Service.SetLiveExecGate, before Start), so the plan-time refusal is |
| + // repeated at the real launch with the build actually being started |
| + // (Astra R1 #6, the R2/R3 declined residuals). It is asked by a flat |
| + // instance's live-link spawn (which then execs the staged path it |
| + // resolved, not the link), a nested instance's live-link start, and a |
| + // worker's spawn from the live link. Nil, on every other box: each of |
| + // those spawns is exactly as it was. It reads only memory, the row it |
| + // is handed and the live link, and writes nothing: the activator asks |
| + // it under its own lock too (convergeWorkerLocked). |
| + LiveExecGate func(rt registry.Route, build string) error |
| // ObservePoll is how often the exit observer (unitproc.go) and a |
| // seed's cgroup-empty wait re-check unit state. 0: one second. A field |
| // so tests can shrink the wait without a package-global var shared |
| @@ -1469,8 +1484,29 @@ func (b *ExecBackend) spawnChildFrom(ctx context.Context, rt registry.Route, soc |
| // A staged spawn names its build from the attempt; the caller fills |
| // Gen.Build. Reading the live link here would name whatever root |
| // installed, not what this process runs. |
| + // |
| + // With LiveExecGate set (a declined box that honours failure records or |
| + // hold-ran markers, Task 9), the live link is resolved ONCE here |
| + // (release.ResolveLive), the gate is asked about that build at the |
| + // launch, and the staged file the link named is what is exec'd, not the |
| + // link. So the build the gate passed is the build that runs: a repoint |
| + // between the gate and the exec changes nothing, where exec'ing the link |
| + // would run whatever it named by then (plan review #6). A link that |
| + // cannot be resolved fails the wake, as a missing one does. With the gate |
| + // nil this is today's read and today's exec of the link. |
| spawnVer := "" |
| - if exe == "" { |
| + var liveGate func() error |
| + if exe == "" && b.LiveExecGate != nil { |
| + id, err := release.ResolveLive(b.OptDir, host) |
| + if err != nil { |
| + if ctl != nil { |
| + ctl.Close() |
| + } |
| + return nil, "", fmt.Errorf("%s: %w", host, err) |
| + } |
| + bin, spawnVer = id.StagedPath, id.Version |
| + liveGate = func() error { return b.LiveExecGate(rt, id.Version) } |
| + } else if exe == "" { |
| spawnVer, _ = b.LiveVersion(host) |
| } |
| // Spawned through the cgroup manager (nil-tolerant: no delegation means |
| @@ -1487,7 +1523,9 @@ func (b *ExecBackend) spawnChildFrom(ctx context.Context, rt registry.Route, soc |
| // change during any of it. A refusal starts nothing; the manager drops |
| // the child cgroup and the error path below unbinds the listener, as for |
| // a failed spawn. With no gate on ctx (every live-link spawn) there is no |
| - // hook and the spawn is exactly as it was. |
| + // hook and the spawn is exactly as it was, unless LiveExecGate is set: |
| + // then its check of the resolved build (liveGate) is the hook, asked at |
| + // the same point. |
| cmd, settle, err := b.spawnInCgroupAtTier(rt, child, ownTier, tier, func() *exec.Cmd { |
| cmd := exec.Command(bin, "--socket", sock, "--db", dbPath) |
| cmd.Env = childEnv(childVars) |
| @@ -1498,7 +1536,7 @@ func (b *ExecBackend) spawnChildFrom(ctx context.Context, rt registry.Route, soc |
| // that piping entails. |
| b.teeStd(cmd, rt) |
| return cmd |
| - }, b.launchGate(ctx, host)) |
| + }, b.launchGateWith(ctx, host, liveGate)) |
| if err != nil { |
| // The listener was bound before the spawn; nothing will ever answer on |
| // it now, so unbind it rather than leaving a live socket for a process |
| @@ -1679,7 +1717,21 @@ func (b *ExecBackend) spawnUnitFrom(ctx context.Context, rt registry.Route, sock |
| if b.launchHook != nil { |
| b.launchHook(rt.Host) |
| } |
| - if err := runExecGate(ctx); err != nil { |
| + err = runExecGate(ctx) |
| + if err == nil && exe == nil && b.LiveExecGate != nil { |
| + // A declined box that honours failure records or hold-ran markers |
| + // (Task 9) asks about the build the live link names now, the one |
| + // spawnVer read immediately above: the plan read the link before the |
| + // lease and the restore, and root may have repointed it at a held |
| + // build since. Residual: root mounts the link itself at a generation |
| + // 0 start (a staged path can be handed it only with generations, |
| + // which a declined box has off), so a repoint landing between this |
| + // read and root's start runs the build it names, where the flat |
| + // spawn execs the path it resolved. The next wake is refused at its |
| + // plan. |
| + err = b.LiveExecGate(rt, spawnVer) |
| + } |
| + if err != nil { |
| if gen != 0 { |
| _ = os.Remove(filepath.Join(p.dataDir, instancectl.GenLaunchFileName(gen))) |
| } |
| @@ -2100,7 +2152,8 @@ func (b *ExecBackend) StartWorker(ctx context.Context, rt registry.Route, extra |
| // What it execs is a staged build BY PATH, never live/<host> (zero-downtime |
| // deploys spec §7.9): |
| // - onto == nil (a wake, or a converge onto a new build): the live link is |
| -// ensured, read ONCE, and the file it names is hashed and exec'd; |
| +// ensured, read ONCE, and the file it names is hashed and exec'd, once |
| +// LiveExecGate (when set) has passed its build; |
| // - onto != nil (a crash or max_run recycle): onto.StagedPath is checked |
| // against onto.Digest (release.VerifyStaged) and exec'd. A copy that has |
| // changed is refused with release.ErrStagedCopyChanged and nothing runs. |
| @@ -2131,6 +2184,17 @@ func (b *ExecBackend) StartWorkerStaged(ctx context.Context, rt registry.Route, |
| return nil, release.StagedIdentity{}, fmt.Errorf("worker %s: %w", rt.Host, err) |
| } |
| id = got |
| + // A declined box that honours failure records or hold-ran markers |
| + // (Task 9) asks about the build just resolved, which is the build |
| + // exec'd (by its staged path), before it is published or anything is |
| + // prepared. A refusal runs nothing and comes back as it is. A recycle |
| + // onto the captured build (onto set) is no move onto the live link, |
| + // and is not asked. |
| + if b.LiveExecGate != nil { |
| + if err := b.LiveExecGate(rt, id.Version); err != nil { |
| + return nil, release.StagedIdentity{}, err |
| + } |
| + } |
| } else { |
| if err := release.VerifyStaged(*onto); err != nil { |
| return nil, release.StagedIdentity{}, fmt.Errorf("worker %s: not recycled onto %s: %w", rt.Host, onto.Version, err) |
diff --git a/internal/activator/exec_gate_test.go b/internal/activator/exec_gate_test.go| index 6919c6ab2..9de1f76b4 100644 |
| --- a/internal/activator/exec_gate_test.go |
| +++ b/internal/activator/exec_gate_test.go |
| @@ -3,6 +3,7 @@ package activator |
| import ( |
| "context" |
| "errors" |
| + "fmt" |
| "os" |
| "path/filepath" |
| "slices" |
| @@ -597,3 +598,306 @@ func TestTheExecGateIsAskedAtEachCgroupStart(t *testing.T) { |
| b.prepGone(t, b.gens()[0]) |
| }) |
| } |
| + |
| +// liveGate is a LiveExecGate for tests: it records every build it is asked |
| +// about and refuses the builds in refuse, as a declined box's failure holds |
| +// would (internal/host). |
| +type liveGate struct { |
| + mu sync.Mutex |
| + asked []string |
| + refuse map[string]bool |
| +} |
| + |
| +var errLiveHeld = errors.New("a failure record holds this build") |
| + |
| +func (g *liveGate) gate(_ registry.Route, build string) error { |
| + g.mu.Lock() |
| + defer g.mu.Unlock() |
| + g.asked = append(g.asked, build) |
| + if g.refuse[build] { |
| + return fmt.Errorf("%w: %s", errLiveHeld, build) |
| + } |
| + return nil |
| +} |
| + |
| +func (g *liveGate) builds() []string { |
| + g.mu.Lock() |
| + defer g.mu.Unlock() |
| + return slices.Clone(g.asked) |
| +} |
| + |
| +// repointLive points b's live/<host> at bin, as root's adopt pass does. |
| +func repointLive(t *testing.T, b *ExecBackend, host, bin string) { |
| + t.Helper() |
| + tmp := b.liveLink(host) + ".tmp" |
| + os.Remove(tmp) |
| + if err := os.Symlink(bin, tmp); err != nil { |
| + t.Error(err) |
| + return |
| + } |
| + if err := os.Rename(tmp, b.liveLink(host)); err != nil { |
| + t.Error(err) |
| + } |
| +} |
| + |
| +// flatLiveFixture is a flat row on a real ExecBackend whose live link names |
| +// v1, with v2 staged beside it. Each build writes its own $0 to out the |
| +// moment it runs, so a test can tell which bytes were exec'd and by which |
| +// path. |
| +func flatLiveFixture(t *testing.T) (b *ExecBackend, rt registry.Route, v1, v2, out string) { |
| + t.Helper() |
| + b = testExecBackend(t) |
| + b.RunDirOverride = t.TempDir() |
| + host := "live.example.com" |
| + out = filepath.Join(t.TempDir(), "argv0") |
| + v2 = stageArgv0Worker(t, b, host, "v2", out) |
| + v1 = stageArgv0Worker(t, b, host, "v1", out) |
| + rt = registry.Route{Host: host, Kind: "instance", Hibernate: true, |
| + DBPath: filepath.Join(b.DataDir, host+".db"), Socket: filepath.Join(b.RunDirOverride, host+".sock")} |
| + return b, rt, v1, v2, out |
| +} |
| + |
| +// Task 9 (Astra R1 #6, the R2/R3 declined residuals; plan review #6): on a |
| +// box that declined CARLOS_SWAP=overlap and honours failure records, the |
| +// flat live-link spawn resolves live/<host> ONCE, asks LiveExecGate about |
| +// that build at the launch (after the cgroup's preparation), and execs the |
| +// staged path it resolved, so a repoint landing between the gate and the |
| +// exec cannot start a build the gate never saw. A refusal starts nothing. |
| +// With the gate nil the spawn execs the live link exactly as before. |
| +func TestALiveExecGateGatesTheFlatLiveLinkSpawn(t *testing.T) { |
| + t.Run("gated build is the exec'd build", func(t *testing.T) { |
| + b, rt, v1, v2, out := flatLiveFixture(t) |
| + g := &liveGate{refuse: map[string]bool{"v2": true}} |
| + b.LiveExecGate = g.gate |
| + // Root repoints at held v2 after the gate has been asked about v1 and |
| + // before the exec. |
| + b.launchHook = onCall(1, func() { repointLive(t, b, rt.Host, v2) }) |
| + p, ver, err := b.spawnChildFrom(context.Background(), rt, rt.Socket, 0, "", cgroup.Limit{}) |
| + if err != nil { |
| + t.Fatalf("spawn = %v", err) |
| + } |
| + defer p.Stop(context.Background()) |
| + if got := strings.TrimSpace(waitForFile(t, out, 5*time.Second)); got != v1 { |
| + t.Fatalf("$0 = %q, want the staged v1 the gate was asked about (%s)", got, v1) |
| + } |
| + if ver != "v1" || !slices.Equal(g.builds(), []string{"v1"}) { |
| + t.Fatalf("spawn version %q, gate asked %v; want v1 and [v1]", ver, g.builds()) |
| + } |
| + }) |
| + t.Run("a held build starts nothing", func(t *testing.T) { |
| + b, rt, _, v2, out := flatLiveFixture(t) |
| + repointLive(t, b, rt.Host, v2) |
| + g := &liveGate{refuse: map[string]bool{"v2": true}} |
| + b.LiveExecGate = g.gate |
| + p, _, err := b.spawnChildFrom(context.Background(), rt, rt.Socket, 0, "", cgroup.Limit{}) |
| + if p != nil { |
| + _ = p.Stop(context.Background()) |
| + } |
| + if !errors.Is(err, errLiveHeld) || p != nil { |
| + t.Fatalf("spawn = %v, %v; want no process and the gate's refusal", p, err) |
| + } |
| + time.Sleep(200 * time.Millisecond) |
| + if _, err := os.Stat(out); err == nil { |
| + t.Fatal("held v2 ran") |
| + } |
| + if _, err := os.Stat(rt.Socket + ".control"); !os.IsNotExist(err) { |
| + t.Fatalf("control socket after a refusal: %v, want it unbound", err) |
| + } |
| + if !slices.Equal(g.builds(), []string{"v2"}) { |
| + t.Fatalf("gate asked %v, want [v2]", g.builds()) |
| + } |
| + }) |
| + t.Run("no gate is today's spawn", func(t *testing.T) { |
| + b, rt, _, _, out := flatLiveFixture(t) |
| + p, ver, err := b.spawnChildFrom(context.Background(), rt, rt.Socket, 0, "", cgroup.Limit{}) |
| + if err != nil { |
| + t.Fatalf("spawn = %v", err) |
| + } |
| + defer p.Stop(context.Background()) |
| + if got := strings.TrimSpace(waitForFile(t, out, 5*time.Second)); got != b.liveLink(rt.Host) || ver != "v1" { |
| + t.Fatalf("$0 = %q, version %q; want today's exec of %s, v1", got, ver, b.liveLink(rt.Host)) |
| + } |
| + }) |
| +} |
| + |
| +// Task 9: a nested live-link start asks LiveExecGate about the build the |
| +// live link names, right before root is asked to start the unit. A refusal |
| +// asks root for nothing. |
| +func TestALiveExecGateGatesTheNestedLiveLinkStart(t *testing.T) { |
| + ctl := &fakeControl{inv: "inv1"} |
| + b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| + g := &liveGate{refuse: map[string]bool{"v1": true}} |
| + b.LiveExecGate = g.gate |
| + _, _, err := b.Start(context.Background(), rt) |
| + if !errors.Is(err, errLiveHeld) { |
| + t.Fatalf("Start = %v, want the gate's refusal", err) |
| + } |
| + if strings.Contains(ctl.joined(), "start ") { |
| + t.Fatalf("root was asked to start the unit after the gate refused: %q", ctl.joined()) |
| + } |
| + if !slices.Equal(g.builds(), []string{"v1"}) { |
| + t.Fatalf("gate asked %v, want [v1]", g.builds()) |
| + } |
| + if _, err := os.Stat(rt.Socket + ".control"); !os.IsNotExist(err) { |
| + t.Fatalf("control socket after a refusal: %v, want it unbound", err) |
| + } |
| +} |
| + |
| +// Task 9: a worker spawned from the live link (onto nil) asks LiveExecGate |
| +// about the build it resolved, before its publication and its exec; a |
| +// refusal publishes and runs nothing. A recycle onto the captured build |
| +// (onto set) is not a move onto the live link, and is not asked. |
| +func TestALiveExecGateGatesAWorkerSpawnFromTheLiveLink(t *testing.T) { |
| + b := testExecBackend(t) |
| + host := "bot.example.com" |
| + out := filepath.Join(t.TempDir(), "argv0") |
| + v1 := stageArgv0Worker(t, b, host, "v1", out) |
| + stageArgv0Worker(t, b, host, "v2", out) |
| + g := &liveGate{refuse: map[string]bool{"v2": true}} |
| + b.LiveExecGate = g.gate |
| + published := 0 |
| + ctx := withBeforeExec(context.Background(), func(release.StagedIdentity) error { published++; return nil }) |
| + p, _, err := b.StartWorkerStaged(ctx, workerRoute(host), nil, nil) |
| + if p != nil { |
| + _ = p.Stop(context.Background()) |
| + } |
| + if !errors.Is(err, errLiveHeld) || p != nil || published != 0 { |
| + t.Fatalf("StartWorkerStaged = %v, %v (published %d); want the gate's refusal, nothing published or run", p, err, published) |
| + } |
| + time.Sleep(200 * time.Millisecond) |
| + if _, err := os.Stat(out); err == nil { |
| + t.Fatal("held v2 ran as the worker") |
| + } |
| + sum, err := release.FileSHA256(v1) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + onto := release.StagedIdentity{Version: "v1", StagedPath: v1, Digest: sum} |
| + p, id, err := b.StartWorkerStaged(ctx, workerRoute(host), nil, &onto) |
| + if err != nil { |
| + t.Fatalf("recycle onto the captured v1 = %v", err) |
| + } |
| + defer p.Stop(context.Background()) |
| + if id != onto || strings.TrimSpace(waitForFile(t, out, 5*time.Second)) != v1 { |
| + t.Fatalf("recycled %+v, want the captured v1", id) |
| + } |
| + if !slices.Equal(g.builds(), []string{"v2"}) { |
| + t.Fatalf("gate asked %v, want only the live-link spawn's [v2]", g.builds()) |
| + } |
| +} |
| + |
| +// Task 9 (brief (b)): a declined worker whose live link moves to a held |
| +// build is not stopped by the sweep's converge. The activator asks the gate |
| +// before it retires the old child, so a refusal leaves the worker running |
| +// its current build instead of stopping it into a refused spawn. |
| +func TestADeclinedWorkerIsNotConvergedOntoAHeldBuild(t *testing.T) { |
| + a, b, clk, _, ver := stagedActivator(t) |
| + g := &liveGate{refuse: map[string]bool{"v2": true}} |
| + a.SetLiveExecGate(g.gate) |
| + host := "bot.x.internal" |
| + if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + ver.Store("v2") |
| + a.sweepOnce() |
| + clk.advance(ConvergeGrace + time.Minute) |
| + a.sweepOnce() |
| + a.sweepOnce() |
| + time.Sleep(100 * time.Millisecond) |
| + if calls := b.calls(); len(calls) != 1 || a.entryRespawning(host) { |
| + t.Fatalf("spawns %v (respawning %v): the worker was converged onto held v2", calls, a.entryRespawning(host)) |
| + } |
| + if b.lastWorker().Exited() { |
| + t.Fatal("the converge stopped the worker serving v1") |
| + } |
| + if got := a.entrySpawnVersion(host); got != "v1" { |
| + t.Fatalf("spawn version %q, want v1", got) |
| + } |
| + if !slices.Contains(g.builds(), "v2") { |
| + t.Fatalf("gate asked %v, want v2 asked", g.builds()) |
| + } |
| + // Once the link names a build the gate allows, the converge runs as |
| + // before. |
| + ver.Store("v3") |
| + a.sweepOnce() |
| + clk.advance(ConvergeGrace + time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the converge onto v3", func() bool { return a.entrySpawnVersion(host) == "v3" && !a.entryRespawning(host) }) |
| +} |
| + |
| +// Task 9 (brief (c)): a declined worker that crashes while its live link |
| +// names a held build respawns its captured build, however long the drift |
| +// lasts: the recycle's converge is refused before anything is stopped. |
| +func TestADeclinedWorkerCrashRecyclesItsCapturedBuildWhenTheLiveLinkIsHeld(t *testing.T) { |
| + a, b, clk, _, ver := stagedActivator(t) |
| + a.SetLiveExecGate((&liveGate{refuse: map[string]bool{"v2": true}}).gate) |
| + host := "bot.x.internal" |
| + if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + ver.Store("v2") |
| + for i := 1; i <= 5; i++ { |
| + b.lastWorker().kill() |
| + clk.advance(sidecarRestartMax + time.Second) |
| + a.sweepOnce() |
| + waitForRestart(t, a, b.fakeBackend, host, i+1) |
| + } |
| + for i, c := range b.calls()[1:] { |
| + if c == nil || *c != fakeIdentity("v1") { |
| + t.Fatalf("respawn %d asked for %v; want the captured v1 every time", i+1, c) |
| + } |
| + } |
| + if got := a.entrySpawnVersion(host); got != "v1" { |
| + t.Fatalf("spawn version %q, want v1", got) |
| + } |
| +} |
| + |
| +// Task 9 (brief (c)): a worker with no captured build (a backend without |
| +// the staged capability) whose live link names a held build is not |
| +// respawned from it: a crashed one stays down until a wake starts it, and |
| +// one whose max_run falls due keeps running. |
| +func TestADeclinedUncapturedWorkerIsNotRespawnedOntoAHeldBuild(t *testing.T) { |
| + t.Run("crashed", func(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Hour) |
| + a.liveVer = func(string) (string, error) { return "v2", nil } |
| + a.SetLiveExecGate((&liveGate{refuse: map[string]bool{"v2": true}}).gate) |
| + host := "bot.x.internal" |
| + if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + b.lastWorker().kill() |
| + for range 3 { |
| + a.sweepOnce() |
| + clk.advance(sidecarRestartMax + time.Second) |
| + } |
| + time.Sleep(100 * time.Millisecond) |
| + if n := b.workerStarts(); n != 1 || a.entryRespawning(host) { |
| + t.Fatalf("workerStarts = %d (respawning %v): respawned onto held v2", n, a.entryRespawning(host)) |
| + } |
| + if !a.Owns(host) { |
| + t.Fatal("the entry went: it stays, down, for a wake to start") |
| + } |
| + }) |
| + t.Run("max_run", func(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Hour) |
| + a.liveVer = func(string) (string, error) { return "v2", nil } |
| + a.SetLiveExecGate((&liveGate{refuse: map[string]bool{"v2": true}}).gate) |
| + host := "bot.x.internal" |
| + rt := alwaysOnWorkerRoute(host) |
| + rt.MaxRunSeconds = 30 |
| + if err := a.WakeWorker(rt); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + for range 3 { |
| + clk.advance(31 * time.Second) |
| + a.sweepOnce() |
| + } |
| + time.Sleep(100 * time.Millisecond) |
| + if n := b.workerStarts(); n != 1 || a.entryRespawning(host) { |
| + t.Fatalf("workerStarts = %d (respawning %v): recycled onto held v2", n, a.entryRespawning(host)) |
| + } |
| + if b.lastWorker().Exited() { |
| + t.Fatal("the running child was stopped by a recycle the gate refused") |
| + } |
| + }) |
| +} |
diff --git a/internal/activator/genbackend.go b/internal/activator/genbackend.go| index fd9e0fd51..7025d74bd 100644 |
| --- a/internal/activator/genbackend.go |
| +++ b/internal/activator/genbackend.go |
| @@ -106,14 +106,70 @@ func runExecGate(ctx context.Context) error { |
| // preparation. Nil when ctx carries no gate and no test seam is set, so a |
| // live-link spawn hands the manager no hook and starts exactly as it did. |
| func (b *ExecBackend) launchGate(ctx context.Context, host string) func() error { |
| - if _, ok := ctx.Value(execGateKey{}).(func() error); !ok && b.launchHook == nil { |
| + return b.launchGateWith(ctx, host, nil) |
| +} |
| + |
| +// launchGateWith is launchGate with live, a live-link spawn's LiveExecGate |
| +// check for the build it resolved (Task 9), asked after ctx's exec gate at |
| +// the same point. A nil live is launchGate exactly, so a spawn on a box with |
| +// no LiveExecGate hands the manager the hook it always did. |
| +func (b *ExecBackend) launchGateWith(ctx context.Context, host string, live func() error) func() error { |
| + if _, ok := ctx.Value(execGateKey{}).(func() error); !ok && b.launchHook == nil && live == nil { |
| return nil |
| } |
| return func() error { |
| if b.launchHook != nil { |
| b.launchHook(host) |
| } |
| - return runExecGate(ctx) |
| + if err := runExecGate(ctx); err != nil { |
| + return err |
| + } |
| + if live != nil { |
| + return live() |
| + } |
| + return nil |
| + } |
| +} |
| + |
| +// liveExecGateSetter is a backend that honours a live-link exec gate |
| +// (ExecBackend.LiveExecGate), or forwards it to one that does |
| +// (SplitBackend). |
| +type liveExecGateSetter interface { |
| + SetLiveExecGate(gate func(rt registry.Route, build string) error) |
| +} |
| + |
| +var ( |
| + _ liveExecGateSetter = (*ExecBackend)(nil) |
| + _ liveExecGateSetter = (*SplitBackend)(nil) |
| +) |
| + |
| +// SetLiveExecGate sets LiveExecGate. Call it before the backend spawns |
| +// anything, as Service.SetLiveExecGate does. |
| +func (b *ExecBackend) SetLiveExecGate(gate func(rt registry.Route, build string) error) { |
| + b.LiveExecGate = gate |
| +} |
| + |
| +// SetLiveExecGate forwards to the exec half: unit-backed routes belong to |
| +// systemd, and the rollout never holds them (rollout.Managed). |
| +func (b *SplitBackend) SetLiveExecGate(gate func(rt registry.Route, build string) error) { |
| + if s, ok := b.Exec.(liveExecGateSetter); ok { |
| + s.SetLiveExecGate(gate) |
| + } |
| +} |
| + |
| +// SetLiveExecGate installs gate as the backend's LiveExecGate and as the |
| +// activator's own check at a worker's converge and recycle, which stop the |
| +// old child before the backend's spawn could refuse the new one. Only a box |
| +// that declined CARLOS_SWAP=overlap and honours failure records or hold-ran |
| +// markers calls it (internal/host, honourDeclinedFailures), and before |
| +// Start, like Planner. gate is asked under a.mu, so it must answer from |
| +// memory, the row and the live link, and never call back into Service. |
| +func (a *Service) SetLiveExecGate(gate func(rt registry.Route, build string) error) { |
| + a.mu.Lock() |
| + a.liveExecGate = gate |
| + a.mu.Unlock() |
| + if s, ok := a.backend.(liveExecGateSetter); ok { |
| + s.SetLiveExecGate(gate) |
| } |
| } |
diff --git a/internal/activator/workerbuild.go b/internal/activator/workerbuild.go| index 9c1e75127..257915203 100644 |
| --- a/internal/activator/workerbuild.go |
| +++ b/internal/activator/workerbuild.go |
| @@ -175,6 +175,13 @@ func (a *Service) startWorker(ctx context.Context, rt registry.Route, env map[st |
| // coalesce never converges. Either way, a converge that is not admitted |
| // recycles the captured build, and the drift keeps its age for a later |
| // sweep. |
| +// |
| +// On a box that declined CARLOS_SWAP=overlap and honours failure records or |
| +// hold-ran markers, a move onto the live link is also asked of the live |
| +// exec gate first (SetLiveExecGate, Task 9), before anything is stopped: a |
| +// converge it refuses recycles the captured build instead, and a worker |
| +// with no captured build is not respawned from the link, so a live child |
| +// keeps running and a dead one stays down until a wake starts it. |
| func (a *Service) recycleWorkerLocked(e *actEntry, stopOld Proc, why string) { |
| if hook := a.CoalesceRecycle; hook != nil && hook(e.host) { |
| log.Printf("activator: worker %s %s; a queued release attempt makes the spawn instead of a recycle", e.host, why) |
| @@ -191,6 +198,10 @@ func (a *Service) recycleWorkerLocked(e *actEntry, stopOld Proc, why string) { |
| log.Printf("activator: worker %s %s; live/%s names %s, but the release scheduler owns build changes, so it recycles its captured build %s", e.host, why, e.host, cur, e.spawnVersion) |
| case a.convergeBudget <= 0: |
| log.Printf("activator: worker %s %s; live/%s names %s, but this sweep's converge budget is spent, so it recycles its captured build %s and converges on a later sweep", e.host, why, e.host, cur, e.spawnVersion) |
| + case a.liveGateRefusesLocked(e, cur, "it recycles its captured build "+e.spawnVersion): |
| + // A declined box that honours a record or marker holding cur |
| + // (Task 9): the captured build is recycled, as for a converge |
| + // not admitted, and the drift keeps its age. |
| default: |
| a.convergeBudget-- |
| log.Printf("activator: worker %s %s; it serves %s but live/%s has named %s for at least %s, so it converges onto the new build instead of recycling", e.host, why, e.spawnVersion, e.host, cur, ConvergeGrace) |
| @@ -205,19 +216,45 @@ func (a *Service) recycleWorkerLocked(e *actEntry, stopOld Proc, why string) { |
| a.uncapturedSaid = map[string]bool{} |
| } |
| a.uncapturedSaid[e.host] = true |
| - left := "the running child keeps running" |
| - if stopOld == nil { |
| - left = "it stays down until a release attempt or a wake starts it" |
| - } |
| + left := uncapturedLeft(stopOld) |
| log.Printf("activator: worker %s %s; it has no captured build, and the release scheduler owns its recycles, so it is not respawned from live/%s: %s", e.host, why, e.host, left) |
| } |
| return |
| + } else if a.liveLinkRefusedLocked(e, uncapturedLeft(stopOld)) { |
| + // No captured build, and the build the live link names is one this |
| + // declined box will not start (Task 9): asked before anything is |
| + // stopped, so a live child keeps running and a dead one stays down |
| + // until a wake starts it, as on an armed box. |
| + return |
| } else { |
| log.Printf("activator: worker %s %s; no captured build to recycle onto, so it respawns from live/%s", e.host, why, e.host) |
| } |
| a.respawnWorkerLocked(e, stopOld, why, onto) |
| } |
| +// liveLinkRefusedLocked is liveGateRefusesLocked for the build live/<host> |
| +// names now. With no live exec gate (every box but a declined one that |
| +// honours something) it reads nothing and refuses nothing, so the recycle is |
| +// today's. A link that cannot be read is left to the spawn's own gate |
| +// (StartWorkerStaged). The caller holds a.mu. |
| +func (a *Service) liveLinkRefusedLocked(e *actEntry, left string) bool { |
| + if a.liveExecGate == nil { |
| + return false |
| + } |
| + cur, err := a.liveVer(e.host) |
| + return err == nil && a.liveGateRefusesLocked(e, cur, left) |
| +} |
| + |
| +// uncapturedLeft is what becomes of a worker with no captured build whose |
| +// recycle is not made: a live child (stopOld) keeps running, a dead one |
| +// stays down. |
| +func uncapturedLeft(stopOld Proc) string { |
| + if stopOld == nil { |
| + return "it stays down until a release attempt or a wake starts it" |
| + } |
| + return "the running child keeps running" |
| +} |
| + |
| // WorkerReportedVersion is what host's worker last said it runs, over the |
| // agent socket: display metadata only (spec §7.9). It may be empty or an |
| // app-chosen string, and it never identifies the executable. "" when no |
diff --git a/internal/host/claimproduction.go b/internal/host/claimproduction.go| index 9f903b913..2fe2926c3 100644 |
| --- a/internal/host/claimproduction.go |
| +++ b/internal/host/claimproduction.go |
| @@ -6,6 +6,7 @@ import ( |
| "os" |
| "path/filepath" |
| "runtime" |
| + "strings" |
| "sync" |
| "time" |
| @@ -449,6 +450,35 @@ func claimServingLaunchSource(backend *activator.ExecBackend, tokens claimAdminT |
| return ClaimServingLaunch{}, err |
| } |
| snapshot, err := backend.SnapshotClaimLaunch(ctx, rt) |
| + if err == nil { |
| + err = declinedClaimGate(backend.OptDir, rt, snapshot) |
| + } |
| return ClaimServingLaunch{Identity: snapshot.Identity, Executable: snapshot.Executable, ExecutableSHA256: snapshot.SHA256, Args: snapshot.Args, Env: snapshot.Env, Roles: snapshot.Roles, Version: snapshot.Version, Limit: snapshot.Limit}, err |
| } |
| } |
| + |
| +// declinedClaimGate is a declined box's launch gate (failureHolds.gate) for |
| +// a claims-backed wake (Task 9, plan review #6). Task 1 makes a claims-on |
| +// box decline CARLOS_SWAP=overlap, so this is a declined box's wake, and its |
| +// launch runs the bytes the snapshot hashed from the live link (the launch |
| +// hashes the executable again before it execs). The snapshot names the |
| +// version from a second read of the link, so the link is resolved once more |
| +// here (release.ResolveLive, one read of the link and a hash of the file it |
| +// names): only a version whose bytes are the snapshot's is gated, and a |
| +// repoint between the snapshot's two reads is a changed claim |
| +// (hostbox.ErrChanged), never a start. With nothing honoured (every box but |
| +// a declined one with records or markers) it is nil and reads nothing. |
| +func declinedClaimGate(optDir string, rt registry.Route, s activator.ClaimLaunchSnapshot) error { |
| + h := heldHolds() |
| + if h == nil { |
| + return nil |
| + } |
| + id, err := release.ResolveLive(optDir, rt.Host) |
| + if err != nil { |
| + return err |
| + } |
| + if id.Version != s.Version || !strings.EqualFold(id.Digest, s.SHA256) { |
| + return hostbox.ErrChanged |
| + } |
| + return h.gate(rt, s.Version) |
| +} |
diff --git a/internal/host/rollout_test.go b/internal/host/rollout_test.go| index 77041afe8..96cdaa83a 100644 |
| --- a/internal/host/rollout_test.go |
| +++ b/internal/host/rollout_test.go |
| @@ -28,6 +28,12 @@ import ( |
| // generations switch as gens says (runAgent sets it from swapArming before |
| // New). The package's rollout state is reset when the test ends. |
| func rolloutFixture(t *testing.T, gens bool) (*activator.Service, *registry.Registry, string) { |
| + t.Helper() |
| + return rolloutFixtureWith(t, gens, &activator.SplitBackend{}) |
| +} |
| + |
| +// rolloutFixtureWith is rolloutFixture over backend. |
| +func rolloutFixtureWith(t *testing.T, gens bool, backend activator.Backend) (*activator.Service, *registry.Registry, string) { |
| t.Helper() |
| prev := activator.GenerationsEnabled() |
| activator.SetGenerations(gens) |
| @@ -39,7 +45,7 @@ func rolloutFixture(t *testing.T, gens bool) (*activator.Service, *registry.Regi |
| t.Fatal(err) |
| } |
| t.Cleanup(func() { reg.Close() }) |
| - svc := activator.New(activator.Config{Bucket: "b", Idle: time.Hour}, &activator.SplitBackend{}, reg) |
| + svc := activator.New(activator.Config{Bucket: "b", Idle: time.Hour}, backend, reg) |
| return svc, reg, dd |
| } |
diff --git a/internal/host/rolloutdeclined.go b/internal/host/rolloutdeclined.go| index 2390885c6..93cc730a7 100644 |
| --- a/internal/host/rolloutdeclined.go |
| +++ b/internal/host/rolloutdeclined.go |
| @@ -31,9 +31,10 @@ const refuseNoSwaps = "this box cannot run two builds side by side, which this i |
| // migration failed must not run again on the live link. So, with generations |
| // off and no scheduler, the box keeps today's paths except that a managed |
| // wake onto a build the book holds for that instance is refused (Planner), |
| -// and the converge leg, the adopt bounce, the restart leg and config sync's |
| -// restart leave such an instance running rather than stop it into a refused |
| -// wake. |
| +// and refused again at the real launch, against the build that launch runs |
| +// (gate, Task 9); and the converge leg, the adopt bounce, the restart leg and |
| +// config sync's restart leave such an instance running rather than stop it |
| +// into a refused wake. The book holds records and hold-ran markers. |
| // |
| // The book is read once, at arming, and written at most once then: when an |
| // armed run left intents in it, the records reconciled from them are saved |
| @@ -52,9 +53,9 @@ type failureHolds struct { |
| } |
| // declinedHolds is set by honourDeclinedFailures, before any leg is built, |
| -// and only when the book holds at least one record. Nil: the flag is off, |
| -// armed, refusing, or declined with no records, and every leg is as its own |
| -// branch says. |
| +// and only when the book holds at least one record or hold-ran marker. Nil: |
| +// the flag is off, armed, refusing, or declined with an empty book, and |
| +// every leg is as its own branch says. |
| var declinedHolds atomic.Pointer[failureHolds] |
| // heldHolds is declinedHolds' value. |
| @@ -62,10 +63,11 @@ func heldHolds() *failureHolds { return declinedHolds.Load() } |
| // honourDeclinedFailures is armRollout's branch for a box that declined the |
| // flag (spec §§6.4, 8.4; final review F6). An absent book, or one with no |
| -// records, leaves everything exactly as today: no hook, no leg changed. An |
| -// unreadable one fails closed as refuseManaged does for an armed box, since |
| -// any record in it could be a hold-down. A readable one with records is |
| -// honoured, read-only, through the Planner and the legs (failureHolds). |
| +// records and no hold-ran markers, leaves everything exactly as today: no |
| +// hook, no leg changed, LiveExecGate nil. An unreadable one fails closed as |
| +// refuseManaged does for an armed box, since any record in it could be a |
| +// hold-down. A readable one with records or markers is honoured, read-only, |
| +// through the Planner, the launch gate and the legs (failureHolds). |
| func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, optDir string) { |
| book, err := rollout.OpenFailureBook(dd) |
| if err != nil { |
| @@ -83,54 +85,97 @@ func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, |
| log.Printf("agent: %s=overlap declined: save the failure book after reconciling it: %v", rollout.SwapEnv, err) |
| } |
| } |
| - recs := book.List() |
| - if len(recs) == 0 { |
| + // Records or hold-ran markers (Task 9): a marker alone still bars the good |
| + // build from an instance whose database a hold build may have migrated. |
| + recs, marks := len(book.List()), book.HoldRanCount() |
| + if recs == 0 && marks == 0 { |
| return |
| } |
| h := &failureHolds{book: book, get: reg.Get, optDir: optDir} |
| declinedHolds.Store(h) |
| svc.Planner = h.plan |
| - log.Printf("agent: %s=overlap declined, but %d failure record(s) from an armed run are honoured: "+ |
| - "no managed instance wakes, or is stopped, onto a build they hold for it", rollout.SwapEnv, len(recs)) |
| + // And at the real launch, with the build actually started (Task 9): the |
| + // plan reads the live link before the lease and the restore, and root |
| + // can repoint it at a held build in between. |
| + svc.SetLiveExecGate(h.gate) |
| + log.Printf("agent: %s=overlap declined, but %d failure record(s) and %d hold-ran marker(s) from an armed run are honoured: "+ |
| + "no managed instance wakes, starts, or is stopped, onto a build they hold for it", rollout.SwapEnv, recs, marks) |
| } |
| -// heldBuild is the build a failure record holds rt off, and whether that |
| -// instance is held down after a failed hold attempt; "" when nothing holds |
| -// it. Only a managed route can be held: the scheduler records nothing else. |
| -// Both the installed build (root's stamp) and the build live/<host> names |
| -// are asked, because a wake execs the live link and root repoints it before |
| -// it stamps the row; an unreadable link is left to the stamp. |
| +// refusal is the wake refusal for starting build on rt, "" when it may |
| +// start. Only a managed route can be held: the scheduler records nothing |
| +// else. |
| // |
| -// The rule is the armed planner's (rollout.decideWake, spec §8.5), with the |
| -// one difference that a declined box cannot boot the good build instead, so |
| -// it refuses where the armed box would recover. A hold-down holds whatever |
| -// the instance served. A plain record holds a build only when the instance |
| -// has a good build to fall back to (a build it has served is never held by a |
| -// record alone); an instance with none boots its installed build as today, |
| -// because refusing it would turn one sibling's failure into an outage for |
| -// every sleeping sibling. |
| -func (h *failureHolds) heldBuild(rt registry.Route) (build string, heldDown bool) { |
| +// The rule is the armed planner's (rollout.decideWake and decideHoldRan, |
| +// spec §8.5, §6.4), with the one difference that a declined box cannot boot |
| +// the good build instead, so it refuses where the armed box would recover. |
| +// A hold-down holds whatever the instance served, in that build's words. |
| +// Then, with a hold-ran marker of rt's app standing for the instance (Task |
| +// 9; Astra R3 N5/N7, ruling 205): |
| +// - the marked build may start, unless the instance has a good build and |
| +// a record holds the marked build for it. A record never holds a build |
| +// the instance served, so a marker naming the installed good build |
| +// itself lets it start (the marked-build rule); |
| +// - the good build is refused: an installed good build is no proof of a |
| +// signed rollback, since root's automatic sibling repoint installs it |
| +// too, and only the pointer pass clears the marker on one (Task 10); |
| +// - any other build a record holds for the instance is refused. |
| +// |
| +// All three refuse in HeldDownRefusal's words for the marked build, as the |
| +// armed planner does, so "roll back or retry" stays true here. Without a |
| +// marker, a plain record holds a build only when the instance has a good |
| +// build to fall back to (a build it has served is never held by a record |
| +// alone); an instance with none boots its installed build as today, because |
| +// refusing it would turn one sibling's failure into an outage for every |
| +// sleeping sibling. It reads only memory, so the activator may ask it under |
| +// its own lock (gate). |
| +func (h *failureHolds) refusal(rt registry.Route, build string) string { |
| + if !rollout.Managed(rt) || build == "" { |
| + return "" |
| + } |
| + acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| + hasGood := rt.GoodVersion != "" |
| + if h.book.HeldDown(acct, app, build, host) { |
| + return rollout.HeldDownRefusal(build) |
| + } |
| + if mark, ok := h.book.HoldRanOn(host); ok && mark.Account == acct && mark.App == app { |
| + isGood := hasGood && build == rt.GoodVersion |
| + switch { |
| + case build == mark.Build && (!isGood || rt.Version == mark.Build): |
| + if hasGood && h.book.Blocks(acct, app, build, host, isGood) { |
| + return rollout.HeldDownRefusal(mark.Build) |
| + } |
| + return "" |
| + case isGood, hasGood && h.book.Blocks(acct, app, build, host, false): |
| + return rollout.HeldDownRefusal(mark.Build) |
| + } |
| + return "" |
| + } |
| + if hasGood && h.book.Blocks(acct, app, build, host, build == rt.GoodVersion) { |
| + return refuseNoSwaps |
| + } |
| + return "" |
| +} |
| + |
| +// heldBuild is the build the book holds rt off and its refusal (refusal), |
| +// "" when nothing holds it. Both the installed build (root's stamp) and the |
| +// build live/<host> names are asked, because a wake execs the live link and |
| +// root repoints it before it stamps the row; an unreadable link is left to |
| +// the stamp. |
| +func (h *failureHolds) heldBuild(rt registry.Route) (build, refuse string) { |
| if !rollout.Managed(rt) { |
| - return "", false |
| + return "", "" |
| } |
| builds := []string{rt.Version} |
| if live, err := release.LiveVersion(h.optDir, rt.Host); err == nil && live != "" && live != rt.Version { |
| builds = append(builds, live) |
| } |
| - acct := release.RouteAccount(rt) |
| - hasGood := rt.GoodVersion != "" |
| for _, b := range builds { |
| - if b == "" { |
| - continue |
| - } |
| - if h.book.HeldDown(acct, rt.App, b, rt.Host) { |
| - return b, true |
| - } |
| - if hasGood && h.book.Blocks(acct, rt.App, b, rt.Host, b == rt.GoodVersion) { |
| - return b, false |
| + if r := h.refusal(rt, b); r != "" { |
| + return b, r |
| } |
| } |
| - return "", false |
| + return "", "" |
| } |
| // plan is the declined box's Planner: a managed wake onto a held build is |
| @@ -138,15 +183,29 @@ func (h *failureHolds) heldBuild(rt registry.Route) (build string, heldDown bool |
| // other wake gets the zero plan, which is today's wake from the live link. |
| // rt is the row doWake has just re-read (Service.Routes). |
| func (h *failureHolds) plan(_ context.Context, rt registry.Route) activator.WakePlan { |
| - build, heldDown := h.heldBuild(rt) |
| - switch { |
| - case build == "": |
| - return activator.WakePlan{} |
| - case heldDown: |
| - return activator.WakePlan{Refuse: rollout.HeldDownRefusal(build)} |
| - default: |
| - return activator.WakePlan{Refuse: refuseNoSwaps} |
| + _, refuse := h.heldBuild(rt) |
| + return activator.WakePlan{Refuse: refuse} |
| +} |
| + |
| +// gate is the declined box's LiveExecGate (Task 9; Astra R1 #6, the R2/R3 |
| +// declined residuals): the plan's refusal asked again at the real launch, |
| +// about the build that launch is about to run, which root may have |
| +// repointed the live link to since the plan. The backend asks it at a flat |
| +// or nested instance's live-link start and a worker's spawn from the live |
| +// link, the activator before a worker's converge or a recycle's move onto |
| +// the live link, and a claims-backed wake once its snapshot has resolved the |
| +// link (claimServingLaunchSource). An empty build (a link outside the |
| +// version store, or one that could not be read) is left to the stamp, as |
| +// heldBuild leaves it. It reads only memory and rt, and writes nothing: the |
| +// activator asks it under its own lock. |
| +func (h *failureHolds) gate(rt registry.Route, build string) error { |
| + if build == "" { |
| + build = rt.Version |
| + } |
| + if r := h.refusal(rt, build); r != "" { |
| + return errors.New(r) |
| } |
| + return nil |
| } |
| // heldRoute reports whether rt is held, logging it once per leg and build. |
| @@ -156,7 +215,7 @@ func (h *failureHolds) heldRoute(leg string, rt registry.Route) bool { |
| return false |
| } |
| if _, said := h.said.LoadOrStore(leg+"\x00"+rt.Host+"\x00"+build, true); !said { |
| - log.Printf("agent: %s: leaving %s running: a failure record from an armed run holds build %s for it, and %s=overlap is declined on this box", |
| + log.Printf("agent: %s: leaving %s running: a failure record or hold-ran marker from an armed run holds build %s for it, and %s=overlap is declined on this box", |
| leg, rt.Host, build, rollout.SwapEnv) |
| } |
| return true |
diff --git a/internal/host/rolloutdeclined_test.go b/internal/host/rolloutdeclined_test.go| index 1fdcaeb08..14fb1efae 100644 |
| --- a/internal/host/rolloutdeclined_test.go |
| +++ b/internal/host/rolloutdeclined_test.go |
| @@ -475,3 +475,261 @@ func TestDeclinedBoxWithAnUnreadableBookFailsClosed(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// liveLaunchBackend is fakeWakeBackend's lease and restore around a REAL |
| +// ExecBackend start, so a wake runs the production live-link spawn. before, |
| +// when set, runs between the plan and the start: root's repoint landing |
| +// there. |
| +type liveLaunchBackend struct { |
| + fakeWakeBackend |
| + eb *activator.ExecBackend |
| + before func() |
| +} |
| + |
| +func (b *liveLaunchBackend) Start(ctx context.Context, rt registry.Route) (activator.Proc, string, error) { |
| + if b.before != nil { |
| + b.before() |
| + } |
| + return b.eb.Start(ctx, rt) |
| +} |
| + |
| +func (b *liveLaunchBackend) LiveVersion(host string) (string, error) { return b.eb.LiveVersion(host) } |
| + |
| +func (b *liveLaunchBackend) SetLiveExecGate(g func(registry.Route, string) error) { |
| + b.eb.SetLiveExecGate(g) |
| +} |
| + |
| +// Task 9 (Astra R1 #6, the R2/R3 declined residuals; plan review #6), brief |
| +// (a): the plan-time refusal is repeated at the real launch. The wake plans |
| +// with the live link on the good build v1, which no record holds; root |
| +// repoints the link at v2, which a record holds for this instance, before |
| +// the start. The start resolves the link, asks the gate about v2, and execs |
| +// nothing. |
| +func TestDeclinedBoxRefusesAHeldBuildAtLaunch(t *testing.T) { |
| + prev := activator.GenerationsEnabled() |
| + activator.SetGenerations(false) |
| + t.Cleanup(func() { activator.SetGenerations(prev) }) |
| + resetRollout(t) |
| + env := newFastFixture(t) |
| + t.Setenv("CARLOS_OPT_DIR", env.optDir) |
| + host := "warm.test" |
| + run := t.TempDir() |
| + rt := registry.Route{Host: host, Kind: "instance", App: "fastapp", Account: "bab", Channel: "edge", |
| + Status: "ready", Hibernate: true, Version: "v1", GoodVersion: "v1", GoodDigest: "sha256:v1", |
| + Socket: filepath.Join(run, host+".sock"), DBPath: filepath.Join(env.dataDir, host+".db")} |
| + if err := env.reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + env.stageLive(t, host, "v1") |
| + mark := filepath.Join(t.TempDir(), "v2.ran") |
| + if err := release.StageArtifact(env.optDir, "bab", "v2", "fastapp", []byte("#!/bin/sh\necho ran > '"+mark+"'\nexec sleep 30\n")); err != nil { |
| + t.Fatal(err) |
| + } |
| + writeFailureBook(t, env.dataDir, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "fastapp", Build: "v2", Host: "sibling.test"}}, nil) |
| + eb := &activator.ExecBackend{Cfg: activator.Config{Bucket: "test-bucket", Idle: time.Hour}, |
| + OptDir: env.optDir, DataDir: t.TempDir(), RunDirOverride: run} |
| + backend := &liveLaunchBackend{eb: eb} |
| + svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg) |
| + svc.Routes = env.reg |
| + t.Cleanup(func() { |
| + sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + svc.Shutdown(sctx) |
| + }) |
| + declineWithBook(t, svc, env.reg, env.dataDir) |
| + if eb.LiveExecGate == nil { |
| + t.Fatal("no LiveExecGate: the launch execs whatever the live link names by then") |
| + } |
| + if p := svc.Planner(context.Background(), rt); !isTodaysWake(p) { |
| + t.Fatalf("plan with the link on v1 = %+v, want today's wake", p) |
| + } |
| + backend.before = func() { |
| + if err := release.PointLive(env.optDir, host, "bab", "v2", "fastapp"); err != nil { |
| + t.Error(err) |
| + } |
| + } |
| + rec := httptest.NewRecorder() |
| + proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) |
| + if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) { |
| + t.Fatal("the activator did not take the request") |
| + } |
| + if rec.Code == http.StatusNoContent { |
| + t.Fatal("the wake onto held v2 was served") |
| + } |
| + time.Sleep(200 * time.Millisecond) |
| + if _, err := os.Stat(mark); err == nil { |
| + t.Fatal("held v2 was exec'd: the gate at the launch did not see the repoint") |
| + } |
| +} |
| + |
| +// Task 9 rule 2 and Correction 3: a declined box honours hold-ran markers |
| +// as the armed planner does (decideHoldRan, planCheck), refusing where the |
| +// armed box would refuse. A book holding only a marker is honoured. The |
| +// gate at the launch answers as the plan does. |
| +func TestDeclinedBoxHonoursHoldRanMarkers(t *testing.T) { |
| + held := rollout.HeldDownRefusal("v2") |
| + route := func(host, installed, good string) registry.Route { |
| + rt := managedRoute(host, installed) |
| + if good != "" { |
| + rt.GoodVersion, rt.GoodDigest = good, "sha256:"+good |
| + } |
| + return rt |
| + } |
| + acct := release.RouteAccount(managedRoute("x", "")) |
| + mark := func(t *testing.T, dd string, marks map[string]string) { |
| + t.Helper() |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for host, build := range marks { |
| + app := "blog" |
| + if host == "other.example.com" { |
| + app = "shop" // another app's marker under the same host name |
| + } |
| + if err := b.MarkHoldRan(host, rollout.HoldRan{Account: acct, App: app, Build: build}); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + } |
| + type want struct { |
| + rt registry.Route |
| + refuse string |
| + } |
| + check := func(t *testing.T, svc *activator.Service, cases []want) { |
| + t.Helper() |
| + h := heldHolds() |
| + for _, tc := range cases { |
| + p := svc.Planner(context.Background(), tc.rt) |
| + if tc.refuse == "" && !isTodaysWake(p) { |
| + t.Errorf("%s (installed %s, good %s): planned %+v, want today's wake", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, p) |
| + } |
| + if tc.refuse != "" && p.Refuse != tc.refuse { |
| + t.Errorf("%s (installed %s, good %s): refusal %q, want %q", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, p.Refuse, tc.refuse) |
| + } |
| + got := "" |
| + if err := h.gate(tc.rt, tc.rt.Version); err != nil { |
| + got = err.Error() |
| + } |
| + if got != tc.refuse { |
| + t.Errorf("%s (installed %s, good %s): gate %q, want %q", tc.rt.Host, tc.rt.Version, tc.rt.GoodVersion, got, tc.refuse) |
| + } |
| + } |
| + } |
| + t.Run("only a marker", func(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + mark(t, dd, map[string]string{"m.example.com": "v2", "mg.example.com": "v1", "other.example.com": "v2"}) |
| + before := snapshotBook(t, dd) |
| + declineWithBook(t, svc, reg, dd) |
| + if heldHolds() == nil || svc.Planner == nil { |
| + t.Fatal("a book holding only a marker was not honoured") |
| + } |
| + if r := rolloutStatusProbe(); r == nil || r.FailureBook != rolloutBookHonoured { |
| + t.Fatalf("status = %+v, want the book %q", r, rolloutBookHonoured) |
| + } |
| + check(t, svc, []want{ |
| + // The installed good build is refused while a marker of another |
| + // build stands: only a signed rollback clears it (Task 10). |
| + {route("m.example.com", "v1", "v1"), held}, |
| + // The marked build starts: no record holds it. |
| + {route("m.example.com", "v2", "v1"), ""}, |
| + // Any other build starts. |
| + {route("m.example.com", "v3", "v1"), ""}, |
| + // With no good build there is nothing to refuse in its favour. |
| + {route("m.example.com", "v2", ""), ""}, |
| + // A marker naming the installed good build itself lets it start. |
| + {route("mg.example.com", "v1", "v1"), ""}, |
| + // Another app's marker under the host's name says nothing here. |
| + {route("other.example.com", "v1", "v1"), ""}, |
| + {route("free.example.com", "v1", "v1"), ""}, |
| + }) |
| + before.unchanged(t, dd) |
| + }) |
| + t.Run("a marker and records", func(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{ |
| + {Account: acct, App: "blog", Build: "v1", Host: "sib.example.com"}, |
| + {Account: acct, App: "blog", Build: "v2", Host: "sib.example.com"}, |
| + {Account: acct, App: "blog", Build: "v3", Host: "sib.example.com"}, |
| + {Account: acct, App: "blog", Build: "v4", Host: "sib.example.com", HeldDown: []string{"m.example.com"}}, |
| + }, nil) |
| + mark(t, dd, map[string]string{"m.example.com": "v2", "mg.example.com": "v1"}) |
| + declineWithBook(t, svc, reg, dd) |
| + check(t, svc, []want{ |
| + // The marked build, blocked by a record while the route has a good |
| + // build. |
| + {route("m.example.com", "v2", "v1"), held}, |
| + // The marked build that is the installed good build: served, so a |
| + // sibling's record does not hold it. |
| + {route("mg.example.com", "v1", "v1"), ""}, |
| + // With no good build a record alone refuses nothing. |
| + {route("m.example.com", "v2", ""), ""}, |
| + // Any other build a record blocks. |
| + {route("m.example.com", "v3", "v1"), held}, |
| + // A hold-down still comes first, in its own build's words. |
| + {route("m.example.com", "v4", "v1"), rollout.HeldDownRefusal("v4")}, |
| + {route("m.example.com", "v5", "v1"), ""}, |
| + }) |
| + }) |
| +} |
| + |
| +// Task 9: a claims-backed wake on a declined box is gated against the build |
| +// its launch snapshot resolved from the live link, before anything is asked |
| +// of root. A box that honours nothing is today's. |
| +func TestDeclinedBoxGatesAClaimsBackedWake(t *testing.T) { |
| + f := newAdminTokenFixture(t, "box-1") |
| + if err := os.WriteFile(f.envPath(), []byte("CARLOS_ADMIN_TOKEN=tok-0123456789abcdef0123456789abcdef\n"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt := f.rt |
| + rt.Kind, rt.Status, rt.Channel, rt.Account, rt.App, rt.Hibernate, rt.Version = "instance", "ready", "edge", "bab", "jam", true, "v1" |
| + rt.GoodVersion, rt.GoodDigest = "v0", "sha256:v0" |
| + launch := claimServingLaunchSource(f.backend, f.tokens) |
| + if _, err := launch(t.Context(), rt); err != nil { |
| + t.Fatalf("launch with nothing honoured = %v, want today's", err) |
| + } |
| + svc, reg, dd := rolloutFixture(t, false) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{Account: release.RouteAccount(rt), App: "jam", Build: "v1", Host: "sib.example.com"}}, nil) |
| + declineWithBook(t, svc, reg, dd) |
| + if _, err := launch(t.Context(), rt); err == nil || err.Error() != refuseNoSwaps { |
| + t.Fatalf("launch of held v1 = %v, want %q", err, refuseNoSwaps) |
| + } |
| + served := rt |
| + served.GoodVersion, served.GoodDigest = "v1", "sha256:v1" |
| + if _, err := launch(t.Context(), served); err != nil { |
| + t.Fatalf("launch of the instance's own good build = %v, want it allowed", err) |
| + } |
| +} |
| + |
| +// Task 9 (brief (e)): with no records and no markers, or the flag off, |
| +// LiveExecGate stays nil, so every spawn is today's. |
| +func TestNoLiveExecGateUnlessADeclinedBoxHonoursSomething(t *testing.T) { |
| + for name, env := range map[string]func(string) string{"flag off": noEnv, "declined, empty book": nil} { |
| + t.Run(name, func(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + eb := &activator.ExecBackend{} |
| + svc, reg, dd := rolloutFixtureWith(t, false, &activator.SplitBackend{Exec: eb}) |
| + if env == nil { |
| + declineWithBook(t, svc, reg, dd) |
| + } else { |
| + armRollout(context.Background(), env, svc, reg, dd, nil, unusedStore) |
| + } |
| + if eb.LiveExecGate != nil { |
| + t.Fatal("LiveExecGate set on a box that honours nothing") |
| + } |
| + legsAreTodays(t, svc, reg) |
| + }) |
| + } |
| + t.Run("declined, a record", func(t *testing.T) { |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + eb := &activator.ExecBackend{} |
| + svc, reg, dd := rolloutFixtureWith(t, false, &activator.SplitBackend{Exec: eb}) |
| + writeFailureBook(t, dd, []rollout.FailureRecord{{Account: "a", App: "blog", Build: "v2", Host: "sib.example.com"}}, nil) |
| + declineWithBook(t, svc, reg, dd) |
| + if eb.LiveExecGate == nil { |
| + t.Fatal("a declined box honouring a record left LiveExecGate nil") |
| + } |
| + }) |
| +} |
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go| index f6532820f..d60f3d6cb 100644 |
| --- a/internal/rollout/failures.go |
| +++ b/internal/rollout/failures.go |
| @@ -690,6 +690,15 @@ func (b *FailureBook) HoldRanOn(host string) (HoldRan, bool) { |
| return h, ok |
| } |
| +// HoldRanCount is how many hold-ran markers stand. A box that declined |
| +// CARLOS_SWAP=overlap honours its book when it holds records or markers |
| +// (internal/host, honourDeclinedFailures). |
| +func (b *FailureBook) HoldRanCount() int { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + return len(b.held) |
| +} |
| + |
| // ClearHoldRan drops host's hold-ran marker, whatever it is, and reports |
| // whether a marker stood. A host retry no longer uses it: its grant clears |
| // only a marker naming the grant's build, at the grant's epoch |