amadan

carlos / platform Public

zdd-5 task 14: Pass forgets hosts no longer on the box

A host whose route was removed, moved away, moved to another app or stopped
being managed stayed in the scheduler's queue and config tracks for the life
of the process, so Status could report it queued and hold a deploy watch
open (Task 13 carry). A host with an attempt running is kept until it ends;
a row passing through draining/moving/landing keeps its place.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude b434bacf87cc6264bdb291cdb23dfb2cd08fa2ec parent 0c41859
2 files changed, +112 −0
  • internal/rollout/manager.go +63 −0
  • internal/rollout/status_test.go +49 −0
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go
index 04776166e..63d115e01 100644
--- a/internal/rollout/manager.go
+++ b/internal/rollout/manager.go
@@ -194,11 +194,74 @@ func (m *Manager) track(host string) *configTrack {
// route installs any more, and lifts the halts those records held.
func (m *Manager) Pass(routes []registry.Route) {
keys := m.observe(routes)
+ m.forget(routes)
m.applyPointers(routes, keys)
m.prune(routes)
m.liftClearedHalts()
}
+// heldOnBox reports whether rt is a route this manager keeps state for:
+// Managed, but with a passing status (draining, moving, landing) read as
+// ready. A row passing through one of those for a Pass keeps its place in
+// the queue, its pending restart or config, and its held ack; a row that is
+// gone, or that stopped being an exec route this scheduler moves, does not.
+func heldOnBox(rt registry.Route) bool {
+ rt.Status = "ready"
+ return Managed(rt)
+}
+
+// forget drops what the manager holds for hosts no longer on the box: a
+// route removed or moved away, or one that now belongs to another app or
+// stopped being managed. Without it such a host stayed in Status for the
+// life of the process, possibly as queued, and could hold a deploy watch
+// open (Task 13). A host with an attempt running is kept: the attempt's
+// done() still needs its slot, and the next Pass after it finishes forgets
+// it. Failure records and cohort halts are not per host and are not
+// touched; prune and liftClearedHalts own those.
+//
+// Manager.mu, then each scheduler.mu: the documented order. Nothing here
+// calls the activator.
+func (m *Manager) forget(routes []registry.Route) {
+ present := make(map[string]appKey, len(routes))
+ for _, rt := range routes {
+ if heldOnBox(rt) {
+ present[rt.Host] = appKey{release.RouteAccount(rt), rt.App}
+ }
+ }
+ m.mu.Lock()
+ running := map[string]bool{}
+ var gone []string
+ for k, s := range m.apps {
+ s.mu.Lock()
+ for host, h := range s.q.hosts {
+ if pk, ok := present[host]; ok && pk == k {
+ continue
+ }
+ if h.running != nil {
+ running[host] = true
+ continue
+ }
+ delete(s.q.hosts, host)
+ gone = append(gone, host)
+ }
+ s.mu.Unlock()
+ }
+ for _, host := range gone {
+ if _, ok := present[host]; ok || running[host] {
+ continue // moved to another app on this box, or still running there
+ }
+ delete(m.hostApp, host)
+ delete(m.cfg, host)
+ delete(m.retried, host)
+ delete(m.observing, host)
+ }
+ m.mu.Unlock()
+ if len(gone) > 0 {
+ m.d.Logf("rollout: forgot %d host(s) no longer on this box: %v", len(gone), gone)
+ m.changed() // the hosts left the report
+ }
+}
+
// Nudge re-reads one host and wakes its scheduler. The activator sweep's
// drift arm calls it (spec §10, OnDrift) on a goroutine of its own, for every
// drifted entry on every sweep, with no dedupe of its own. So nudges for one
diff --git a/internal/rollout/status_test.go b/internal/rollout/status_test.go
index 57ecb6797..5a5f3c89b 100644
--- a/internal/rollout/status_test.go
+++ b/internal/rollout/status_test.go
@@ -161,3 +161,52 @@ func TestStatusReportsAConfigRevisionDeliveredButNotActive(t *testing.T) {
close(gate)
waitFor(t, "the config to activate", func() bool { return !h.m.Status().Hosts["a"].ConfigPending })
}
+
+// Carried from Task 13: a host whose route left the box is forgotten by the
+// next Pass, queue slot and config track alike, so Status never reports a
+// departed host as queued. One with an attempt running is kept until the
+// attempt finishes, and a row only passing through "moving" keeps its place.
+func TestPassForgetsHostsThatLeftTheBox(t *testing.T) {
+ h := newHarness(t, "a", "b", "c")
+ h.stage("v2")
+ gate := h.act.hold("a")
+ h.install("v2", "a", "b", "c")
+ h.pass()
+ waitFor(t, "a's canary", func() bool { _, s, _ := h.act.snapshot(); return len(s) == 1 })
+ h.m.ConfigDelivered("b", "rev1")
+ if r := h.m.Status(); r.Hosts["b"].Attempt != status.AttemptQueued || !r.Hosts["b"].ConfigPending {
+ t.Fatalf("b = %+v, want queued with its config pending", r.Hosts["b"])
+ }
+ moving := h.rows.get("c")
+ moving.Status = "moving"
+ h.rows.put(moving)
+ h.rows.mu.Lock()
+ delete(h.rows.rows, "a")
+ delete(h.rows.rows, "b")
+ h.rows.mu.Unlock()
+ h.pass()
+ r := h.m.Status()
+ if _, ok := r.Hosts["b"]; ok {
+ t.Fatalf("b = %+v after its route left the box", r.Hosts["b"])
+ }
+ h.m.mu.Lock()
+ _, track := h.m.cfg["b"]
+ _, mapped := h.m.hostApp["b"]
+ h.m.mu.Unlock()
+ if track || mapped {
+ t.Fatalf("b's config track (%v) or app mapping (%v) outlived its route", track, mapped)
+ }
+ if got := r.Hosts["a"]; got.Attempt != status.AttemptSettling {
+ t.Fatalf("a = %+v: a host with its attempt running was forgotten", got)
+ }
+ if got := r.Hosts["c"]; got.Attempt != status.AttemptQueued || got.Installed != "v2" {
+ t.Fatalf("c = %+v: a row passing through moving lost its place", got)
+ }
+ close(gate)
+ waitFor(t, "a's attempt to finish", func() bool { calls, _, _ := h.act.snapshot(); return len(calls) >= 1 })
+ waitFor(t, "a forgotten once its attempt finished", func() bool {
+ h.pass()
+ _, ok := h.m.Status().Hosts["a"]
+ return !ok
+ })
+}