amadan
zdd-5 task 6: a retried host is a canary whether or not its cohort is halted
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 files changed,
+41
−12
internal/rollout/admit_test.go+26 −2internal/rollout/queue.go+15 −10
diff --git a/internal/rollout/admit_test.go b/internal/rollout/admit_test.go| index 94818e9f7..d7c6c3215 100644 |
| --- a/internal/rollout/admit_test.go |
| +++ b/internal/rollout/admit_test.go |
| @@ -276,8 +276,32 @@ func TestARetriedInstanceWaitsForTheCohortsUnsettledCanary(t *testing.T) { |
| t.Fatalf("admitted %+v beside a's unsettled canary", more) |
| } |
| q.finish(first[0], activator.SwapDone) |
| - if ds := q.admit(currentView(q), t0); len(ds) != 1 || ds[0].ID.Host != "b" { |
| - t.Fatalf("after the settle admitted %+v, want b", ds) |
| + if ds := q.admit(currentView(q), t0); len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| + t.Fatalf("after the settle admitted %+v, want b as a canary", ds) |
| + } |
| +} |
| + |
| +// Spec §8.4: the next attempt after `instances retry` runs as a canary. That |
| +// holds in a cohort that is not halted (a halt lives in memory and an agent |
| +// restart loses it, and a second retry can follow a settled canary), where |
| +// the retried host would otherwise run as an ordinary swap with no settle |
| +// window. |
| +func TestARetriedInstanceIsACanaryInACohortThatIsNotHalted(t *testing.T) { |
| + q := awakeQueue(noBlocks, "a", "b") |
| + releaseAll(q, "v2", "a", "b") |
| + first := q.admit(currentView(q), t0) |
| + if len(first) != 1 || first[0].ID.Host != "a" || !first[0].Canary { |
| + t.Fatalf("admitted %+v, want a as the canary", first) |
| + } |
| + q.finish(first[0], activator.SwapDone) |
| + q.retryHost("b") |
| + ds := q.admit(currentView(q), t0) |
| + if len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| + t.Fatalf("admitted %+v, want b as a canary", ds) |
| + } |
| + // It claims the slot: the cohort has an unsettled canary again. |
| + if c := q.cohorts[ds[0].Cohort]; c.canary != "b" || c.settled { |
| + t.Fatalf("cohort %+v, want b's canary unsettled", *c) |
| } |
| } |
diff --git a/internal/rollout/queue.go b/internal/rollout/queue.go| index e4cc04531..d839b1c12 100644 |
| --- a/internal/rollout/queue.go |
| +++ b/internal/rollout/queue.go |
| @@ -296,9 +296,9 @@ func (q *appQueue) admit(view map[string]string, now time.Time) []dispatch { |
| // every attempt for the host until notBefore, and a failed config attempt |
| // waits out its backoff. A cohort's first attempt is its canary and runs |
| // alone until it settles, and that holds for a retried host too: a retry |
| -// overrides the halt of an adoption cohort and nothing else, so it still |
| -// waits for any canary of the cohort that is running and unsettled. |
| -// A halted adoption cohort admits nothing but such a retried host. |
| +// makes the host's next adoption a canary and lifts the halt for it, but it |
| +// still waits for any other canary of the cohort that is running and |
| +// unsettled. A halted adoption cohort admits nothing but such a retried host. |
| func (q *appQueue) gate(h *hostSlot, now time.Time) (dispatch, bool) { |
| d, ok := q.pending(h) |
| if !ok || now.Before(h.notBefore) { |
| @@ -308,17 +308,22 @@ func (q *appQueue) gate(h *hostSlot, now time.Time) (dispatch, bool) { |
| return dispatch{}, false |
| } |
| c := q.cohortFor(d.Cohort) |
| + // The flag is tied to the build it was set for, so a retry on a host with |
| + // nothing pending cannot lift a later cohort's halt or turn an unrelated |
| + // attempt into a canary. |
| + forced := d.Class == classAdopt && h.forceBuild != "" && h.forceBuild == d.ID.Build |
| switch { |
| - case c.halted && d.Class == classAdopt: |
| - // The flag is tied to the build it was set for, so a retry on a host |
| - // with nothing pending cannot lift a later cohort's halt. |
| - if h.forceBuild == "" || h.forceBuild != d.ID.Build { |
| - return dispatch{}, false |
| - } |
| - if c.canary != "" && !c.settled { |
| + case forced: |
| + // The next attempt after a retry is a canary (spec §8.4) whether or |
| + // not the cohort is halted: halts live in memory and a restart loses |
| + // them, and a second retry can follow a settled canary. It still |
| + // waits out another host's running canary. |
| + if c.canary != "" && c.canary != h.host && !c.settled { |
| return dispatch{}, false |
| } |
| d.Canary = true |
| + case c.halted && d.Class == classAdopt: |
| + return dispatch{}, false |
| case c.canary == "": |
| d.Canary = true |
| case !c.settled: |