amadan
zdd-readiness: hold-ran markers keep records, block worker fallback, and need a signed rollback (N5, N7, ruling 205)
A hold-ran marker no longer lets the marked build bypass a failure record that holds it for an instance with a good build (N7): decideHoldRan and planCheck refuse it in the hold-down words, and the simulation's N7 exemption is gone. A scheduled worker replacement carries no fallback while a marker of its app stands on the host (N5). The installed good build no longer boots over a marker on equality alone (ruling 205): the planner and planCheck refuse it, and the pointer pass clears the marker only when the route's channel pointer names the good build signed after the marker (HoldRan.At, stamped by the book's clock); observe makes that read due at once when a marked host's installed build moves. Status reports the two new refusals as down.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10 files changed,
+458
−78
internal/rollout/failures.go+58 −9internal/rollout/fake_test.go+1 −0internal/rollout/holdran_seq_test.go+15 −9internal/rollout/holdran_test.go+184 −13internal/rollout/manager.go+53 −1internal/rollout/sim_test.go+63 −16internal/rollout/status.go+10 −4internal/rollout/wakeplan.go+37 −23internal/rollout/worker.go+11 −3internal/rollout/worker_test.go+26 −0
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go| index a2fcbffe3..6ffd6c6fc 100644 |
| --- a/internal/rollout/failures.go |
| +++ b/internal/rollout/failures.go |
| @@ -54,15 +54,29 @@ type FailureRecord struct { |
| // down there (an attempt superseded or stopped after its exec, a first wake |
| // whose observation was cancelled). It may have migrated the instance's |
| // database, so nothing falls back on that instance automatically (spec |
| -// §6.4): while the marker stands the planner never boots the good build in |
| -// place of the installed one (decideWake, planCheck). It is a restriction on |
| -// the instance, not a failure of the build: it blocks no sibling and halts |
| -// no cohort. Its own good-build write (a pass) or `carlos instances retry` |
| -// clears it. |
| +// §6.4): while the marker stands the planner never boots the good build |
| +// (decideWake, planCheck), and a worker's replacement carries no fallback |
| +// onto it (executeWorker, Astra R3 N5). It is a restriction on the |
| +// instance, not a failure of the build: it blocks no sibling and halts no |
| +// cohort, and it lifts no failure record either (Astra R3 N7). Its own |
| +// good-build write (a pass), `carlos instances retry`, or a signed rollback |
| +// pointer newer than the marker (applyPointers, ruling 205) clears it. |
| type HoldRan struct { |
| Account string `json:"account"` |
| App string `json:"app"` |
| Build string `json:"build"` |
| + // At is when the book set the marker (its clock). Only a rollback |
| + // pointer signed after it lifts the marker (ruling 205, Astra R3): a |
| + // pointer from before the hold build ran says nothing about recovering |
| + // from it. Zero in a file an earlier agent wrote, which any signed |
| + // rollback pointer is newer than. |
| + At time.Time `json:"at,omitempty"` |
| +} |
| + |
| +// same reports whether h and o mark the same build of the same app. At is |
| +// left out: marking the build again changes nothing worth a write. |
| +func (h HoldRan) same(o HoldRan) bool { |
| + return h.Account == o.Account && h.App == o.App && h.Build == o.Build |
| } |
| type failureFile struct { |
| @@ -90,6 +104,16 @@ type FailureBook struct { |
| // every later mutating call and Prune retries the save until it lands, |
| // so the record still survives an agent restart (spec §8.4). |
| dirty bool |
| + // now stamps a new marker's At. Nil means time.Now; tests and the |
| + // simulation set their own clock. |
| + now func() time.Time |
| +} |
| + |
| +func (b *FailureBook) clock() time.Time { |
| + if b.now != nil { |
| + return b.now() |
| + } |
| + return time.Now() |
| } |
| // cloneRec copies r's host lists so the copy shares no array with r. |
| @@ -194,15 +218,18 @@ func (b *FailureBook) Record(r FailureRecord) (bool, error) { |
| return false, b.saveLocked() |
| } |
| -// markLocked sets host's hold-ran marker to h and reports whether that |
| -// changed anything. The caller holds b.mu and saves. |
| +// markLocked sets host's hold-ran marker to h, stamped now by the book's |
| +// clock, and reports whether that changed anything. A marker already |
| +// standing for the same build is left as it is, its first stamp included, |
| +// so marking it again writes nothing. The caller holds b.mu and saves. |
| func (b *FailureBook) markLocked(host string, h HoldRan) bool { |
| - if cur, ok := b.held[host]; ok && cur == h { |
| + if cur, ok := b.held[host]; ok && cur.same(h) { |
| return false |
| } |
| if b.held == nil { |
| b.held = map[string]HoldRan{} |
| } |
| + h.At = b.clock() |
| b.held[host] = h |
| return true |
| } |
| @@ -210,7 +237,8 @@ func (b *FailureBook) markLocked(host string, h HoldRan) bool { |
| // MarkHoldRan sets host's hold-ran marker (HoldRan) and saves it: the |
| // outcome that set it changed nothing else in the book. A marker already |
| // standing for another build is replaced: the newest hold build to run on |
| -// the instance is the one its database may now need. |
| +// the instance is the one its database may now need. h.At is ignored; the |
| +// book stamps the marker itself. |
| func (b *FailureBook) MarkHoldRan(host string, h HoldRan) error { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| @@ -241,6 +269,27 @@ func (b *FailureBook) ClearHoldRan(host string) (bool, error) { |
| return true, b.saveLocked() |
| } |
| +// ClearHoldRanOnPointer is an explicit rollback as this box sees it (ruling |
| +// 205, Astra R3): host's route installs its good build, and the signed |
| +// pointer of its channel names that build with promotedAt. The caller has |
| +// checked both. It clears host's marker only when the marker is (acct, |
| +// app)'s and older than the pointer: `carlos rollback` signs a fresh |
| +// pointer, while root's automatic sibling repoint signs none, so the |
| +// pointer the good build was first promoted under, from before the hold |
| +// build ran, lifts nothing. The check and the clear are one hold of the |
| +// lock, so a marker set by a newer run between the caller's reads and this |
| +// call is never the one cleared. |
| +func (b *FailureBook) ClearHoldRanOnPointer(host, acct, app string, promotedAt time.Time) (bool, error) { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + cur, ok := b.held[host] |
| + if !ok || cur.Account != acct || cur.App != app || !promotedAt.After(cur.At) { |
| + return false, b.retryLocked() |
| + } |
| + delete(b.held, host) |
| + return true, b.saveLocked() |
| +} |
| + |
| // Lookup returns the record for (acct, app, build), if one stands. |
| func (b *FailureBook) Lookup(acct, app, build string) (FailureRecord, bool) { |
| b.mu.Lock() |
diff --git a/internal/rollout/fake_test.go b/internal/rollout/fake_test.go| index adc673fb1..16f56f2ae 100644 |
| --- a/internal/rollout/fake_test.go |
| +++ b/internal/rollout/fake_test.go |
| @@ -331,6 +331,7 @@ func newHarness(t *testing.T, hosts ...string) *harness { |
| demands: map[string]string{}, mismatch: map[string]bool{}, demandErr: map[string]error{}, noDigest: map[string]bool{}, |
| pointers: map[string]release.ChannelPointer{}, |
| events: &eventLog{}} |
| + h.book.now = h.clock // a marker's At follows the harness clock |
| h.stage("v1") |
| for _, host := range hosts { |
| h.rows.put(registry.Route{Host: host, Kind: "instance", App: "blog", Account: "bab", Status: "ready", |
diff --git a/internal/rollout/holdran_seq_test.go b/internal/rollout/holdran_seq_test.go| index 6efb41f6f..e87db41e5 100644 |
| --- a/internal/rollout/holdran_seq_test.go |
| +++ b/internal/rollout/holdran_seq_test.go |
| @@ -21,10 +21,12 @@ func (h *harness) sleepHost(host string) { |
| // first wake then fails v2, and the cohort-failure transition supersedes h's |
| // attempt, which the activator abandons after v2 ran (Executed). Nothing is |
| // held down on h: it was superseded, not seen to fail. h stops. Its next |
| -// wake must not boot v1: v2 may have migrated h's database (spec §6.4). The |
| -// sibling's record blocks v2 for h, so before the marker the planner sent |
| -// the wake to the good build. It boots v2 again instead, in hold mode, with |
| -// nothing behind it. |
| +// wake must not boot v1: v2 may have migrated h's database (spec §6.4). Nor |
| +// may it boot v2 (Astra R3 N7): the sibling's record holds v2 for h, which |
| +// only passed nothing, and the marker lifts no record (spec §8.4 exempts an |
| +// instance that passed on the build, not one that merely ran it). So h |
| +// stays down until a retry (`carlos instances retry`, `carlos promote |
| +// --retry`) or a newer build, and its plan's check refuses both builds. |
| func TestASupersededHoldThatRanKeepsTheInstanceOffItsGoodBuild(t *testing.T) { |
| ctx := context.Background() |
| h := newHarness(t, "h", "s") |
| @@ -66,13 +68,17 @@ func TestASupersededHoldThatRanKeepsTheInstanceOffItsGoodBuild(t *testing.T) { |
| h.sleepHost("h") |
| h.wakeHost("h") |
| p := h.m.Plan(ctx, h.rows.get("h")) |
| - if p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| - t.Fatalf("h's plan %+v: want v2 again, a hold first wake with no fallback, never the good v1", p) |
| + if p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| + t.Fatalf("h's plan %+v: want the wake refused in the hold-down words, neither v2 nor the good v1", p) |
| } |
| - if p.Check == nil || p.Check("v2") != nil { |
| - t.Fatal("h's check refuses v2, the build its database may now need") |
| + // The same rule at the exec, for a plan made before the record landed: |
| + // sp's check is the sibling's, so build one for h from a plan that boots |
| + // v2 under the marker with no record standing. |
| + check := h.m.planCheck(h.rows.get("h"), activator.WakePlan{Boot: sp.Boot, Mode: activator.ModeHold, FirstWake: true}) |
| + if check("v2") == nil { |
| + t.Fatal("h's check lets v2 boot while the sibling's record of it stands") |
| } |
| - if p.Check("v1") == nil { |
| + if check("v1") == nil { |
| t.Fatal("h's check lets the good v1 boot after the hold build ran") |
| } |
| } |
diff --git a/internal/rollout/holdran_test.go b/internal/rollout/holdran_test.go| index f7c2948b5..45c2c3880 100644 |
| --- a/internal/rollout/holdran_test.go |
| +++ b/internal/rollout/holdran_test.go |
| @@ -8,6 +8,7 @@ import ( |
| "path/filepath" |
| "strings" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/appconfig" |
| @@ -18,7 +19,8 @@ var markV2 = HoldRan{Account: "bab", App: "blog", Build: "v2"} |
| // Final review wave 2, N2: decideWake's rule for an instance with a |
| // hold-ran marker. The good build never boots in place of the installed |
| -// build. |
| +// build, and (Astra R3 N7, ruling 205) the marker lifts no failure record |
| +// and the installed good build waits for a signed rollback. |
| func TestDecideWakeWithAHoldRanMarker(t *testing.T) { |
| overlap := DefaultPolicy() |
| book := func(recs ...FailureRecord) *FailureBook { |
| @@ -32,10 +34,19 @@ func TestDecideWakeWithAHoldRanMarker(t *testing.T) { |
| sibV2 := FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "x", At: t0} |
| sibV3 := FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "x", At: t0} |
| - // The marked build, blocked for the siblings: it boots, hold, no fallback. |
| + // N7: the marked build, blocked for the instance by a sibling's record, |
| + // with a good build: refused, never v1 and never v2. |
| p, err := decideWake(inputs(wakeRoute("v2", "v1"), book(sibV2), overlap)) |
| - if err != nil || p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| - t.Fatalf("installed is the marked build: want a hold first wake of v2 with no fallback, got %+v, %v", p, err) |
| + if err != nil || p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| + t.Fatalf("installed is the marked build, blocked: want the wake refused in the hold-down words, got %+v, %v", p, err) |
| + } |
| + // With no good build a record alone refuses nothing (spec §8.5): the |
| + // marked build boots, hold, no fallback. |
| + noGood := wakeRoute("v2", "") |
| + noGood.GoodDigest = "" |
| + if p, err := decideWake(inputs(noGood, book(sibV2), overlap)); err != nil || p.Refuse != "" || p.Boot.Version != "v2" || |
| + !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| + t.Fatalf("installed is the marked build, blocked, no good build: want a hold first wake of v2, got %+v, %v", p, err) |
| } |
| // Unblocked, and its release allows overlap: still hold, no fallback. |
| if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), book(), overlap)); p.Boot.Version != "v2" || p.Fallback != nil || p.Mode != activator.ModeHold { |
| @@ -55,9 +66,11 @@ func TestDecideWakeWithAHoldRanMarker(t *testing.T) { |
| p.Fallback != nil || p.Mode != activator.ModeOverlap { |
| t.Fatalf("another unblocked build: want an overlap first wake of v3 with no fallback, got %+v", p) |
| } |
| - // The good build installed by hand (a rollback): what was installed boots. |
| - if p, _ := decideWake(inputs(wakeRoute("v1", "v1"), book(), overlap)); p.Boot.Version != "v1" || p.FirstWake || p.Refuse != "" { |
| - t.Fatalf("the good build installed: want its ordinary wake, got %+v", p) |
| + // Ruling 205: the good build installed is no proof of a rollback (root's |
| + // sibling repoint installs it too). The planner refuses while the marker |
| + // stands; the pointer pass lifts the marker on a signed rollback. |
| + if p, _ := decideWake(inputs(wakeRoute("v1", "v1"), book(), overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| + t.Fatalf("the good build installed under a marker: want the wake refused, got %+v", p) |
| } |
| // A staging refusal never boots the good build instead. |
| in := inputs(wakeRoute("v2", "v1"), book(), overlap) |
| @@ -129,16 +142,30 @@ func TestAFirstWakeCancelledAfterItsHoldBuildRanAndItsClearingByAPass(t *testing |
| h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true, |
| Token: p.Token, Executed: true}) |
| - if got, ok := h.book.HoldRanOn("s"); !ok || got != markV2 { |
| - t.Fatalf("marker = %+v, %v: want v2's", got, ok) |
| + if got, ok := h.book.HoldRanOn("s"); !ok || !got.same(markV2) || !got.At.Equal(t0) { |
| + t.Fatalf("marker = %+v, %v: want v2's, stamped at the book's clock", got, ok) |
| } |
| if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| t.Fatal("a cancelled observation recorded a failure") |
| } |
| - h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| + // A sibling then fails v2: the record holds v2 for s too (Astra R3 N7), |
| + // and the marker holds v1, so s stays down rather than boot either. |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: t0, At: t0}) |
| h.sleepHost("s") |
| h.wakeHost("s") |
| + if p := h.m.Plan(ctx, h.rows.get("s")); p.Refuse != HeldDownRefusal("v2") { |
| + t.Fatalf("plan %+v: want the wake refused while the sibling's record of v2 stands", p) |
| + } |
| + // `carlos promote --retry` of v2 clears the record; v2 boots again, in |
| + // hold mode with nothing behind it, never the good v1. |
| + h.advance(time.Minute) |
| + h.pointer("stable", "v2", h.clock()) |
| + h.pass() |
| + if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| + t.Fatal("the retry pointer left v2's record standing") |
| + } |
| + h.wakeHost("s") |
| p = h.m.Plan(ctx, h.rows.get("s")) |
| if p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| t.Fatalf("plan %+v: want v2 again, never the good v1", p) |
| @@ -248,7 +275,7 @@ func TestWhichAttemptOutcomesLeaveTheHoldRanMarker(t *testing.T) { |
| t.Fatal(err) |
| } |
| got, ok := again.HoldRanOn("a") |
| - if ok != tc.want || (ok && got != markV2) { |
| + if ok != tc.want || (ok && !got.same(markV2)) { |
| t.Fatalf("marker on disk = %+v, %v; want set %v", got, ok, tc.want) |
| } |
| if tc.res.Outcome == activator.SwapFailed { |
| @@ -262,16 +289,22 @@ func TestWhichAttemptOutcomesLeaveTheHoldRanMarker(t *testing.T) { |
| // Final review wave 2, N2: status shows an instance with a marker as held, |
| // in the existing words (HeldBy, AttemptDown), never new ones. |
| +// |
| +// Astra R3 N7 and ruling 205 add two down cases: the marked build itself |
| +// held by a sibling's record (d), and the good build installed with no |
| +// signed rollback yet (c). Their wakes are refused, so they are down. |
| func TestStatusShowsAHoldRanInstanceAsHeld(t *testing.T) { |
| - h := newHarness(t, "a", "b") |
| + h := newHarness(t, "a", "b", "c") |
| h.stage("v2", "v3") |
| h.install("v2", "a") |
| h.install("v3", "b") |
| h.sleepHost("a") |
| h.sleepHost("b") |
| + h.sleepHost("c") |
| h.pass() |
| h.book.MarkHoldRan("a", markV2) |
| h.book.MarkHoldRan("b", markV2) |
| + h.book.MarkHoldRan("c", markV2) |
| h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", Reason: "exited early", At: t0}) |
| r := h.m.Status() |
| if got := r.Hosts["a"]; got.HeldBy != "v2" || got.Attempt != "" { |
| @@ -280,6 +313,20 @@ func TestStatusShowsAHoldRanInstanceAsHeld(t *testing.T) { |
| if got := r.Hosts["b"]; got.HeldBy != "v3" || got.Attempt != status.AttemptDown || got.Reason != status.ReasonExit { |
| t.Fatalf("b = %+v: want down, held by v3 (its wakes are refused, not fallen back)", got) |
| } |
| + if got := r.Hosts["c"]; got.HeldBy != "v2" || got.Attempt != status.AttemptDown { |
| + t.Fatalf("c = %+v: want down, held by v2 (its good build v1 is installed, but no signed rollback lifted the marker)", got) |
| + } |
| + |
| + d := newHarness(t, "d") |
| + d.stage("v2") |
| + d.install("v2", "d") |
| + d.sleepHost("d") |
| + d.pass() |
| + d.book.MarkHoldRan("d", markV2) |
| + d.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) |
| + if got := d.m.Status().Hosts["d"]; got.HeldBy != "v2" || got.Attempt != status.AttemptDown || got.Reason != status.ReasonExit { |
| + t.Fatalf("d = %+v: want down, held by v2 (the marked build is held by a sibling's record)", got) |
| + } |
| } |
| // Final review wave 2, N2: the markers survive a reopen; a record written |
| @@ -310,7 +357,7 @@ func TestTheHoldRanMarkerSurvivesReopen(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if got, ok := again.HoldRanOn("a.example.com"); !ok || got != markV2 { |
| + if got, ok := again.HoldRanOn("a.example.com"); !ok || !got.same(markV2) { |
| t.Fatalf("a's marker after reopen = %+v, %v", got, ok) |
| } |
| if got, ok := again.HoldRanOn("c.example.com"); !ok || got.Build != "v3" { |
| @@ -377,3 +424,127 @@ func TestTheHoldRanMarkerIsSchemaAdditive(t *testing.T) { |
| t.Fatalf("the marker is not in the file:\n%s", raw) |
| } |
| } |
| + |
| +// Astra R3 N7: a marker never turns into permission to run a build a |
| +// sibling's failure record holds. The instance has a good build, so it stays |
| +// down, at the plan and at the exec of a plan made before the record. |
| +func TestAMarkedBuildStillHonoursASiblingsFailureRecord(t *testing.T) { |
| + ctx := context.Background() |
| + h := newHarness(t, "h", "s") |
| + h.stage("v2") |
| + h.install("v2", "h", "s") |
| + h.book.MarkHoldRan("h", markV2) |
| + h.sleepHost("h") |
| + h.wakeHost("h") |
| + before := h.m.Plan(ctx, h.rows.get("h")) |
| + if before.Refuse != "" || before.Boot.Version != "v2" || before.Mode != activator.ModeHold || before.Fallback != nil { |
| + t.Fatalf("plan %+v: want a hold first wake of v2 while no record holds it", before) |
| + } |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", At: t0}) // a sibling's plain failure |
| + if err := before.Check("v2"); err == nil { |
| + t.Fatal("the earlier plan's check lets v2 boot after a sibling's record of it landed") |
| + } |
| + h.wakeHost("h") |
| + if p := h.m.Plan(ctx, h.rows.get("h")); p.Refuse != HeldDownRefusal("v2") { |
| + t.Fatalf("plan %+v: want the hold-down refusal for v2", p) |
| + } |
| +} |
| + |
| +// Ruling 205 (Astra R3): the installed good build is no proof of an explicit |
| +// rollback, because root's automatic sibling repoint installs it too. Only a |
| +// signed pointer naming the good build, newer than the marker (`carlos |
| +// rollback` re-signs one), lifts the marker; the pointer pass does it. Until |
| +// then the plan is refused, and so is the good build at the exec of any |
| +// earlier plan. |
| +func TestASignedRollbackClearsTheMarkerAndTheGoodBuildBoots(t *testing.T) { |
| + ctx := context.Background() |
| + h := newHarness(t, "h") |
| + h.stage("v2") |
| + h.sleepHost("h") |
| + h.wakeHost("h") |
| + early := h.m.Plan(ctx, h.rows.get("h")) |
| + if early.Refuse != "" || early.Boot.Version != "v1" || early.FirstWake || early.Check == nil { |
| + t.Fatalf("plan %+v: want the ordinary wake of the installed good v1 before any marker", early) |
| + } |
| + // Hold v2 ran on h, and root repointed h to its good build: v1 is |
| + // installed, under a marker stamped at t0. v1's pointer predates it. |
| + h.book.MarkHoldRan("h", markV2) |
| + h.pointer("stable", "v1", t0.Add(-time.Hour)) |
| + h.advance(time.Minute) |
| + h.pass() |
| + if _, ok := h.book.HoldRanOn("h"); !ok { |
| + t.Fatal("a pointer older than the marker cleared it") |
| + } |
| + h.wakeHost("h") |
| + if p := h.m.Plan(ctx, h.rows.get("h")); p.Refuse != HeldDownRefusal("v2") { |
| + t.Fatalf("no rollback pointer: plan %+v, want refused", p) |
| + } |
| + if err := early.Check("v1"); err == nil || err.Error() != HeldDownRefusal("v2") { |
| + t.Fatalf("the earlier plan's check on v1 = %v, want the hold-down refusal for v2", err) |
| + } |
| + |
| + // `carlos rollback` signs v1's pointer afresh. |
| + h.pointer("stable", "v1", h.clock()) |
| + h.advance(pointerReadEvery) |
| + h.pass() |
| + if got, ok := h.book.HoldRanOn("h"); ok { |
| + t.Fatalf("marker %+v survived a signed rollback newer than it", got) |
| + } |
| + h.wakeHost("h") |
| + p := h.m.Plan(ctx, h.rows.get("h")) |
| + if p.Refuse != "" || p.Boot.Version != "v1" || p.FirstWake { |
| + t.Fatalf("after a signed rollback: plan %+v, want v1's ordinary wake", p) |
| + } |
| + if err := p.Check("v1"); err != nil { |
| + t.Fatalf("after a signed rollback the check refuses v1: %v", err) |
| + } |
| +} |
| + |
| +// Ruling 205: a marked host's installed build moving makes the app's pointer |
| +// read due at once, so a rollback onto the good build is honoured at the |
| +// pass that sees it, not up to pointerReadEvery later. |
| +func TestAnInstallOntoTheGoodBuildMakesTheRollbackReadDueAtOnce(t *testing.T) { |
| + h := newHarness(t, "h") |
| + h.stage("v2", "v3") |
| + h.install("v2", "h") |
| + h.sleepHost("h") |
| + // A record of another build keeps the app's pointers read; this pass |
| + // reads them, so the next read is not due for pointerReadEvery. |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", PromotedAt: t0, At: t0}) |
| + h.pass() |
| + h.book.MarkHoldRan("h", markV2) |
| + h.advance(5 * time.Second) |
| + h.pointer("stable", "v1", h.clock()) // `carlos rollback` to v1 |
| + h.install("v1", "h") |
| + h.pass() |
| + if got, ok := h.book.HoldRanOn("h"); ok { |
| + t.Fatalf("marker %+v survived the pass that saw the rollback's install", got) |
| + } |
| +} |
| + |
| +// Ruling 205's book half: a marker is stamped once by the book's clock (a |
| +// re-mark of the same build keeps it, and writes nothing), and the rollback |
| +// clear lifts only (acct, app)'s marker, and only for a pointer newer than it. |
| +func TestTheHoldRanMarkerStampAndItsRollbackClear(t *testing.T) { |
| + now := t0 |
| + b := &FailureBook{now: func() time.Time { return now }} |
| + b.MarkHoldRan("a", markV2) |
| + now = now.Add(time.Minute) |
| + b.MarkHoldRan("a", markV2) |
| + if got, _ := b.HoldRanOn("a"); !got.At.Equal(t0) { |
| + t.Fatalf("marker At %v after a re-mark of the same build, want %v", got.At, t0) |
| + } |
| + if ok, _ := b.ClearHoldRanOnPointer("a", "bab", "blog", t0); ok { |
| + t.Fatal("a pointer as old as the marker cleared it") |
| + } |
| + if ok, _ := b.ClearHoldRanOnPointer("a", "bab", "other", t0.Add(time.Second)); ok { |
| + t.Fatal("another app's pointer cleared the marker") |
| + } |
| + if ok, _ := b.ClearHoldRanOnPointer("a", "bab", "blog", t0.Add(time.Second)); !ok { |
| + t.Fatal("a newer pointer left the marker standing") |
| + } |
| + b.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v3"}) |
| + if got, _ := b.HoldRanOn("a"); got.Build != "v3" || !got.At.Equal(now) { |
| + t.Fatalf("marker %+v: want v3's, stamped at %v", got, now) |
| + } |
| +} |
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go| index 154719138..75d7316a5 100644 |
| --- a/internal/rollout/manager.go |
| +++ b/internal/rollout/manager.go |
| @@ -366,6 +366,7 @@ func (m *Manager) observe(routes []registry.Route) []appKey { |
| } |
| } |
| m.mu.Unlock() |
| + rearm := false |
| s.mu.Lock() |
| s.q.concurrency = policy.Concurrency |
| for _, x := range list { |
| @@ -385,9 +386,21 @@ func (m *Manager) observe(routes []registry.Route) []appKey { |
| } |
| if s.q.wantBuild(x.rt.Host, cause, x.rt.Version, now) { |
| moved = true |
| + // A marked host's installed build moved: it may be a rollback |
| + // onto its good build, so the app's pointers are read at the |
| + // next applyPointers rather than up to pointerReadEvery later |
| + // (ruling 205). The book's lock is the last in the order. |
| + if _, ok := holdRanFor(m.d.Book, k.acct, k.app, x.rt.Host); ok { |
| + rearm = true |
| + } |
| } |
| } |
| s.mu.Unlock() |
| + if rearm { |
| + m.mu.Lock() |
| + delete(m.pointerRead, k) |
| + m.mu.Unlock() |
| + } |
| s.poke() |
| } |
| if moved { |
| @@ -415,6 +428,17 @@ func routeChannel(rt registry.Route) string { |
| // read again, clears nothing. A second retry of the same build is the same |
| // rule again: the retry's own failure recorded the newer PromotedAt, so only |
| // a pointer signed after it clears the record. |
| +// |
| +// It is also the rollback signal for hold-ran markers (ruling 205, Astra |
| +// R3). An app with a marked host whose route installs its good build is read |
| +// too, on the same schedule, and that host's marker is cleared when its |
| +// channel's pointer names the good build with a PromotedAt newer than the |
| +// marker: `carlos rollback` re-signs the pointer, root's automatic sibling |
| +// repoint does not, so only an explicit rollback lets the good build boot |
| +// over the marker. The signature is the pointer read's to verify |
| +// (release.Channel). observe makes the read due at once when a marked |
| +// host's installed build moves, so a rollback is not left waiting out |
| +// pointerReadEvery. |
| func (m *Manager) applyPointers(routes []registry.Route, keys []appKey) { |
| if m.d.Pointer == nil { |
| return |
| @@ -422,7 +446,18 @@ func (m *Manager) applyPointers(routes []registry.Route, keys []appKey) { |
| now := m.d.Now() |
| for _, k := range keys { |
| recs := m.d.Book.ForApp(k.acct, k.app) |
| - if len(recs) == 0 { |
| + // The marked hosts a rollback could clear, by channel. |
| + rollback := map[string][]registry.Route{} |
| + for _, rt := range routes { |
| + if !Managed(rt) || release.RouteAccount(rt) != k.acct || rt.App != k.app || rt.GoodVersion == "" || rt.Version != rt.GoodVersion { |
| + continue |
| + } |
| + if _, ok := holdRanFor(m.d.Book, k.acct, k.app, rt.Host); ok { |
| + ch := routeChannel(rt) |
| + rollback[ch] = append(rollback[ch], rt) |
| + } |
| + } |
| + if len(recs) == 0 && len(rollback) == 0 { |
| continue |
| } |
| m.mu.Lock() |
| @@ -461,6 +496,23 @@ func (m *Manager) applyPointers(routes []registry.Route, keys []appKey) { |
| m.retryBuild(k, r.Build, "") |
| } |
| } |
| + unmarked := false |
| + for _, rt := range rollback[ch] { |
| + if ptr.Version != rt.GoodVersion { |
| + continue |
| + } |
| + ok, err := m.d.Book.ClearHoldRanOnPointer(rt.Host, k.acct, k.app, ptr.PromotedAt) |
| + if err != nil { |
| + m.d.Logf("rollout: %s: clear hold-ran marker on a rollback to %s: %v (cleared in memory)", rt.Host, rt.GoodVersion, err) |
| + } |
| + if ok { |
| + m.d.Logf("rollout: %s: a signed rollback to %s (pointer of %s) cleared its hold-ran marker", rt.Host, rt.GoodVersion, ptr.PromotedAt.UTC().Format(time.RFC3339)) |
| + unmarked = true |
| + } |
| + } |
| + if unmarked { |
| + m.changed() |
| + } |
| } |
| } |
| } |
diff --git a/internal/rollout/sim_test.go b/internal/rollout/sim_test.go| index d3541d442..134f5608b 100644 |
| --- a/internal/rollout/sim_test.go |
| +++ b/internal/rollout/sim_test.go |
| @@ -44,8 +44,11 @@ import ( |
| // the one refusal it allows, and spec §8.5's bootstrap rule exempts an |
| // instance with no good build. An instance where a hold build ran |
| // without passing (the hold-ran marker, final review wave 2, N2) is |
| -// restricted, not failed: its wakes may boot that build whatever record |
| -// blocks it, may be refused, and carry no fallback (I5 is its rule). |
| +// restricted, not failed: its wakes carry no fallback, and may be |
| +// refused for a build a record holds for it (the marked build included: |
| +// the marker lifts no record, Astra R3 N7) or for its installed good |
| +// build until a signed rollback lifts the marker (ruling 205). I5 is the |
| +// restriction's own rule. |
| // - I2: swap_concurrency is never exceeded. |
| // - I3: a failure record never blocks an instance already serving that |
| // build. With capacity free, its restart and config attempts onto that |
| @@ -60,7 +63,12 @@ import ( |
| // first wake ended with no verdict after its exec), no wake of that |
| // instance boots its good build in place of another installed build, |
| // as a recovery or a fallback, until the instance records a good build |
| -// or a host retry names it (spec §6.4). |
| +// or a host retry names it (spec §6.4). A Pass may also lift the |
| +// restriction, but only on rollback evidence (ruling 205): the |
| +// instance's installed build is its good build, and its channel's |
| +// pointer names that build signed after the hold build ran. A marker |
| +// the Pass cleared without it is an I5 violation. (The book is read |
| +// here only to see what the Pass cleared; the evidence is the model's.) |
| // |
| // Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000). |
| // ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some |
| @@ -252,9 +260,12 @@ type sim struct { |
| exempt map[string]map[string]bool |
| heldDown map[string]map[string]bool |
| // holdRan is the hold-ran restriction per host: the hold build that ran |
| - // there without passing, until the host records a good build (goodAt) |
| - // or a host retry names it. |
| - holdRan map[string]string |
| + // there without passing, until the host records a good build (goodAt), |
| + // a host retry names it, or a Pass lifts it on rollback evidence. |
| + // holdRanAt is when it was set: the book's stamp, which a rollback |
| + // pointer must be newer than. |
| + holdRan map[string]string |
| + holdRanAt map[string]time.Time |
| swaps map[string]*simSwap |
| events chan simEvent |
| @@ -268,7 +279,8 @@ func newSim(seed int64, opt string, staged map[string]bool) *sim { |
| pointers: map[string]release.ChannelPointer{}, builds: map[string]*simBuild{}, |
| lastStamp: map[string]time.Time{}, lastSigned: map[string]time.Time{}, failedEver: map[string]bool{}, |
| exempt: map[string]map[string]bool{}, heldDown: map[string]map[string]bool{}, holdRan: map[string]string{}, |
| - swaps: map[string]*simSwap{}, events: make(chan simEvent)} |
| + holdRanAt: map[string]time.Time{}, swaps: map[string]*simSwap{}, events: make(chan simEvent)} |
| + s.book.now = func() time.Time { return s.now } |
| s.hold = s.r.Intn(4) == 0 |
| s.concurrency = 1 + s.r.Intn(3) |
| s.addBuild("b0", &simBuild{}) |
| @@ -641,10 +653,33 @@ func (s *sim) pick(ok func(h *simHost) bool) *simHost { |
| return c[s.r.Intn(len(c))] |
| } |
| -// pass is the agent pass (Manager.Pass) over every row. |
| +// pass is the agent pass (Manager.Pass) over every row. The only marker a |
| +// Pass may clear is one a signed rollback lifts (applyPointers, ruling 205), |
| +// so each marker it cleared is checked against the model's evidence, then |
| +// lifted in the model too. |
| func (s *sim) pass() { |
| + marked := map[string]bool{} |
| + for _, h := range s.hosts { |
| + if _, ok := s.book.HoldRanOn(h.name); ok { |
| + marked[h.name] = true |
| + } |
| + } |
| rs, _ := s.rows.List() |
| s.m.Pass(rs) |
| + for _, h := range s.hosts { |
| + if _, still := s.book.HoldRanOn(h.name); !marked[h.name] || still { |
| + continue |
| + } |
| + if mark := s.holdRan[h.name]; mark != "" { |
| + p, ok := s.pointers[s.channels[h.ch]] |
| + if h.good == "" || s.installed[h.ch] != h.good || !ok || p.Version != h.good || !p.PromotedAt.After(s.holdRanAt[h.name]) { |
| + violate("I5", "a pass cleared %s's hold-ran restriction (hold build %s ran at %s) with no signed rollback onto its good build %q (installed %s, pointer %+v)", |
| + h.name, mark, s.holdRanAt[h.name].Sub(t0), h.good, s.installed[h.ch], p) |
| + } |
| + delete(s.holdRan, h.name) |
| + s.logf("ground truth: a signed rollback onto %s lifted %s's hold-ran restriction (%s)", h.good, h.name, mark) |
| + } |
| + } |
| } |
| // promote is a promotion and root's install onto one channel's hosts, then |
| @@ -766,7 +801,12 @@ func (s *sim) heldDownOn(host, build string) bool { |
| // holdRanOn is the ground truth of a hold build that ran on h without |
| // passing or being held down there (N2). |
| +// The stamp is the first run's: running the same build again under its own |
| +// restriction leaves the book's marker, and its stamp, as they were. |
| func (s *sim) holdRanOn(h *simHost, build string) { |
| + if s.holdRan[h.name] != build { |
| + s.holdRanAt[h.name] = s.now |
| + } |
| s.holdRan[h.name] = build |
| s.logf("ground truth: hold build %s ran on %s without a verdict", build, h.name) |
| } |
| @@ -779,9 +819,12 @@ func (s *sim) goodAt(h *simHost, build string) { |
| } |
| // checkI5 is I5 at the exec of build on h, for a wake planned against rt. |
| +// The installed good build is barred too (ruling 205): root's sibling |
| +// repoint installs it without any rollback, and only a Pass that saw a |
| +// signed one lifts the restriction. |
| func (s *sim) checkI5(h *simHost, rt registry.Route, build string) { |
| - if mark := s.holdRan[h.name]; mark != "" && build == h.good && build != rt.Version { |
| - violate("I5", "%s boots its good build %s in place of the installed %s after hold build %s ran there", h.name, build, rt.Version, mark) |
| + if mark := s.holdRan[h.name]; mark != "" && build == h.good { |
| + violate("I5", "%s boots its good build %s (installed %s) after hold build %s ran there, with no signed rollback since", h.name, build, rt.Version, mark) |
| } |
| } |
| @@ -820,14 +863,16 @@ func (s *sim) checkWakeRefused(h *simHost, installed, why string) { |
| return |
| } |
| if mark := s.holdRan[h.name]; mark != "" { |
| - // Restricted (N2): refused only for another installed build held for |
| - // the instance, or one not staged correctly. The marked build itself |
| - // boots, whatever record blocks it. "Held" is read loosely here (a |
| + // Restricted (N2): refused for an installed build held for the |
| + // instance, the marked build included (Astra R3 N7: the marker lifts |
| + // no record), for its installed good build until a signed rollback |
| + // lifts the restriction (ruling 205; pass checks the lifting), or for |
| + // a build not staged correctly. "Held" is read loosely here (a |
| // failure of it was recorded at some point and no host retry exempts |
| // the instance): the book clears a record only at its next due |
| // pointer read, after the model's re-sign. |
| held := s.failedEver[installed] && !s.exempt[h.name][installed] |
| - if (installed != mark && held) || s.builds[installed].misstaged { |
| + if held || installed == h.good || s.builds[installed].misstaged { |
| return |
| } |
| violate("I1", "%s (good build %s, hold build %s ran there) has %s installed but its wake boots nothing: %s", h.name, h.good, mark, installed, why) |
| @@ -835,9 +880,11 @@ func (s *sim) checkWakeRefused(h *simHost, installed, why string) { |
| violate("I1", "%s has good build %s but its wake boots nothing: %s", h.name, h.good, why) |
| } |
| -// checkBoot is I1 at the exec of build on h. |
| +// checkBoot is I1 at the exec of build on h. A hold-ran restriction exempts |
| +// nothing (Astra R3 N7): spec §8.4 exempts an instance that served the |
| +// build, not one where it merely ran. |
| func (s *sim) checkBoot(h *simHost, build string) { |
| - if h.good != "" && build != h.good && !h.served[build] && s.blockedFor(h.name, build) && build != s.holdRan[h.name] { |
| + if h.good != "" && build != h.good && !h.served[build] && s.blockedFor(h.name, build) { |
| violate("I1", "%s (good build %s) boots %s, whose failure on this box stands", h.name, h.good, build) |
| } |
| } |
diff --git a/internal/rollout/status.go b/internal/rollout/status.go| index 909e05109..132457348 100644 |
| --- a/internal/rollout/status.go |
| +++ b/internal/rollout/status.go |
| @@ -78,18 +78,24 @@ func (m *Manager) Status() status.RolloutReport { |
| } |
| } |
| // A hold build ran here without passing (final review wave 2, N2): |
| - // the instance is held to it, in the same words. Asleep on a |
| - // different held build, its wakes are refused rather than fall |
| - // back, so it is down. Awake, it serves what it serves. |
| + // the instance is held to it, in the same words. Asleep with a |
| + // good build, its wakes are refused (decideHoldRan), so it is down |
| + // when a record holds its installed build, the marked build |
| + // included (Astra R3 N7), and when its installed build is the good |
| + // build, until a signed rollback lifts the marker (ruling 205). |
| + // Awake, it serves what it serves. |
| if mark, ok := holdRanFor(m.d.Book, s.key.acct, s.key.app, host); ok && h.serving == "" { |
| switch { |
| - case hr.HeldBy != "" && hr.HeldBy != mark.Build && h.good != "": |
| + case hr.HeldBy != "" && h.good != "": |
| hr.Attempt = status.AttemptDown |
| case hr.Attempt == "": |
| hr.HeldBy = mark.Build |
| if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok { |
| hr.Reason = reasonWord(rec.Reason) |
| } |
| + if h.good != "" && h.wantBuild == h.good { |
| + hr.Attempt = status.AttemptDown |
| + } |
| } |
| } |
| r.Hosts[host] = hr |
diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go| index 6ca56c0ad..5b86911a5 100644 |
| --- a/internal/rollout/wakeplan.go |
| +++ b/internal/rollout/wakeplan.go |
| @@ -74,7 +74,7 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) { |
| return activator.WakePlan{Refuse: HeldDownRefusal(installed)}, nil |
| } |
| if mark, ok := holdRanFor(in.Book, acct, app, host); ok { |
| - return decideHoldRan(in, mark, hasGood, recovery) |
| + return decideHoldRan(in, mark, hasGood) |
| } |
| if hasGood && installed == rt.GoodVersion { |
| return recovery, nil |
| @@ -110,29 +110,36 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) { |
| // decideHoldRan is decideWake for an instance with a hold-ran marker (final |
| // review wave 2, N2; spec §6.4: once a hold build has executed against an |
| // instance's database, nothing falls back automatically on that instance). |
| -// The good build never boots in place of the installed build, as a |
| -// recovery or as a fallback: |
| -// - the installed build is the marked build: it boots, in hold mode with |
| -// no fallback, even when a sibling's failure record blocks it for the |
| -// instance's siblings, and its failure here is a hold-down; |
| -// - the installed build is the good build (an explicit rollback): it is |
| -// what was installed, not a fallback, so it boots as an ordinary wake; |
| -// - the installed build is another build held for the instance (a record |
| -// that would send the wake to the good build): the wake is refused, and |
| -// the instance stays down until a retry or a newer build; |
| -// - any other installed build boots as a first wake with no fallback. |
| +// The good build never boots, as a recovery or as a fallback, and the |
| +// marker never lifts a failure record. With a good build: |
| +// - the installed build is the good build: refused. An installed good |
| +// build is no proof of an explicit rollback, because root's automatic |
| +// sibling repoint installs it too (ruling 205, Astra R3). The proof is |
| +// a signed pointer naming the good build, newer than the marker, and the |
| +// pointer pass (applyPointers) clears the marker when it sees one; the |
| +// next wake then boots the good build as an ordinary wake. The planner |
| +// itself reads no pointer. |
| +// - the installed build is held for the instance by a failure record, |
| +// the marked build included: refused (Astra R3 N7). Spec §8.4 exempts an |
| +// instance that passed on the build, not one where it merely ran, so a |
| +// sibling's failure of the marked build holds this instance too. With |
| +// the good build barred as well, it stays down until a retry or a newer |
| +// build. |
| +// |
| +// Otherwise the installed build boots as a first wake with no fallback: the |
| +// marked build in hold mode, whose failure here is a hold-down, any other |
| +// build in its own mode. With no good build a record alone refuses nothing |
| +// (spec §8.5), so the marked build boots even when a record blocks it. |
| // |
| // The hold-down refusal above this call still comes first. The refusal |
| // words are HeldDownRefusal's for the marked build: no new member text. |
| -func decideHoldRan(in wakeInputs, mark HoldRan, hasGood bool, recovery activator.WakePlan) (activator.WakePlan, error) { |
| +func decideHoldRan(in wakeInputs, mark HoldRan, hasGood bool) (activator.WakePlan, error) { |
| rt := in.Route |
| acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| installed := rt.Version |
| switch { |
| - case installed == mark.Build: |
| - // Planned below, whatever record blocks it for the siblings. |
| case hasGood && installed == rt.GoodVersion: |
| - return recovery, nil |
| + return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| case hasGood && in.Book.Blocks(acct, app, installed, host, false): |
| return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| } |
| @@ -292,10 +299,17 @@ func (m *Manager) planRow(ctx context.Context, rt registry.Route, tok uint64) ac |
| // until it first serves (spec §8.5), so refusing it would be an outage. |
| // |
| // A hold-ran marker (final review wave 2, N2), set since the plan or |
| -// before it, applies decideHoldRan's rule at the exec: the marked build may |
| -// boot unless it is held down there, whatever record blocks it for the |
| -// siblings, and the good build may not boot in place of another installed |
| -// build, as a recovery, a fallback or an untried first wake's stand-in. |
| +// before it, applies decideHoldRan's rule at the exec: |
| +// - the good build may not boot, whatever the route installs: as a |
| +// recovery, a fallback, an untried first wake's stand-in, or the |
| +// installed build itself (ruling 205: only the pointer pass's clearing |
| +// of the marker on a signed rollback lets it boot again, and the check |
| +// reads no pointer); |
| +// - the marked build may not boot while it is held down there or, with a |
| +// good build, while a failure record holds it for the instance (Astra |
| +// R3 N7: the marker lifts no record); |
| +// - any other build is checked as without a marker. |
| +// |
| // The book is in memory; nothing here blocks. |
| func (m *Manager) planCheck(rt registry.Route, plan activator.WakePlan) func(build string) error { |
| acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| @@ -309,13 +323,13 @@ func (m *Manager) planCheck(rt registry.Route, plan activator.WakePlan) func(bui |
| return func(build string) error { |
| if mark, ok := holdRanFor(m.d.Book, acct, app, host); ok { |
| switch { |
| + case hasGood && build == rt.GoodVersion: |
| + return errors.New(HeldDownRefusal(mark.Build)) |
| case build == mark.Build: |
| - if m.d.Book.HeldDown(acct, app, build, host) { |
| + if m.d.Book.HeldDown(acct, app, build, host) || (hasGood && m.d.Book.Blocks(acct, app, build, host, false)) { |
| return held(build) |
| } |
| return nil |
| - case hasGood && build == rt.GoodVersion && rt.Version != rt.GoodVersion: |
| - return errors.New(HeldDownRefusal(mark.Build)) |
| } |
| } |
| if !plan.FirstWake || build != boot { |
diff --git a/internal/rollout/worker.go b/internal/rollout/worker.go| index 8ae9be6e7..eb06786ec 100644 |
| --- a/internal/rollout/worker.go |
| +++ b/internal/rollout/worker.go |
| @@ -14,6 +14,11 @@ import ( |
| // build, by its retained identity, goes with it as the fallback for a failed |
| // adoption in overlap mode; the activator decides whether it applies. |
| // |
| +// While a hold-ran marker of the app stands on the host, the replacement |
| +// carries no fallback (Astra R3 N5, spec §6.4): a hold build ran there |
| +// without passing and may have migrated the database, so a failed |
| +// replacement must not start the good build against it. |
| +// |
| // An abandoned result (a staging refusal, a busy entry, a re-check that |
| // refused) is deferred by done(), as a refused swap is: while the attempt is |
| // still the host's newest it waits abandonDeferral, and one superseded by a |
| @@ -21,9 +26,12 @@ import ( |
| func (s *scheduler) executeWorker(ctx context.Context, rt registry.Route, a activator.Attempt) activator.SwapResult { |
| m := s.m |
| var fallback *release.StagedIdentity |
| - if g := rt.Good(); g.Version != "" && g.Digest != "" && g.Version != a.Build { |
| - fallback = &release.StagedIdentity{Version: g.Version, Digest: g.Digest, Demand: g.Demand, |
| - StagedPath: release.VersionPath(m.d.OptDir, release.RouteAccount(rt), g.Version, rt.App)} |
| + acct := release.RouteAccount(rt) |
| + if _, marked := holdRanFor(m.d.Book, acct, rt.App, rt.Host); !marked { |
| + if g := rt.Good(); g.Version != "" && g.Digest != "" && g.Version != a.Build { |
| + fallback = &release.StagedIdentity{Version: g.Version, Digest: g.Digest, Demand: g.Demand, |
| + StagedPath: release.VersionPath(m.d.OptDir, acct, g.Version, rt.App)} |
| + } |
| } |
| m.emit(rt, EvSwapping, "info", textSwapping(a.Build, a.Cause)) |
| return m.d.Act.ReplaceWorker(ctx, a, fallback, s.newest) |
diff --git a/internal/rollout/worker_test.go b/internal/rollout/worker_test.go| index 7a462b266..07848a96a 100644 |
| --- a/internal/rollout/worker_test.go |
| +++ b/internal/rollout/worker_test.go |
| @@ -40,6 +40,32 @@ func TestAWorkerReleaseReplacesOnTheSameEntry(t *testing.T) { |
| } |
| } |
| +// Astra R3 N5: a hold build that ran on a worker without passing (a |
| +// superseded hold, its marker set by done()) may have migrated its |
| +// database, so the next scheduled replacement carries no fallback onto the |
| +// good build. Another app's marker under the same host name changes nothing. |
| +func TestAWorkerReplacementCarriesNoFallbackWhileAMarkerStands(t *testing.T) { |
| + h := workerHarness(t, "bot") |
| + h.stage("v2", "v3") |
| + h.book.MarkHoldRan("bot", markV2) |
| + h.install("v3", "bot") |
| + h.pass() |
| + waitFor(t, "the replacement onto v3", func() bool { return len(h.act.replacements()) == 1 }) |
| + if r := h.act.replacements(); r[0].a.Build != "v3" || r[0].fallback != nil { |
| + t.Fatalf("replacement %+v (fallback %+v): want v3 with no fallback while v2's marker stands", r[0].a, r[0].fallback) |
| + } |
| + |
| + other := workerHarness(t, "bot") |
| + other.stage("v3") |
| + other.book.MarkHoldRan("bot", HoldRan{Account: "bab", App: "other", Build: "v2"}) |
| + other.install("v3", "bot") |
| + other.pass() |
| + waitFor(t, "the replacement onto v3", func() bool { return len(other.act.replacements()) == 1 }) |
| + if r := other.act.replacements(); r[0].fallback == nil || r[0].fallback.Version != "v1" { |
| + t.Fatalf("replacement %+v: another app's marker took away the good v1 fallback", r[0]) |
| + } |
| +} |
| + |
| func TestAFailedWorkerReleaseFallsBackAndHalts(t *testing.T) { |
| h := workerHarness(t, "bot") |
| h.stage("v2") |