amadan
zdd-readiness: a marked good build boots as the marked build, markers carry their execution's start, and moves onto the good build wait for a signed rollback (Task 3 fix round 1)
Finding 1: when the marker names the installed build, the marked-build rule applies even if it is also the good build (decideHoldRan, planCheck, status). Finding 2: HoldRan.At is the execution's start, an attempt's dispatch (dispatch.Started) or a planned wake's plan time (kept per token in Manager.wakeAt), so a carlos rollback signed while the hold build ran clears the marker; setting a marker makes the pointer read due at once. Finding 3: scheduler.execute abandons, unexecuted, a move (release, install, retry) onto the installed good build while another build's marker stands, for socket instances and workers alike; the queue derives that move as classRecover, so the gate is on the cause. Simulation: execution-start stamps, checkI5 exempts the marked build, checkWakeRefused requires a boot of a marked installed good build.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
8 files changed,
+515
−89
internal/rollout/failures.go+28 −15internal/rollout/holdran_test.go+265 −24internal/rollout/manager.go+93 −9internal/rollout/queue.go+4 −0internal/rollout/scheduler.go+34 −2internal/rollout/sim_test.go+42 −17internal/rollout/status.go+6 −3internal/rollout/wakeplan.go+43 −19
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go| index 6ffd6c6fc..7725c2a1a 100644 |
| --- a/internal/rollout/failures.go |
| +++ b/internal/rollout/failures.go |
| @@ -47,6 +47,9 @@ type FailureRecord struct { |
| // that failed in hold mode, ran its build, and was not held down. It is |
| // an instruction to Record, never stored in the record. |
| MarkHoldRan bool `json:"-"` |
| + // HoldRanAt is the marker's At for MarkHoldRan: when the attempt that |
| + // ran the build started (its dispatch). An instruction too, never stored. |
| + HoldRanAt time.Time `json:"-"` |
| } |
| // HoldRan is an instance's "hold ran" marker (final review wave 2, N2): a |
| @@ -65,11 +68,15 @@ type HoldRan struct { |
| Account string `json:"account"` |
| App string `json:"app"` |
| Build string `json:"build"` |
| - // At is when the book set the marker (its clock). Only a rollback |
| + // At is when the execution that set the marker started: an attempt's |
| + // dispatch, a planned wake's plan (Task 3 fix round 1). Only a rollback |
| // pointer signed after it lifts the marker (ruling 205, Astra R3): a |
| - // pointer from before the hold build ran says nothing about recovering |
| - // from it. Zero in a file an earlier agent wrote, which any signed |
| - // rollback pointer is newer than. |
| + // pointer from before the hold build started says nothing about |
| + // recovering from it, while `carlos rollback` signed during the |
| + // execution (the rollback that superseded it) does. It is not the time |
| + // the marker was recorded: that comes after the supersession, so the |
| + // rollback that caused it would always be older. Zero in a file an |
| + // earlier agent wrote, which any signed rollback pointer is newer than. |
| At time.Time `json:"at,omitempty"` |
| } |
| @@ -104,8 +111,8 @@ type FailureBook struct { |
| // every later mutating call and Prune retries the save until it lands, |
| // so the record still survives an agent restart (spec §8.4). |
| dirty bool |
| - // now stamps a new marker's At. Nil means time.Now; tests and the |
| - // simulation set their own clock. |
| + // now stamps a marker whose caller gave no At (markLocked). Nil means |
| + // time.Now; tests and the simulation set their own clock. |
| now func() time.Time |
| } |
| @@ -188,8 +195,8 @@ func (b *FailureBook) find(acct, app, build string) int { |
| func (b *FailureBook) Record(r FailureRecord) (bool, error) { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| - marked := r.MarkHoldRan && b.markLocked(r.Host, HoldRan{Account: r.Account, App: r.App, Build: r.Build}) |
| - r.MarkHoldRan = false |
| + marked := r.MarkHoldRan && b.markLocked(r.Host, HoldRan{Account: r.Account, App: r.App, Build: r.Build, At: r.HoldRanAt}) |
| + r.MarkHoldRan, r.HoldRanAt = false, time.Time{} |
| i := b.find(r.Account, r.App, r.Build) |
| if i < 0 { |
| r.Retried = nil |
| @@ -218,10 +225,14 @@ func (b *FailureBook) Record(r FailureRecord) (bool, error) { |
| return false, b.saveLocked() |
| } |
| -// markLocked sets host's hold-ran marker to h, stamped now by the book's |
| -// clock, and reports whether that changed anything. A marker already |
| -// standing for the same build is left as it is, its first stamp included, |
| -// so marking it again writes nothing. The caller holds b.mu and saves. |
| +// markLocked sets host's hold-ran marker to h and reports whether that |
| +// changed anything. h.At is the execution's start, from the caller. A zero |
| +// one (a caller that could not know it) is stamped with the book's clock |
| +// now: later than any real start, so only a pointer signed later still |
| +// lifts the marker, the safe direction. A marker already standing for the |
| +// same build is left as it is, its first stamp included, so marking it |
| +// again writes nothing (Task 4 gives each execution its own identity). The |
| +// caller holds b.mu and saves. |
| func (b *FailureBook) markLocked(host string, h HoldRan) bool { |
| if cur, ok := b.held[host]; ok && cur.same(h) { |
| return false |
| @@ -229,7 +240,9 @@ func (b *FailureBook) markLocked(host string, h HoldRan) bool { |
| if b.held == nil { |
| b.held = map[string]HoldRan{} |
| } |
| - h.At = b.clock() |
| + if h.At.IsZero() { |
| + h.At = b.clock() |
| + } |
| b.held[host] = h |
| return true |
| } |
| @@ -237,8 +250,8 @@ func (b *FailureBook) markLocked(host string, h HoldRan) bool { |
| // MarkHoldRan sets host's hold-ran marker (HoldRan) and saves it: the |
| // outcome that set it changed nothing else in the book. A marker already |
| // standing for another build is replaced: the newest hold build to run on |
| -// the instance is the one its database may now need. h.At is ignored; the |
| -// book stamps the marker itself. |
| +// the instance is the one its database may now need. h.At is the |
| +// execution's start (markLocked). |
| func (b *FailureBook) MarkHoldRan(host string, h HoldRan) error { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
diff --git a/internal/rollout/holdran_test.go b/internal/rollout/holdran_test.go| index 45c2c3880..6eeb741c2 100644 |
| --- a/internal/rollout/holdran_test.go |
| +++ b/internal/rollout/holdran_test.go |
| @@ -4,9 +4,11 @@ import ( |
| "context" |
| "encoding/json" |
| "errors" |
| + "fmt" |
| "os" |
| "path/filepath" |
| "strings" |
| + "sync" |
| "testing" |
| "time" |
| @@ -72,6 +74,21 @@ func TestDecideWakeWithAHoldRanMarker(t *testing.T) { |
| if p, _ := decideWake(inputs(wakeRoute("v1", "v1"), book(), overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| t.Fatalf("the good build installed under a marker: want the wake refused, got %+v", p) |
| } |
| + // Task 3 fix round 1: the marked build is also the good build, and it is |
| + // installed. The marked-build rule applies, not the good-build arm: it |
| + // boots, in hold mode with no fallback. A sibling's record of it does not |
| + // hold an instance that served it (spec §8.4, Blocks with served); only a |
| + // hold-down refuses it. |
| + for name, b := range map[string]*FailureBook{"no record": book(), "a sibling's record": book(sibV2)} { |
| + if p, err := decideWake(inputs(wakeRoute("v2", "v2"), b, overlap)); err != nil || p.Refuse != "" || p.Boot.Version != "v2" || |
| + !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold { |
| + t.Fatalf("marked build = installed = good, %s: want a hold first wake of v2 with no fallback, got %+v, %v", name, p, err) |
| + } |
| + } |
| + downGood := book(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0}) |
| + if p, _ := decideWake(inputs(wakeRoute("v2", "v2"), downGood, overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" { |
| + t.Fatalf("marked build = installed = good, held down: want the hold-down refusal, got %+v", p) |
| + } |
| // A staging refusal never boots the good build instead. |
| in := inputs(wakeRoute("v2", "v1"), book(), overlap) |
| in.Demand = func(string, string) (string, string, error) { return "", "", errors.New("digest mismatch") } |
| @@ -143,7 +160,7 @@ func TestAFirstWakeCancelledAfterItsHoldBuildRanAndItsClearingByAPass(t *testing |
| h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true, |
| Token: p.Token, Executed: true}) |
| if got, ok := h.book.HoldRanOn("s"); !ok || !got.same(markV2) || !got.At.Equal(t0) { |
| - t.Fatalf("marker = %+v, %v: want v2's, stamped at the book's clock", got, ok) |
| + t.Fatalf("marker = %+v, %v: want v2's, stamped at its wake's plan", got, ok) |
| } |
| if _, ok := h.book.Lookup("bab", "blog", "v2"); ok { |
| t.Fatal("a cancelled observation recorded a failure") |
| @@ -236,7 +253,10 @@ func TestRetryHostClearsTheHoldRanMarker(t *testing.T) { |
| // that ran nothing, an overlap attempt, a restart, a hold-down and a pass do |
| // not. |
| func TestWhichAttemptOutcomesLeaveTheHoldRanMarker(t *testing.T) { |
| - adopt := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRelease, Build: "v2"}, SchedRev: 1, Class: classAdopt, Cohort: cohortKey("v2", classAdopt)} |
| + // Started is the dispatch's (scheduler.execute): the marker's At, on disk |
| + // too, whichever save sets it (Task 3 fix round 1). |
| + started := t0.Add(-time.Minute) |
| + adopt := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRelease, Build: "v2"}, SchedRev: 1, Class: classAdopt, Cohort: cohortKey("v2", classAdopt), Started: started} |
| restart := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRestart, Build: "v1", RestartID: "r1"}, SchedRev: 1, Class: classRecover, Cohort: cohortKey("v1", classRecover)} |
| for _, tc := range []struct { |
| name string |
| @@ -275,8 +295,8 @@ func TestWhichAttemptOutcomesLeaveTheHoldRanMarker(t *testing.T) { |
| t.Fatal(err) |
| } |
| got, ok := again.HoldRanOn("a") |
| - if ok != tc.want || (ok && !got.same(markV2)) { |
| - t.Fatalf("marker on disk = %+v, %v; want set %v", got, ok, tc.want) |
| + if ok != tc.want || (ok && (!got.same(markV2) || !got.At.Equal(started))) { |
| + t.Fatalf("marker on disk = %+v, %v; want set %v, stamped at the dispatch %v", got, ok, tc.want, started) |
| } |
| if tc.res.Outcome == activator.SwapFailed { |
| if _, ok := again.Lookup("bab", "blog", "v2"); !ok { |
| @@ -327,6 +347,54 @@ func TestStatusShowsAHoldRanInstanceAsHeld(t *testing.T) { |
| if got := d.m.Status().Hosts["d"]; got.HeldBy != "v2" || got.Attempt != status.AttemptDown || got.Reason != status.ReasonExit { |
| t.Fatalf("d = %+v: want down, held by v2 (the marked build is held by a sibling's record)", got) |
| } |
| + |
| + // Task 3 fix round 1: the marked build is also the installed good build. |
| + // Its wake boots it (the marked-build rule), so it is held, not down. |
| + e := newHarness(t, "e") |
| + e.stage("v2") |
| + r2 := e.rows.get("e") |
| + r2.Version, r2.GoodVersion, r2.GoodDigest = "v2", "v2", "sha-v2" |
| + e.rows.put(r2) |
| + e.sleepHost("e") |
| + e.pass() |
| + e.book.MarkHoldRan("e", markV2) |
| + e.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) |
| + if got := e.m.Status().Hosts["e"]; got.HeldBy != "v2" || got.Attempt != "" { |
| + t.Fatalf("e = %+v: want held by v2 and not down (its marked build is its installed good build, which its wake boots)", got) |
| + } |
| +} |
| + |
| +// Task 3 fix round 1: when the marker names the installed good build, the |
| +// marked-build rule applies at the plan and at the exec, even with a |
| +// sibling's record of the build standing (the instance served it). The good |
| +// build is still refused at the exec of any other plan while the marker |
| +// stands. |
| +func TestAMarkerOnTheInstalledGoodBuildBootsIt(t *testing.T) { |
| + ctx := context.Background() |
| + h := newHarness(t, "h") |
| + h.stage("v2", "v3") |
| + r := h.rows.get("h") |
| + r.Version, r.GoodVersion, r.GoodDigest = "v2", "v2", "sha-v2" |
| + h.rows.put(r) |
| + h.book.MarkHoldRan("h", markV2) |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| + h.sleepHost("h") |
| + h.wakeHost("h") |
| + p := h.m.Plan(ctx, h.rows.get("h")) |
| + if p.Refuse != "" || p.Boot.Version != "v2" || p.Mode != activator.ModeHold || p.Fallback != nil || p.Check == nil { |
| + t.Fatalf("plan %+v: want a hold wake of v2 with no fallback", p) |
| + } |
| + if err := p.Check("v2"); err != nil { |
| + t.Fatalf("the plan's check refuses v2, the marked and installed good build: %v", err) |
| + } |
| + // Another build installed: the good v2 is not the installed build, and |
| + // the good-build arm refuses it at that plan's exec. |
| + h.install("v3", "h") |
| + other := h.rows.get("h") |
| + check := h.m.planCheck(other, activator.WakePlan{Boot: p.Boot, Mode: activator.ModeHold, FirstWake: true}) |
| + if err := check("v2"); err == nil || err.Error() != HeldDownRefusal("v2") { |
| + t.Fatalf("check of the good v2 with v3 installed = %v, want the hold-down refusal", err) |
| + } |
| } |
| // Final review wave 2, N2: the markers survive a reopen; a record written |
| @@ -500,31 +568,124 @@ func TestASignedRollbackClearsTheMarkerAndTheGoodBuildBoots(t *testing.T) { |
| } |
| } |
| -// Ruling 205: a marked host's installed build moving makes the app's pointer |
| -// read due at once, so a rollback onto the good build is honoured at the |
| -// pass that sees it, not up to pointerReadEvery later. |
| +// Ruling 205, in the order it happens (Task 3 fix round 1): hold build v2 is |
| +// dispatched and running on h; `carlos rollback` signs v1 and root installs |
| +// v1 on h; the attempt is superseded after v2 ran, and marks h. The marker's |
| +// At is the attempt's dispatch, so the rollback signed while v2 ran is newer |
| +// than it, and the pass after the mark clears it at once (setting the marker |
| +// made the pointer read due; the pass that saw the install came before any |
| +// marker). The sleeping h's next wake boots v1. |
| +// |
| +// A pointer naming v1 signed before the attempt started clears nothing: it |
| +// says nothing about recovering from v2. |
| func TestAnInstallOntoTheGoodBuildMakesTheRollbackReadDueAtOnce(t *testing.T) { |
| - h := newHarness(t, "h") |
| - h.stage("v2", "v3") |
| - h.install("v2", "h") |
| - h.sleepHost("h") |
| - // A record of another build keeps the app's pointers read; this pass |
| - // reads them, so the next read is not due for pointerReadEvery. |
| - h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", PromotedAt: t0, At: t0}) |
| - h.pass() |
| - h.book.MarkHoldRan("h", markV2) |
| - h.advance(5 * time.Second) |
| - h.pointer("stable", "v1", h.clock()) // `carlos rollback` to v1 |
| - h.install("v1", "h") |
| + ctx := context.Background() |
| + for _, tc := range []struct { |
| + name string |
| + signedAt time.Duration // the v1 pointer's PromotedAt, from the attempt's dispatch |
| + cleared bool |
| + }{ |
| + {"rollback signed while the hold build ran", 5 * time.Second, true}, |
| + {"pointer signed before the hold build started", -time.Minute, false}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + h := newHarness(t, "h") |
| + h.swap = appconfig.SwapHold |
| + h.stage("v2", "v3") |
| + h.act.mode = activator.ModeHold |
| + h.act.executed = func(a activator.Attempt, _ activator.SwapOutcome) bool { return a.Build == "v2" } |
| + // A record of another build keeps the app's pointers read; the |
| + // first pass reads them, so no read is due for pointerReadEvery |
| + // unless something makes it so. |
| + h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", PromotedAt: t0, At: t0}) |
| + gate := h.act.hold("h") |
| + h.install("v2", "h") |
| + h.pass() |
| + waitFor(t, "h's hold attempt onto v2", func() bool { |
| + _, started, _ := h.act.snapshot() |
| + return len(started) == 1 && started[0] == "h:v2" |
| + }) |
| + start := h.clock() |
| + |
| + h.advance(5 * time.Second) |
| + h.pointer("stable", "v1", start.Add(tc.signedAt)) // `carlos rollback` to v1 (or an older pointer) |
| + h.install("v1", "h") // root installs it |
| + h.pass() // the newer desire supersedes the attempt |
| + h.advance(5 * time.Second) |
| + close(gate) |
| + waitFor(t, "h's attempt to finish", func() bool { return !h.m.attemptRunning("h") }) |
| + got, ok := h.book.HoldRanOn("h") |
| + if !ok || !got.same(markV2) || !got.At.Equal(start) { |
| + t.Fatalf("marker = %+v, %v: want v2's, stamped at the attempt's dispatch %v", got, ok, start) |
| + } |
| + |
| + h.sleepHost("h") |
| + h.pass() |
| + if _, still := h.book.HoldRanOn("h"); still == tc.cleared { |
| + t.Fatalf("after the pass the marker stands: %v, want %v", still, !tc.cleared) |
| + } |
| + h.wakeHost("h") |
| + p := h.m.Plan(ctx, h.rows.get("h")) |
| + if tc.cleared { |
| + if p.Refuse != "" || p.Boot.Version != "v1" || p.FirstWake { |
| + t.Fatalf("plan %+v after the rollback: want v1's ordinary wake", p) |
| + } |
| + if err := p.Check("v1"); err != nil { |
| + t.Fatalf("after the rollback the check refuses v1: %v", err) |
| + } |
| + return |
| + } |
| + if p.Refuse != HeldDownRefusal("v2") { |
| + t.Fatalf("plan %+v with only an older pointer: want refused", p) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Ruling 205 (Task 3 fix round 1): a planned wake's marker is stamped with |
| +// its own plan's time, the start of its execution, kept per token: a newer |
| +// plan for the host does not change an older wake's start. A wake whose |
| +// start is not known (no token) falls back to the book's clock, the safe |
| +// direction. |
| +func TestAWakesHoldRanMarkerIsStampedWithItsPlanTime(t *testing.T) { |
| + ctx := context.Background() |
| + h := newHarness(t, "s") |
| + h.swap = appconfig.SwapHold |
| + h.stage("v2") |
| + h.install("v2", "s") |
| + h.sleepHost("s") |
| h.pass() |
| - if got, ok := h.book.HoldRanOn("h"); ok { |
| - t.Fatalf("marker %+v survived the pass that saw the rollback's install", got) |
| + h.wakeHost("s") |
| + first := h.m.Plan(ctx, h.rows.get("s")) |
| + if !first.FirstWake || first.Boot.Version != "v2" || first.Mode != activator.ModeHold { |
| + t.Fatalf("plan %+v: want a hold first wake of v2", first) |
| + } |
| + planned := h.clock() |
| + h.advance(time.Minute) |
| + h.wakeHost("s") |
| + if second := h.m.Plan(ctx, h.rows.get("s")); second.Token == first.Token || second.Boot.Version != "v2" { |
| + t.Fatalf("second plan %+v: want another plan of v2", second) |
| + } |
| + h.advance(time.Minute) |
| + h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: activator.Gen{Build: "v2"}, |
| + Mode: activator.ModeHold, FirstWake: true, Token: first.Token, Executed: true}) |
| + if got, ok := h.book.HoldRanOn("s"); !ok || !got.same(markV2) || !got.At.Equal(planned) { |
| + t.Fatalf("marker = %+v, %v: want v2's, stamped at its wake's plan %v", got, ok, planned) |
| + } |
| + |
| + h.book.ClearHoldRan("s") |
| + h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: activator.Gen{Build: "v2"}, |
| + Mode: activator.ModeHold, FirstWake: true, Executed: true}) |
| + if got, ok := h.book.HoldRanOn("s"); !ok || !got.At.Equal(h.clock()) { |
| + t.Fatalf("marker = %+v, %v: a wake of unknown start, want the book's clock %v", got, ok, h.clock()) |
| } |
| } |
| -// Ruling 205's book half: a marker is stamped once by the book's clock (a |
| -// re-mark of the same build keeps it, and writes nothing), and the rollback |
| -// clear lifts only (acct, app)'s marker, and only for a pointer newer than it. |
| +// Ruling 205's book half: a marker keeps the At its caller gives (the |
| +// execution's start), and one given none is stamped by the book's clock; a |
| +// re-mark of the same build keeps the first stamp, and writes nothing. The |
| +// rollback clear lifts only (acct, app)'s marker, and only for a pointer |
| +// newer than it. |
| func TestTheHoldRanMarkerStampAndItsRollbackClear(t *testing.T) { |
| now := t0 |
| b := &FailureBook{now: func() time.Time { return now }} |
| @@ -547,4 +708,84 @@ func TestTheHoldRanMarkerStampAndItsRollbackClear(t *testing.T) { |
| if got, _ := b.HoldRanOn("a"); got.Build != "v3" || !got.At.Equal(now) { |
| t.Fatalf("marker %+v: want v3's, stamped at %v", got, now) |
| } |
| + started := now.Add(-30 * time.Second) |
| + b.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v4", At: started}) |
| + if got, _ := b.HoldRanOn("a"); got.Build != "v4" || !got.At.Equal(started) { |
| + t.Fatalf("marker %+v: want v4's, stamped at its execution's start %v, not the book's clock", got, started) |
| + } |
| +} |
| + |
| +// Ruling 205 at the scheduler (Task 3 fix round 1): an awake instance (a |
| +// socket instance, or a worker) serving v2 under v2's hold-ran marker, with |
| +// its good v1 installed (root's repoint, say), is not moved onto v1 until a |
| +// signed rollback lifts the marker. The gated attempt ends abandoned with |
| +// nothing run, is derived again after the abandon deferral, and is gated |
| +// again; once the pointer pass sees `carlos rollback`'s pointer and clears |
| +// the marker, the next derivation runs the move. |
| +func TestAMoveOntoTheInstalledGoodBuildWaitsForASignedRollback(t *testing.T) { |
| + for _, worker := range []bool{false, true} { |
| + name := "socket instance" |
| + if worker { |
| + name = "worker" |
| + } |
| + t.Run(name, func(t *testing.T) { |
| + var h *harness |
| + if worker { |
| + h = workerHarness(t, "h") |
| + } else { |
| + h = newHarness(t, "h") |
| + } |
| + var ( |
| + mu sync.Mutex |
| + gated int |
| + ) |
| + h.m.d.Logf = func(format string, args ...any) { |
| + if strings.Contains(fmt.Sprintf(format, args...), "waiting for a signed rollback") { |
| + mu.Lock() |
| + gated++ |
| + mu.Unlock() |
| + } |
| + } |
| + gatedTimes := func() int { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + return gated |
| + } |
| + moves := func() int { |
| + calls, _, _ := h.act.snapshot() |
| + return len(calls) + len(h.act.replacements()) |
| + } |
| + h.stage("v2") |
| + h.act.serve("h", "v2") |
| + h.book.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2", At: t0}) |
| + h.pass() |
| + waitFor(t, "the move onto v1 to be gated", func() bool { return gatedTimes() == 1 && !h.m.attemptRunning("h") }) |
| + if n := moves(); n != 0 { |
| + t.Fatalf("%d Swap/ReplaceWorker calls while v2's marker stands, want none", n) |
| + } |
| + // After the abandon deferral it is derived again, and gated again. |
| + h.advance(abandonDeferral + time.Second) |
| + h.pass() |
| + waitFor(t, "the move onto v1 to be gated again", func() bool { return gatedTimes() == 2 && !h.m.attemptRunning("h") }) |
| + if n := moves(); n != 0 { |
| + t.Fatalf("%d Swap/ReplaceWorker calls while v2's marker stands, want none", n) |
| + } |
| + |
| + // `carlos rollback` signs v1; the pointer pass clears the marker, |
| + // and the next derivation moves h onto v1. |
| + h.pointer("stable", "v1", h.clock()) |
| + h.advance(pointerReadEvery) |
| + h.pass() |
| + if got, ok := h.book.HoldRanOn("h"); ok { |
| + t.Fatalf("marker %+v survived the signed rollback", got) |
| + } |
| + waitFor(t, "the move onto v1", func() bool { return moves() == 1 }) |
| + if calls, _, _ := h.act.snapshot(); len(calls) == 1 && (worker || calls[0].a.Build != "v1") { |
| + t.Fatalf("swap %+v: want one onto v1 (and none for a worker)", calls[0].a) |
| + } |
| + if r := h.act.replacements(); len(r) == 1 && (!worker || r[0].a.Build != "v1") { |
| + t.Fatalf("replacement %+v: want one onto v1 (and none for a socket instance)", r[0].a) |
| + } |
| + }) |
| + } |
| } |
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go| index 75d7316a5..a1218b66d 100644 |
| --- a/internal/rollout/manager.go |
| +++ b/internal/rollout/manager.go |
| @@ -5,6 +5,7 @@ import ( |
| "errors" |
| "log" |
| "os" |
| + "slices" |
| "strings" |
| "sync" |
| "time" |
| @@ -140,6 +141,15 @@ type Manager struct { |
| // wake's, and acts on nothing that belongs to the newer one. |
| wakeTok map[string]uint64 |
| tokenCount uint64 |
| + // wakeAt holds, per host, the plan time of each planned first wake that |
| + // may still end with its build run (wakeStart, by token): the start of |
| + // that wake's execution, which a hold-ran marker the wake leaves carries |
| + // as its At (ruling 205, Task 3 fix round 1). Kept per token, not only |
| + // the newest plan's, because the marker is not token-scoped: a late |
| + // event of an older wake still marks the instance, with its own start. |
| + // An entry goes with its wake's last event (any kind but healthy), with |
| + // the host (forget), or when the host has wakeStartsKept newer ones. |
| + wakeAt map[string][]wakeStart |
| // goodMu serialises every serving- and good-build write with its |
| // acknowledgement check (served, publishServing, writeGood), so an older |
| @@ -158,7 +168,7 @@ func NewManager(ctx context.Context, d Deps) *Manager { |
| m := &Manager{d: d, ctx: ctx, |
| apps: map[appKey]*scheduler{}, hostApp: map[string]appKey{}, cfg: map[string]*configTrack{}, |
| retried: map[string]string{}, pointerRead: map[appKey]time.Time{}, nudging: map[string]bool{}, observing: map[string]uint64{}, |
| - planned: map[string]plannedWake{}, wakeTok: map[string]uint64{}} |
| + planned: map[string]plannedWake{}, wakeTok: map[string]uint64{}, wakeAt: map[string][]wakeStart{}} |
| m.policies = &policyCache{load: d.Policy, ttl: 30 * time.Second, now: d.Now, m: map[appKey]policyEntry{}} |
| return m |
| } |
| @@ -277,6 +287,7 @@ func (m *Manager) forget(routes []registry.Route) { |
| // A later event of the host's last wake then matches no token: the |
| // host is gone, and there is nothing of its wake left to act on. |
| delete(m.wakeTok, host) |
| + delete(m.wakeAt, host) |
| } |
| m.mu.Unlock() |
| if len(gone) > 0 { |
| @@ -397,9 +408,7 @@ func (m *Manager) observe(routes []registry.Route) []appKey { |
| } |
| s.mu.Unlock() |
| if rearm { |
| - m.mu.Lock() |
| - delete(m.pointerRead, k) |
| - m.mu.Unlock() |
| + m.pointerReadDue(k) |
| } |
| s.poke() |
| } |
| @@ -411,6 +420,19 @@ func (m *Manager) observe(routes []registry.Route) []appKey { |
| return keys |
| } |
| +// pointerReadDue makes the next applyPointers read k's pointers at once |
| +// rather than up to pointerReadEvery later (ruling 205). observe calls it |
| +// when a marked host's installed build moves; done() and OnGen call it when |
| +// they set a marker, because the rollback that superseded the hold build is |
| +// signed and installed while that build runs, before any marker exists, so |
| +// the pass that saw the install could not make the read due. The caller |
| +// holds no lock (Manager.mu is taken here). |
| +func (m *Manager) pointerReadDue(k appKey) { |
| + m.mu.Lock() |
| + delete(m.pointerRead, k) |
| + m.mu.Unlock() |
| +} |
| + |
| // routeChannel is the channel a route follows: "" means stable, as |
| // release's own (unexported) routeChannel reads it. |
| func routeChannel(rt registry.Route) string { |
| @@ -436,9 +458,11 @@ func routeChannel(rt registry.Route) string { |
| // marker: `carlos rollback` re-signs the pointer, root's automatic sibling |
| // repoint does not, so only an explicit rollback lets the good build boot |
| // over the marker. The signature is the pointer read's to verify |
| -// (release.Channel). observe makes the read due at once when a marked |
| -// host's installed build moves, so a rollback is not left waiting out |
| -// pointerReadEvery. |
| +// (release.Channel). The marker's At is the start of the execution that set |
| +// it (an attempt's dispatch, a planned wake's plan), so the rollback that |
| +// superseded a running hold build is newer than it. The read is made due at |
| +// once when a marked host's installed build moves and when a marker is set |
| +// (pointerReadDue), so a rollback is not left waiting out pointerReadEvery. |
| func (m *Manager) applyPointers(routes []registry.Route, keys []appKey) { |
| if m.d.Pointer == nil { |
| return |
| @@ -630,12 +654,72 @@ type plannedWake struct { |
| // newWakeToken starts a plan for host: a fresh token, recorded as the |
| // host's newest. It is taken before the plan registers anything (Plan), so |
| // the plan's registration and every event of its wake carry it. |
| -func (m *Manager) newWakeToken(host string) uint64 { |
| +// It also returns the plan's time, the start of the wake's execution |
| +// (noteWakeStart). |
| +func (m *Manager) newWakeToken(host string) (uint64, time.Time) { |
| + now := m.d.Now() |
| m.mu.Lock() |
| defer m.mu.Unlock() |
| m.tokenCount++ |
| m.wakeTok[host] = m.tokenCount |
| - return m.tokenCount |
| + return m.tokenCount, now |
| +} |
| + |
| +// wakeStart is a planned first wake's plan time, by its token |
| +// (Manager.wakeAt). |
| +type wakeStart struct { |
| + token uint64 |
| + at time.Time |
| +} |
| + |
| +// wakeStartsKept bounds Manager.wakeAt per host. A wake that ends with no |
| +// event (refused at its lease) leaves its entry until this many newer plans |
| +// push it out; a live wake is never that far behind its host's newest plan, |
| +// because the activator waits for a planned wake's outcome before it plans |
| +// again (outcomeFenceWait). |
| +const wakeStartsKept = 4 |
| + |
| +// noteWakeStart keeps the plan time of a planned first wake (tok), the only |
| +// kind whose end can leave a hold-ran marker (OnGen). Any mode is kept: the |
| +// activator reports the mode the wake ran in, which the plan does not fix. |
| +func (m *Manager) noteWakeStart(host string, tok uint64, at time.Time, plan activator.WakePlan) { |
| + if tok == 0 || !plan.FirstWake || plan.Refuse != "" || plan.Boot.StagedPath == "" { |
| + return |
| + } |
| + m.mu.Lock() |
| + defer m.mu.Unlock() |
| + starts := append(m.wakeAt[host], wakeStart{token: tok, at: at}) |
| + if len(starts) > wakeStartsKept { |
| + starts = slices.Delete(starts, 0, len(starts)-wakeStartsKept) |
| + } |
| + m.wakeAt[host] = starts |
| +} |
| + |
| +// wakeStartedAt is when the wake whose token is tok was planned, and drops |
| +// its entry when ended (the wake's last event). Zero means the start is not |
| +// known (a token pushed out by newer plans, or a host forgotten): the book |
| +// then stamps the marker with its own clock, later than the real start, so |
| +// only a pointer signed later still lifts it (the safe direction). |
| +func (m *Manager) wakeStartedAt(host string, tok uint64, ended bool) time.Time { |
| + m.mu.Lock() |
| + defer m.mu.Unlock() |
| + if tok == 0 { |
| + return time.Time{} |
| + } |
| + starts := m.wakeAt[host] |
| + i := slices.IndexFunc(starts, func(w wakeStart) bool { return w.token == tok }) |
| + if i < 0 { |
| + return time.Time{} |
| + } |
| + at := starts[i].at |
| + if ended { |
| + if starts = slices.Delete(starts, i, i+1); len(starts) == 0 { |
| + delete(m.wakeAt, host) |
| + } else { |
| + m.wakeAt[host] = starts |
| + } |
| + } |
| + return at |
| } |
| // notePlanned records what host's newest plan boots: an entry for a first |
diff --git a/internal/rollout/queue.go b/internal/rollout/queue.go| index c59763273..cfd535b2c 100644 |
| --- a/internal/rollout/queue.go |
| +++ b/internal/rollout/queue.go |
| @@ -47,6 +47,10 @@ type dispatch struct { |
| // An attempt that ends with nothing of its build run gets it back |
| // (rearmRetry; final review F8). |
| Forced bool |
| + // Started is when execute handed the attempt to the activator (Swap or |
| + // ReplaceWorker): its execution's start, the At of a hold-ran marker it |
| + // leaves (ruling 205, Task 3 fix round 1). Zero until then. |
| + Started time.Time |
| } |
| // hostSlot is one instance's desired state. The attempt to run is derived |
diff --git a/internal/rollout/scheduler.go b/internal/rollout/scheduler.go| index d7cfa4028..c2016ca50 100644 |
| --- a/internal/rollout/scheduler.go |
| +++ b/internal/rollout/scheduler.go |
| @@ -101,6 +101,24 @@ func (s *scheduler) execute(ctx context.Context, d dispatch) { |
| s.done(d, activator.SwapResult{Outcome: activator.SwapAbandoned, Reason: "route not readable"}, "") |
| return |
| } |
| + if movesBuild(d.ID.Cause) && rt.GoodVersion != "" && d.ID.Build == rt.GoodVersion { |
| + if mark, ok := holdRanFor(m.d.Book, s.key.acct, s.key.app, d.ID.Host); ok && mark.Build != d.ID.Build { |
| + // Ruling 205 at the scheduler (Task 3 fix round 1): a move onto |
| + // the installed good build over another build's hold-ran marker |
| + // is the planner's refused case, for an awake instance, a worker |
| + // or a sidecar member. Root's sibling repoint installs the good |
| + // build too, so only a signed rollback may run it. The queue |
| + // derives such a move as classRecover (the instance has served |
| + // its good build), so the gate is on the cause, not the class: a |
| + // restart or config attempt re-runs the build the instance |
| + // serves and is not gated. Nothing ran: abandoned unexecuted, so |
| + // done() defers the host (abandonDeferral) and the next |
| + // derivation tries again. The pointer pass, due at once on the |
| + // install (observe), clears the marker on a signed rollback. |
| + s.done(d, activator.SwapResult{Outcome: activator.SwapAbandoned, Reason: "a hold build ran here; waiting for a signed rollback"}, "") |
| + return |
| + } |
| + } |
| a, err := m.resolve(ctx, d, *rt) |
| if err != nil { |
| m.stagingRefused(*rt, d.ID.Build, err) |
| @@ -115,6 +133,10 @@ func (s *scheduler) execute(ctx context.Context, d dispatch) { |
| if d.Class == classAdopt { |
| d.FailedUnder = m.failedUnder(*rt, d.ID.Build) |
| } |
| + // The execution's start, after every read and before the activator runs |
| + // anything: a hold-ran marker this attempt leaves carries it (ruling 205), |
| + // so a rollback signed while it runs is newer than the marker. |
| + d.Started = m.d.Now() |
| if rt.IsSidecar() { |
| // Workers and native sidecar members keep stop-then-start on the |
| // same entry (spec §7.9). done() runs the cohort-failure transition |
| @@ -128,6 +150,13 @@ func (s *scheduler) execute(ctx context.Context, d dispatch) { |
| s.done(d, m.d.Act.Swap(ctx, a, s.newest), a.Mode) |
| } |
| +// movesBuild reports whether an attempt of cause moves the instance onto |
| +// its desired build (a release, install or retry), rather than re-running |
| +// the build it serves (a restart or config attempt). |
| +func movesBuild(c activator.Cause) bool { |
| + return c != activator.CauseRestart && c != activator.CauseConfig |
| +} |
| + |
| // done records an attempt's outcome and acts on it. mode is the attempt's |
| // swap mode, "" when no Swap ran. |
| func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.SwapMode) { |
| @@ -216,10 +245,10 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw |
| // The whole cohort-failure transition (spec §8.1 steps 1 and 2): the |
| // record (in memory at once, on disk as it can) and the halt. |
| r := *rec |
| - r.MarkHoldRan = mark |
| + r.MarkHoldRan, r.HoldRanAt = mark, d.Started |
| added, superseded, recErr = failCohort(m.d.Book, s.q, r) |
| } else if mark { |
| - markErr = m.d.Book.MarkHoldRan(d.ID.Host, HoldRan{Account: s.key.acct, App: s.key.app, Build: d.ID.Build}) |
| + markErr = m.d.Book.MarkHoldRan(d.ID.Host, HoldRan{Account: s.key.acct, App: s.key.app, Build: d.ID.Build, At: d.Started}) |
| } |
| if res.Outcome == activator.SwapAbandoned && !res.Executed { |
| // It is proven that nothing of the build ran (a refusal before its |
| @@ -245,6 +274,9 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw |
| serving = h.serving |
| } |
| s.mu.Unlock() |
| + if mark { |
| + m.pointerReadDue(s.key) |
| + } |
| defer s.poke() |
| defer m.changed() // runs first: the slot is free and the record is written |
| if recErr != nil { |
diff --git a/internal/rollout/sim_test.go b/internal/rollout/sim_test.go| index 134f5608b..5d76141f5 100644 |
| --- a/internal/rollout/sim_test.go |
| +++ b/internal/rollout/sim_test.go |
| @@ -66,9 +66,12 @@ import ( |
| // or a host retry names it (spec §6.4). A Pass may also lift the |
| // restriction, but only on rollback evidence (ruling 205): the |
| // instance's installed build is its good build, and its channel's |
| -// pointer names that build signed after the hold build ran. A marker |
| -// the Pass cleared without it is an I5 violation. (The book is read |
| -// here only to see what the Pass cleared; the evidence is the model's.) |
| +// pointer names that build signed after the execution that ran the hold |
| +// build started (its dispatch, or its wake's plan). A marker the Pass |
| +// cleared without it is an I5 violation. (The book is read here only to |
| +// see what the Pass cleared; the evidence is the model's.) When the |
| +// marked build is itself the installed good build, it boots under the |
| +// marked-build rule, and I1 requires its wake to boot. |
| // |
| // Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000). |
| // ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some |
| @@ -188,6 +191,10 @@ type simHost struct { |
| plan activator.WakePlan |
| planRoute registry.Route |
| wakeStart time.Time |
| + // planAt is when the plan the wake runs was made (the first plan, or |
| + // its replan at the exec): the start of its execution, which a hold-ran |
| + // marker the wake leaves is stamped with (ruling 205). |
| + planAt time.Time |
| stamp time.Time |
| failLater bool // observing: the build exits inside its observation |
| // underMark: the wake's plan was made while a hold-ran restriction |
| @@ -210,6 +217,9 @@ type simSwap struct { |
| newest func(string) uint64 |
| release chan struct{} |
| stamp time.Time // the FailedUnder the model expects |
| + // started is when the attempt was dispatched (scheduler.execute's |
| + // d.Started): the At of a hold-ran marker it leaves (ruling 205). |
| + started time.Time |
| // Set by finishSwap for the I3 record check: whether the instance |
| // serves the attempt's build, and the book's record for it beforehand. |
| serves bool |
| @@ -262,7 +272,8 @@ type sim struct { |
| // holdRan is the hold-ran restriction per host: the hold build that ran |
| // there without passing, until the host records a good build (goodAt), |
| // a host retry names it, or a Pass lifts it on rollback evidence. |
| - // holdRanAt is when it was set: the book's stamp, which a rollback |
| + // holdRanAt is the start of the execution that set it (an attempt's |
| + // dispatch, a planned wake's plan; Task 3 fix round 1), which a rollback |
| // pointer must be newer than. |
| holdRan map[string]string |
| holdRanAt map[string]time.Time |
| @@ -490,7 +501,7 @@ func (s *sim) finishSwap(sw *simSwap) activator.SwapResult { |
| if a.Mode == activator.ModeHold { |
| h.phase, h.gen = simAsleep, activator.Gen{} |
| if !sw.serves { |
| - s.holdRanOn(h, a.Build) |
| + s.holdRanOn(h, a.Build, sw.started) |
| } |
| } |
| } |
| @@ -522,7 +533,7 @@ func (s *sim) finishSwap(sw *simSwap) activator.SwapResult { |
| } |
| if res.Outcome == activator.SwapFailed && a.Mode == activator.ModeHold && !sw.serves { |
| // A hold adoption that ran, failed and was not held down. |
| - s.holdRanOn(h, a.Build) |
| + s.holdRanOn(h, a.Build, sw.started) |
| } |
| return res |
| } |
| @@ -583,6 +594,8 @@ func (s *sim) launch(d dispatch) { |
| return |
| } |
| ev.swap.stamp = stamp |
| + // execute ran up to Swap within this step, so its dispatch time is now. |
| + ev.swap.started = s.now |
| s.swaps[d.ID.Host] = ev.swap |
| s.logf("launch %s %s %s rev %d canary %v class %d", d.ID.Host, d.ID.Cause, d.ID.Build, d.SchedRev, d.Canary, d.Class) |
| } |
| @@ -800,12 +813,12 @@ func (s *sim) heldDownOn(host, build string) bool { |
| } |
| // holdRanOn is the ground truth of a hold build that ran on h without |
| -// passing or being held down there (N2). |
| -// The stamp is the first run's: running the same build again under its own |
| -// restriction leaves the book's marker, and its stamp, as they were. |
| -func (s *sim) holdRanOn(h *simHost, build string) { |
| +// passing or being held down there (N2), in the execution that started at |
| +// start. The stamp is the first run's: running the same build again under |
| +// its own restriction leaves the book's marker, and its stamp, as they were. |
| +func (s *sim) holdRanOn(h *simHost, build string, start time.Time) { |
| if s.holdRan[h.name] != build { |
| - s.holdRanAt[h.name] = s.now |
| + s.holdRanAt[h.name] = start |
| } |
| s.holdRan[h.name] = build |
| s.logf("ground truth: hold build %s ran on %s without a verdict", build, h.name) |
| @@ -821,9 +834,11 @@ func (s *sim) goodAt(h *simHost, build string) { |
| // checkI5 is I5 at the exec of build on h, for a wake planned against rt. |
| // The installed good build is barred too (ruling 205): root's sibling |
| // repoint installs it without any rollback, and only a Pass that saw a |
| -// signed one lifts the restriction. |
| +// signed one lifts the restriction. The marked build itself is exempt even |
| +// when it is the good build (Task 3 fix round 1): booting it is the |
| +// marked-build rule, not a boot of the good build in its place. |
| func (s *sim) checkI5(h *simHost, rt registry.Route, build string) { |
| - if mark := s.holdRan[h.name]; mark != "" && build == h.good { |
| + if mark := s.holdRan[h.name]; mark != "" && build == h.good && build != mark { |
| violate("I5", "%s boots its good build %s (installed %s) after hold build %s ran there, with no signed rollback since", h.name, build, rt.Version, mark) |
| } |
| } |
| @@ -853,7 +868,7 @@ func (s *sim) wakeBegin() { |
| s.checkWakeRefused(h, row.Version, "its plan refused: "+plan.Refuse) |
| return |
| } |
| - h.plan, h.planRoute, h.wakeStart, h.stamp = plan, row, s.now, s.stampFor(h, plan.Boot.Version) |
| + h.plan, h.planRoute, h.wakeStart, h.planAt, h.stamp = plan, row, s.now, s.now, s.stampFor(h, plan.Boot.Version) |
| h.underMark = s.holdRan[h.name] != "" |
| s.logf("wake %s planned: boot %s first %v fallback %v mode %s", h.name, plan.Boot.Version, plan.FirstWake, plan.Fallback != nil, plan.Mode) |
| } |
| @@ -863,6 +878,16 @@ func (s *sim) checkWakeRefused(h *simHost, installed, why string) { |
| return |
| } |
| if mark := s.holdRan[h.name]; mark != "" { |
| + if installed == mark && installed == h.good { |
| + // The marked build is the installed good build (Task 3 fix round |
| + // 1): it boots under the marked-build rule. A record does not |
| + // hold a build the instance served, and the hold-down returned |
| + // above, so only a staging refusal may refuse it. |
| + if s.builds[installed].misstaged { |
| + return |
| + } |
| + violate("I1", "%s has its good build %s installed, the hold build that ran there, but its wake boots nothing: %s", h.name, installed, why) |
| + } |
| // Restricted (N2): refused for an installed build held for the |
| // instance, the marked build included (Astra R3 N7: the marker lifts |
| // no record), for its installed good build until a signed rollback |
| @@ -917,7 +942,7 @@ func (s *sim) wakeExec() { |
| s.checkWakeRefused(h, s.rows.get(h.name).Version, "the plan went stale and the replan refused") |
| return |
| } |
| - plan, h.plan, h.stamp = next, next, s.stampFor(h, next.Boot.Version) |
| + plan, h.plan, h.planAt, h.stamp = next, next, s.now, s.stampFor(h, next.Boot.Version) |
| h.underMark = s.holdRan[h.name] != "" |
| s.logf("wake %s replanned: boot %s first %v", h.name, plan.Boot.Version, plan.FirstWake) |
| } |
| @@ -967,7 +992,7 @@ func (s *sim) wakeExec() { |
| h.phase, h.gen = simAsleep, activator.Gen{} |
| s.logf("wake %s: first wake of %s ended after its exec, no verdict (mode %s)", h.name, boot.Version, plan.Mode) |
| if plan.Mode == activator.ModeHold { |
| - s.holdRanOn(h, boot.Version) |
| + s.holdRanOn(h, boot.Version, h.planAt) |
| s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Gen: activator.Gen{Build: boot.Version}, Mode: plan.Mode, |
| FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev, Executed: true}) |
| } |
| @@ -1060,7 +1085,7 @@ func (s *sim) sleep() { |
| // The observed build was healthy, so it ran: a hold one leaves the |
| // hold-ran restriction (N2). |
| if h.plan.Mode == activator.ModeHold { |
| - s.holdRanOn(h, g.Build) |
| + s.holdRanOn(h, g.Build, h.planAt) |
| } |
| s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.planRoute, Gen: g, Mode: h.plan.Mode, FirstWake: true, At: s.now, |
| Token: h.plan.Token, ConfigRev: h.plan.ConfigRev, Executed: true}) |
diff --git a/internal/rollout/status.go b/internal/rollout/status.go| index 132457348..ddc5c1eb1 100644 |
| --- a/internal/rollout/status.go |
| +++ b/internal/rollout/status.go |
| @@ -82,8 +82,9 @@ func (m *Manager) Status() status.RolloutReport { |
| // good build, its wakes are refused (decideHoldRan), so it is down |
| // when a record holds its installed build, the marked build |
| // included (Astra R3 N7), and when its installed build is the good |
| - // build, until a signed rollback lifts the marker (ruling 205). |
| - // Awake, it serves what it serves. |
| + // build, until a signed rollback lifts the marker (ruling 205), |
| + // unless the good build is the marked build itself, which its wake |
| + // boots. Awake, it serves what it serves. |
| if mark, ok := holdRanFor(m.d.Book, s.key.acct, s.key.app, host); ok && h.serving == "" { |
| switch { |
| case hr.HeldBy != "" && h.good != "": |
| @@ -93,7 +94,9 @@ func (m *Manager) Status() status.RolloutReport { |
| if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok { |
| hr.Reason = reasonWord(rec.Reason) |
| } |
| - if h.good != "" && h.wantBuild == h.good { |
| + // The marked build installed boots under its own rule, even |
| + // when it is the good build (Task 3 fix round 1). |
| + if h.good != "" && h.wantBuild == h.good && h.wantBuild != mark.Build { |
| hr.Attempt = status.AttemptDown |
| } |
| } |
diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go| index 5b86911a5..91f055cca 100644 |
| --- a/internal/rollout/wakeplan.go |
| +++ b/internal/rollout/wakeplan.go |
| @@ -110,8 +110,17 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) { |
| // decideHoldRan is decideWake for an instance with a hold-ran marker (final |
| // review wave 2, N2; spec §6.4: once a hold build has executed against an |
| // instance's database, nothing falls back automatically on that instance). |
| -// The good build never boots, as a recovery or as a fallback, and the |
| -// marker never lifts a failure record. With a good build: |
| +// The good build never boots in place of the marked build, as a recovery |
| +// or as a fallback, and the marker never lifts a failure record. |
| +// - the installed build is the marked build: it boots, in hold mode with |
| +// no fallback, unless a failure record holds it for the instance (Astra |
| +// R3 N7: spec §8.4 exempts an instance that passed on the build, not one |
| +// where it merely ran). That holds even when the marked build is also |
| +// the good build (Task 3 fix round 1): the instance has served it, so a |
| +// sibling's record does not hold it (Blocks with served), and booting |
| +// it is the marked-build rule, not a fallback. |
| +// |
| +// With a good build, for any other installed build: |
| // - the installed build is the good build: refused. An installed good |
| // build is no proof of an explicit rollback, because root's automatic |
| // sibling repoint installs it too (ruling 205, Astra R3). The proof is |
| @@ -119,12 +128,9 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) { |
| // pointer pass (applyPointers) clears the marker when it sees one; the |
| // next wake then boots the good build as an ordinary wake. The planner |
| // itself reads no pointer. |
| -// - the installed build is held for the instance by a failure record, |
| -// the marked build included: refused (Astra R3 N7). Spec §8.4 exempts an |
| -// instance that passed on the build, not one where it merely ran, so a |
| -// sibling's failure of the marked build holds this instance too. With |
| -// the good build barred as well, it stays down until a retry or a newer |
| -// build. |
| +// - the installed build is held for the instance by a failure record: |
| +// refused. With the good build barred as well, the instance stays down |
| +// until a retry or a newer build. |
| // |
| // Otherwise the installed build boots as a first wake with no fallback: the |
| // marked build in hold mode, whose failure here is a hold-down, any other |
| @@ -138,6 +144,10 @@ func decideHoldRan(in wakeInputs, mark HoldRan, hasGood bool) (activator.WakePla |
| acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| installed := rt.Version |
| switch { |
| + case installed == mark.Build: |
| + if hasGood && in.Book.Blocks(acct, app, installed, host, installed == rt.GoodVersion) { |
| + return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| + } |
| case hasGood && installed == rt.GoodVersion: |
| return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| case hasGood && in.Book.Blocks(acct, app, installed, host, false): |
| @@ -194,10 +204,11 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W |
| if !Managed(rt) { |
| return activator.WakePlan{} |
| } |
| - tok := m.newWakeToken(rt.Host) |
| + tok, planAt := m.newWakeToken(rt.Host) |
| defer func() { |
| plan.Token = tok |
| m.notePlanned(rt, plan, tok) |
| + m.noteWakeStart(rt.Host, tok, planAt, plan) |
| }() |
| if m.attemptRunning(rt.Host) { |
| // The instance's attempt has not reported yet (spec §6.4, §8.1): a |
| @@ -300,14 +311,16 @@ func (m *Manager) planRow(ctx context.Context, rt registry.Route, tok uint64) ac |
| // |
| // A hold-ran marker (final review wave 2, N2), set since the plan or |
| // before it, applies decideHoldRan's rule at the exec: |
| -// - the good build may not boot, whatever the route installs: as a |
| -// recovery, a fallback, an untried first wake's stand-in, or the |
| +// - the marked build may not boot while it is held down there or, with a |
| +// good build, while a failure record holds it for the instance (Astra |
| +// R3 N7: the marker lifts no record). When it is also the good build it |
| +// is under this rule only as the installed build (Task 3 fix round 1), |
| +// and a record does not hold a build the instance served; |
| +// - otherwise the good build may not boot, whatever the route installs: |
| +// as a recovery, a fallback, an untried first wake's stand-in, or the |
| // installed build itself (ruling 205: only the pointer pass's clearing |
| // of the marker on a signed rollback lets it boot again, and the check |
| // reads no pointer); |
| -// - the marked build may not boot while it is held down there or, with a |
| -// good build, while a failure record holds it for the instance (Astra |
| -// R3 N7: the marker lifts no record); |
| // - any other build is checked as without a marker. |
| // |
| // The book is in memory; nothing here blocks. |
| @@ -322,14 +335,15 @@ func (m *Manager) planCheck(rt registry.Route, plan activator.WakePlan) func(bui |
| } |
| return func(build string) error { |
| if mark, ok := holdRanFor(m.d.Book, acct, app, host); ok { |
| + isGood := hasGood && build == rt.GoodVersion |
| switch { |
| - case hasGood && build == rt.GoodVersion: |
| - return errors.New(HeldDownRefusal(mark.Build)) |
| - case build == mark.Build: |
| - if m.d.Book.HeldDown(acct, app, build, host) || (hasGood && m.d.Book.Blocks(acct, app, build, host, false)) { |
| + case build == mark.Build && (!isGood || rt.Version == mark.Build): |
| + if m.d.Book.HeldDown(acct, app, build, host) || (hasGood && m.d.Book.Blocks(acct, app, build, host, isGood)) { |
| return held(build) |
| } |
| return nil |
| + case isGood: |
| + return errors.New(HeldDownRefusal(mark.Build)) |
| } |
| } |
| if !plan.FirstWake || build != boot { |
| @@ -375,6 +389,10 @@ func (m *Manager) attemptRunning(host string) bool { |
| // activator serves (acknowledged). A token of zero matches nothing. |
| func (m *Manager) OnGen(ev activator.GenEvent) { |
| rt := ev.Route |
| + // The wake's plan time, the start of its execution: a hold-ran marker it |
| + // leaves carries it (ruling 205). Every kind but healthy is the wake's |
| + // last event, which lets the entry go. |
| + started := m.wakeStartedAt(rt.Host, ev.Token, ev.Kind != activator.GenHealthy) |
| m.mu.Lock() |
| newest := ev.Token != 0 && m.wakeTok[rt.Host] == ev.Token |
| // The wake has an outcome: its build is no longer only planned, so its |
| @@ -431,10 +449,16 @@ func (m *Manager) OnGen(ev activator.GenEvent) { |
| // must not boot the good build. A fact about the instance, like a |
| // failure, so it is not token-scoped. |
| if ev.Executed && ev.FirstWake && ev.Mode == activator.ModeHold && ev.Gen.Build != "" { |
| - mark := HoldRan{Account: release.RouteAccount(rt), App: rt.App, Build: ev.Gen.Build} |
| + // At is the wake's plan time, its execution's start (ruling 205): |
| + // a rollback signed while the wake ran is newer than the marker. |
| + if started.IsZero() { |
| + m.d.Logf("rollout: %s: the start of the wake that ran %s is not known; its hold-ran marker is stamped now", rt.Host, ev.Gen.Build) |
| + } |
| + mark := HoldRan{Account: release.RouteAccount(rt), App: rt.App, Build: ev.Gen.Build, At: started} |
| if err := m.d.Book.MarkHoldRan(rt.Host, mark); err != nil { |
| m.d.Logf("rollout: %s: write hold-ran marker for %s: %v (held in memory)", rt.Host, mark.Build, err) |
| } |
| + m.pointerReadDue(appKey{mark.Account, mark.App}) |
| } |
| } |
| } |