amadan
zdd-readiness: write-ahead intents and schema 2 keep failures across a failed save and a restart (F5)
An adoption (in execute, after resolve) and a planned first wake (at the end of Plan) write an intent to the failure book before anything runs; a failed write refuses the execution. The intent ends after the outcome's own book write: in done() for attempts, on the wake's first terminal OnGen event for wakes (never on dropUnownedWakes, plan review #1). OpenFailureBook turns intents left on disk into failure records (held down and marked in hold mode, marker Exec = intent ID, At = intent start); armRollout flushes them or refuses managed starts, a declined box flushes and logs. The book is written as schema 2 and reads 1 and 2. The intent records the effective mode (Activator.EffectiveMode, foreign rows run overlap as hold, plan review #2). In the activator every planned wake with a token now ends with one terminal event: refused/replaced/abandoned exits send GenCancelled (Executed as the build ran), outside the outcome fence. The unreadable-book refusal is one shared constant (rollout.BookUnreadableRefusal). The simulation models the new events and checks I6: no intent outlives its execution.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
19 files changed,
+1359
−61
internal/activator/holdran_event_test.go+25 −9internal/activator/intent_events_test.go+152 −0internal/activator/swap.go+13 −0internal/activator/swap_hold_test.go+18 −0internal/activator/wake_token_test.go+16 −1internal/activator/wakeplan.go+53 −1internal/host/rollout.go+17 −2internal/host/rollout_test.go+98 −0internal/host/rolloutdeclined.go+14 −3internal/host/rolloutdeclined_test.go+38 −0internal/rollout/bookseq_test.go+7 −0internal/rollout/failures.go+208 −18internal/rollout/fake_test.go+11 −0internal/rollout/holdran_test.go+7 −4internal/rollout/intent_test.go+506 −0internal/rollout/manager.go+21 −12internal/rollout/scheduler.go+40 −0internal/rollout/sim_test.go+53 −9internal/rollout/wakeplan.go+62 −2
diff --git a/internal/activator/holdran_event_test.go b/internal/activator/holdran_event_test.go| index 0368db5d5..3cc5bf872 100644 |
| --- a/internal/activator/holdran_event_test.go |
| +++ b/internal/activator/holdran_event_test.go |
| @@ -15,9 +15,11 @@ import ( |
| // observation of a healthy first wake, or the wake's lifecycle ends it after |
| // the exec (here Shutdown during the health check). The second, which |
| // reported nothing before, is reported for a hold first wake only, under the |
| -// host's outcome fence, so no new wake plans before the scheduler has it. A |
| -// wake that ran nothing reports nothing, and an overlap wake ended the same |
| -// way is unchanged (nothing reported). |
| +// host's outcome fence, so no new wake plans before the scheduler has it. |
| +// Since Task 5 every planned wake with a token ends with a terminal event: |
| +// one that ran nothing is cancelled, not Executed, and an overlap wake ended |
| +// the same way is cancelled Executed. Neither records anything, so neither |
| +// is delivered under the fence. |
| func TestAFirstWakeEndedAfterItsExecReportsThatItRan(t *testing.T) { |
| const host = "one.example.com" |
| endInHealth := func(t *testing.T, mode SwapMode, refuse bool) (*Service, *genLog, *planGenBackend, chan bool) { |
| @@ -69,17 +71,31 @@ func TestAFirstWakeEndedAfterItsExecReportsThatItRan(t *testing.T) { |
| } |
| }) |
| t.Run("overlap, ended in its health check", func(t *testing.T) { |
| - _, evs, _, _ := endInHealth(t, ModeOverlap, false) |
| + _, evs, _, fenced := endInHealth(t, ModeOverlap, false) |
| + ev := evs.wait(t, GenCancelled) |
| + if !ev.Executed || ev.Gen.Build != "v2" || ev.Mode != ModeOverlap || ev.Token != 41 { |
| + t.Fatalf("event %+v: want GenCancelled, Executed, naming v2, in overlap mode, with the plan's token", ev) |
| + } |
| + if <-fenced { |
| + t.Fatal("an overlap wake's cancellation records nothing, yet it held the outcome fence") |
| + } |
| time.Sleep(100 * time.Millisecond) |
| - if k := evs.kinds(); len(k) != 0 { |
| - t.Fatalf("events %v: an overlap first wake ended by its lifecycle reports nothing, as before", k) |
| + if k := evs.kinds(); len(k) != 1 { |
| + t.Fatalf("events %v: want the one cancellation", k) |
| } |
| }) |
| t.Run("hold, refused before its exec", func(t *testing.T) { |
| - _, evs, b, _ := endInHealth(t, ModeHold, true) |
| + _, evs, b, fenced := endInHealth(t, ModeHold, true) |
| + ev := evs.wait(t, GenCancelled) |
| + if ev.Executed || ev.Token != 41 || len(b.started()) != 0 { |
| + t.Fatalf("event %+v, started %+v: a wake that ran nothing is cancelled, not Executed", ev, b.started()) |
| + } |
| + if <-fenced { |
| + t.Fatal("a cancellation with nothing run held the outcome fence") |
| + } |
| time.Sleep(100 * time.Millisecond) |
| - if k := evs.kinds(); len(k) != 0 || len(b.started()) != 0 { |
| - t.Fatalf("events %v, started %+v: a wake that ran nothing reports nothing", k, b.started()) |
| + if k := evs.kinds(); len(k) != 1 { |
| + t.Fatalf("events %v: want the one cancellation", k) |
| } |
| }) |
| t.Run("hold, its observation cancelled", func(t *testing.T) { |
diff --git a/internal/activator/intent_events_test.go b/internal/activator/intent_events_test.go| new file mode 100644 |
| index 000000000..c164caaec |
| --- /dev/null |
| +++ b/internal/activator/intent_events_test.go |
| @@ -0,0 +1,152 @@ |
| +package activator |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "sync/atomic" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// Task 5, plan review #1: the scheduler ends a planned first wake's |
| +// write-ahead intent only on that wake's own terminal event, never on the |
| +// activator letting the host go (a failed wake's entry goes before its |
| +// outcome is delivered). So every planned wake that carries a token ends |
| +// with exactly one token-carrying terminal event. The exits that sent none |
| +// before now send GenCancelled: not Executed when the plan's build never |
| +// ran (refused at the lease, at its check, replaced by a replan), Executed |
| +// when it ran and its lifecycle ended it with no verdict. None of them is an |
| +// outcome the fence must cover: they record nothing, so a new wake may plan |
| +// before they arrive. |
| + |
| +// terminal reports the token's events that settle its build's fate (the |
| +// scheduler's End): every kind but a first wake's healthy. |
| +func terminal(evs []GenEvent, tok uint64) []GenEvent { |
| + var out []GenEvent |
| + for _, ev := range evs { |
| + if ev.Token == tok && (ev.Kind != GenHealthy || !ev.FirstWake) { |
| + out = append(out, ev) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// settle waits until the activator has gone quiet: events arrive on the |
| +// wake's goroutine after the request returns. |
| +func settle() { time.Sleep(100 * time.Millisecond) } |
| + |
| +func TestAPlannedWakeRefusedAtItsLeaseIsCancelledUnrun(t *testing.T) { |
| + dir := t.TempDir() |
| + good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| + a, b, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Token: 51, ConfigRev: "c1"}) |
| + var fenced atomic.Bool |
| + a.OnGen = func(ev GenEvent) { |
| + a.mu.Lock() |
| + _, held := a.outcomeFences["one.example.com"] |
| + a.mu.Unlock() |
| + fenced.Store(fenced.Load() || held) |
| + evs.add(ev) |
| + } |
| + b.mu.Lock() |
| + b.acquireErr = errors.New("lease held by a previous incarnation") |
| + b.mu.Unlock() |
| + if _, err := a.beginRequest(context.Background(), testRoute("one.example.com")); err == nil { |
| + t.Fatal("the wake was served without its lease") |
| + } |
| + settle() |
| + got := terminal(evs.all(), 51) |
| + if len(got) != 1 || got[0].Kind != GenCancelled || got[0].Executed || got[0].ConfigRev != "c1" { |
| + t.Fatalf("events %+v: want one GenCancelled, not executed, with the plan's token", evs.all()) |
| + } |
| + if len(evs.all()) != 1 { |
| + t.Fatalf("events %+v: want the one", evs.all()) |
| + } |
| + if fenced.Load() { |
| + t.Fatal("an event that records nothing was delivered under the outcome fence") |
| + } |
| +} |
| + |
| +// A plan refused by its check before the exec, whose replan's check refuses |
| +// too: both plans are cancelled unrun, each once, by its own token. |
| +func TestARefusedReplanCancelsBothPlans(t *testing.T) { |
| + dir := t.TempDir() |
| + good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| + a, b, evs := planActivator(t, WakePlan{}) |
| + refuse := func(string) error { return errors.New("held") } |
| + var plans atomic.Uint64 |
| + a.Planner = func(context.Context, registry.Route) WakePlan { |
| + return WakePlan{Boot: next, Mode: ModeHold, FirstWake: true, Fallback: &good, Check: refuse, Token: 60 + plans.Add(1)} |
| + } |
| + if _, err := a.beginRequest(context.Background(), testRoute("one.example.com")); !errors.Is(err, errPlanStale) { |
| + t.Fatalf("err = %v, want the stale plan", err) |
| + } |
| + settle() |
| + if gens := b.started(); len(gens) != 0 { |
| + t.Fatalf("spawned %+v", gens) |
| + } |
| + all := evs.all() |
| + if len(all) != 2 { |
| + t.Fatalf("events %+v: want one per plan", all) |
| + } |
| + for _, tok := range []uint64{61, 62} { |
| + got := terminal(all, tok) |
| + if len(got) != 1 || got[0].Kind != GenCancelled || got[0].Executed { |
| + t.Fatalf("token %d: events %+v, want one GenCancelled, not executed", tok, all) |
| + } |
| + } |
| +} |
| + |
| +// A plan replaced by a replan that boots: the replaced plan is cancelled |
| +// unrun, and the replan's own events follow. |
| +func TestAReplacedPlanIsCancelledUnrun(t *testing.T) { |
| + dir := t.TempDir() |
| + good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2") |
| + a, b, evs := planActivator(t, WakePlan{}) |
| + held := func(build string) error { |
| + if build == "v2" { |
| + return errors.New("v2 failed on a sibling") |
| + } |
| + return nil |
| + } |
| + var plans atomic.Int32 |
| + a.Planner = func(context.Context, registry.Route) WakePlan { |
| + if plans.Add(1) == 1 { |
| + return WakePlan{Boot: next, Mode: ModeOverlap, FirstWake: true, Fallback: &good, Check: held, Token: 71} |
| + } |
| + return WakePlan{Boot: good, Mode: ModeOverlap, Check: held, Token: 72} |
| + } |
| + mustServe(t, a) |
| + settle() |
| + if gens := b.started(); len(gens) != 1 || gens[0].Build != "v1" { |
| + t.Fatalf("spawned %+v, want the good v1 alone", gens) |
| + } |
| + old := terminal(evs.all(), 71) |
| + if len(old) != 1 || old[0].Kind != GenCancelled || old[0].Executed { |
| + t.Fatalf("replaced plan's events %+v: want one GenCancelled, not executed", evs.all()) |
| + } |
| + cur := terminal(evs.all(), 72) |
| + if len(cur) != 1 || cur[0].Kind != GenHealthy { |
| + t.Fatalf("replan's events %+v: want its healthy wake", evs.all()) |
| + } |
| +} |
| + |
| +// A planned wake whose worker token could not be minted reports no |
| +// wokenInstance at all; it still ends with its cancellation. And the flag |
| +// off (no OnGen) hears nothing, whatever the plan carries. |
| +func TestAWakeWithNoResultIsStillCancelled(t *testing.T) { |
| + plan := WakePlan{Boot: stagedID(t, t.TempDir(), "v2"), Mode: ModeHold, FirstWake: true, Token: 81} |
| + a, _, evs := planActivator(t, plan) |
| + e := &actEntry{host: "one.example.com", route: testRoute("one.example.com"), plan: plan} |
| + a.afterWake(e, nil, errors.New("start: no token")) |
| + if got := terminal(evs.all(), 81); len(got) != 1 || got[0].Kind != GenCancelled || got[0].Executed { |
| + t.Fatalf("events %+v: want one GenCancelled, not executed", evs.all()) |
| + } |
| + a.OnGen = nil |
| + a.afterWake(e, nil, errors.New("start: no token")) // must not panic or emit |
| + a.afterWake(e, &wokenInstance{}, errors.New("refused")) |
| + if len(evs.all()) != 1 { |
| + t.Fatalf("events %+v after OnGen was unset", evs.all()) |
| + } |
| +} |
diff --git a/internal/activator/swap.go b/internal/activator/swap.go| index 0142fc7e8..5c47e68bc 100644 |
| --- a/internal/activator/swap.go |
| +++ b/internal/activator/swap.go |
| @@ -282,6 +282,19 @@ func (a *Service) Swap(ctx context.Context, at Attempt, newest func(host string) |
| return abandon(errSwapMode) |
| } |
| +// EffectiveMode is the mode Swap runs an attempt of mode m on host in: an |
| +// overlap on a foreign-layout row runs as a hold (foreignRow, spec §2.9), |
| +// anything else as given. The scheduler writes it into the attempt's |
| +// write-ahead intent before calling Swap (rollout Task 5, plan review |
| +// finding 2), so an agent that crashes mid-swap reopens its failure book |
| +// with the hold it was running. Called without a.mu held. |
| +func (a *Service) EffectiveMode(host string, m SwapMode) SwapMode { |
| + if m == ModeOverlap && a.foreignRow(host) { |
| + return ModeHold |
| + } |
| + return m |
| +} |
| + |
| // attemptCtx is the attempt's own lifetime. It ends with the caller's ctx, |
| // with the agent (Shutdown cancels lifeCtx), or with the barrier, which |
| // calls the returned cancel through the member. |
diff --git a/internal/activator/swap_hold_test.go b/internal/activator/swap_hold_test.go| index fc1ce0a33..299edd80e 100644 |
| --- a/internal/activator/swap_hold_test.go |
| +++ b/internal/activator/swap_hold_test.go |
| @@ -707,6 +707,24 @@ func TestSwapOverlapOnAForeignRowRunsAsAHold(t *testing.T) { |
| } |
| } |
| +// Task 5, plan review #2: the scheduler writes an attempt's intent in the |
| +// mode Swap will run it in, which EffectiveMode answers with Swap's own |
| +// override: an overlap on a foreign row is a hold, anything else is as given. |
| +func TestEffectiveModeIsTheModeSwapRuns(t *testing.T) { |
| + f := newSwapFixture(t) |
| + host := f.attempt(ModeOverlap).Host |
| + if m := f.a.EffectiveMode(host, ModeOverlap); m != ModeOverlap { |
| + t.Fatalf("flat row: %s, want overlap", m) |
| + } |
| + f.b.layoutFn = func(registry.Route) (Layout, string) { return LayoutForeign, "hand-set socket" } |
| + if m := f.a.EffectiveMode(host, ModeOverlap); m != ModeHold { |
| + t.Fatalf("foreign row: %s, want hold", m) |
| + } |
| + if m := f.a.EffectiveMode(host, ModeHold); m != ModeHold { |
| + t.Fatalf("foreign row, hold: %s", m) |
| + } |
| +} |
| + |
| // A nested row can hold-swap now that root mounts the staged build (Task 9): |
| // the refusal Task 6 made for it is gone, and the hold runs to SwapDone. |
| func TestSwapHoldAcceptsANestedRow(t *testing.T) { |
diff --git a/internal/activator/wake_token_test.go b/internal/activator/wake_token_test.go| index aac2a7e45..fdd76c088 100644 |
| --- a/internal/activator/wake_token_test.go |
| +++ b/internal/activator/wake_token_test.go |
| @@ -110,7 +110,22 @@ func TestEveryGenEventCarriesItsWakesToken(t *testing.T) { |
| } |
| mustServe(t, a) |
| evs.wait(t, GenHealthy) |
| - return evs.all() |
| + // The replaced plan's own cancellation (Task 5) carries its own |
| + // token; every other event is the replan's. |
| + var out []GenEvent |
| + for _, ev := range evs.all() { |
| + if ev.Token == token-1 { |
| + if ev.Kind != GenCancelled || ev.Executed || ev.ConfigRev != "cfg-6" { |
| + t.Fatalf("replaced plan's event %+v: want GenCancelled, not executed, with its own token and config", ev) |
| + } |
| + continue |
| + } |
| + out = append(out, ev) |
| + } |
| + if len(out) == len(evs.all()) { |
| + t.Fatalf("events %+v: the replaced plan was not cancelled", evs.all()) |
| + } |
| + return out |
| }}, |
| } { |
| t.Run(tc.name, func(t *testing.T) { |
diff --git a/internal/activator/wakeplan.go b/internal/activator/wakeplan.go| index fb035f276..3849e2989 100644 |
| --- a/internal/activator/wakeplan.go |
| +++ b/internal/activator/wakeplan.go |
| @@ -92,12 +92,25 @@ func (a *Service) checkPlan(ctx context.Context, e *actEntry, rt registry.Route, |
| } |
| if next.Check != nil { |
| if nerr := next.Check(next.Boot.Version); nerr != nil { |
| + // The replan never becomes the wake's plan, so afterWake will |
| + // report only the first plan: the replan's own terminal event |
| + // is sent here (rollout Task 5). Nothing of it ran. A planner |
| + // that handed out the same token again gets one event for it. |
| + if next.Token != plan.Token { |
| + a.cancelPlanned(rt, next, false) |
| + } |
| return plan, fmt.Errorf("%w: %v", errPlanStale, nerr) |
| } |
| } |
| a.mu.Lock() |
| e.plan, e.firstWake = next, next.FirstWake |
| a.mu.Unlock() |
| + // The replaced plan's build never ran (its check refused it before the |
| + // exec), and afterWake now reports the replan: the replaced plan's |
| + // terminal event is sent here (rollout Task 5). |
| + if plan.Token != next.Token { |
| + a.cancelPlanned(rt, plan, false) |
| + } |
| // The replanned build (and its fallback) is published before it execs |
| // (spec §4.2, §8.5). A failed write execs neither: the wake fails with |
| // errStagedBuildNotPublished, which is abandonment, never a failure. |
| @@ -646,11 +659,46 @@ func (a *Service) emitGen(ev GenEvent) { |
| } |
| } |
| +// cancelPlanned reports that a planned wake of plan ended with no verdict |
| +// on its build and nothing owed to the outcome fence (rollout Task 5, plan |
| +// review #1): refused at its lease, its exec or its check, replaced by a |
| +// replan, or (ran true) an overlap or ordinary wake its lifecycle ended |
| +// after the exec. It records nothing; it is the wake's terminal event, which |
| +// ends the scheduler's write-ahead intent for it. Only a plan with a token |
| +// is reported, and only to an OnGen that is set, so a box with the flag off |
| +// hears nothing. Never called with a.mu held (emitGen's rule). |
| +func (a *Service) cancelPlanned(rt registry.Route, plan WakePlan, ran bool) { |
| + if a.OnGen == nil || plan.Token == 0 || !plan.planned() { |
| + return |
| + } |
| + ev := GenEvent{Kind: GenCancelled, Route: rt, Mode: plan.Mode, FirstWake: plan.FirstWake, At: time.Now(), Executed: ran} |
| + if ran { |
| + ev.Gen = genOf(plan.Boot) |
| + } |
| + a.emitPlanned(plan, ev) |
| +} |
| + |
| // afterWake reports a planned wake's outcome. wake() calls it after |
| // e.ready has closed, without a.mu (emitGen's rule). |
| +// |
| +// Every planned wake with a token ends with exactly one terminal event for |
| +// it (rollout Task 5): its failure, its fallback, its verdict or its |
| +// cancellation here, or the first-wake observer's. A failed wake that owed |
| +// nothing to the outcome fence (outcomeOwedLocked) reports its cancellation |
| +// (cancelPlanned) after its entry is gone and admission has reopened; that |
| +// is safe because the event records nothing, and a newer wake's intent |
| +// replaces this one's, so its late arrival ends nothing it should not. |
| func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) { |
| plan := e.plan // written only by this wake's own goroutine |
| - if a.OnGen == nil || w == nil || !plan.planned() { |
| + if a.OnGen == nil || !plan.planned() { |
| + return |
| + } |
| + if w == nil { |
| + // No result at all (a worker's token not minted, a claimed wake that |
| + // ran nothing of the planned build): nothing of it ran. |
| + if err != nil { |
| + a.cancelPlanned(e.route, plan, false) |
| + } |
| return |
| } |
| now := time.Now() |
| @@ -661,6 +709,10 @@ func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) { |
| a.emitPlanned(plan, GenEvent{Kind: GenFailed, Route: e.route, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder}) |
| case w.holdRanUnjudged(plan): |
| a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: e.route, Gen: genOf(plan.Boot), Mode: plan.Mode, FirstWake: true, At: now, Executed: true}) |
| + default: |
| + // Refused before the exec (its lease, a fence, its check), or an |
| + // overlap or ordinary wake ended after the exec with no verdict. |
| + a.cancelPlanned(e.route, plan, w.bootRan) |
| } |
| case w.fellBack: |
| a.emitPlanned(plan, GenEvent{Kind: GenFellBack, Route: e.route, Gen: w.planned, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder}) |
diff --git a/internal/host/rollout.go b/internal/host/rollout.go| index 7fd4ccef9..b4d4ccd1e 100644 |
| --- a/internal/host/rollout.go |
| +++ b/internal/host/rollout.go |
| @@ -43,11 +43,13 @@ const ( |
| // refuseTexts is the wake refusal each refuseManaged code carries. It reaches |
| // a member through the activator's "wake failed:" event, so it names the |
| -// missing fact and never the raw error, which goes to the agent's log. |
| +// missing fact and never the raw error, which goes to the agent's log. The |
| +// unreadable book's text is the scheduler's own (rollout Task 5 refuses a |
| +// wake whose intent it cannot write with it), shared rather than copied. |
| var refuseTexts = map[string]string{ |
| rolloutNoStore: "this box cannot start instances right now: the deploy store is unavailable", |
| rolloutNoReleaseKey: "this box cannot start instances right now: the release key is unavailable", |
| - rolloutBookUnreadable: "this box cannot start instances right now: its record of failed builds cannot be read", |
| + rolloutBookUnreadable: rollout.BookUnreadableRefusal, |
| } |
| var rolloutMgr atomic.Pointer[rollout.Manager] |
| @@ -331,6 +333,19 @@ func armRollout(ctx context.Context, getenv func(string) string, svc *activator. |
| refuseManaged(svc, rolloutBookUnreadable) |
| return nil |
| } |
| + if n := book.Reconciled(); n > 0 { |
| + // Executions an earlier agent began and never saw recorded (Task 5, |
| + // spec §8.4): their intents are failure records now, saved before |
| + // anything runs. A box that cannot save them must not run either: |
| + // its next restart would find the intents again, and nothing it |
| + // recorded meanwhile would be kept. |
| + log.Printf("agent: %s=overlap: %d execution(s) an earlier agent left unrecorded are now failure records", rollout.SwapEnv, n) |
| + if err := book.Flush(); err != nil { |
| + log.Printf("agent: %s=overlap: save the failure book after reconciling it: %v", rollout.SwapEnv, err) |
| + refuseManaged(svc, rolloutBookUnreadable) |
| + return nil |
| + } |
| + } |
| platform := runtime.GOOS + "-" + runtime.GOARCH |
| m := rollout.NewManager(ctx, rollout.Deps{ |
| Act: svc, Routes: reg, Book: book, OptDir: release.OptDir(), DataDir: dd, Box: BoxLabel(), |
diff --git a/internal/host/rollout_test.go b/internal/host/rollout_test.go| index ac39aae6d..77041afe8 100644 |
| --- a/internal/host/rollout_test.go |
| +++ b/internal/host/rollout_test.go |
| @@ -293,6 +293,104 @@ func TestUnavailableStateRefusesManagedStarts(t *testing.T) { |
| } |
| } |
| +// leaveIntent leaves in's intent for host in dd's failure book, as an agent |
| +// that died mid-execution would have. |
| +func leaveIntent(t *testing.T, dd, host string, in rollout.Intent) { |
| + t.Helper() |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := b.Begin(host, in); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// Task 5 rule 5: an armed box whose book holds an intent an earlier agent |
| +// left turns it into the failure it may have been, and saves that before |
| +// the scheduler runs: a reopen finds the record and no intent. |
| +func TestAnArmedBoxReconcilesAnIntentLeftInItsBook(t *testing.T) { |
| + svc, reg, dd := rolloutFixture(t, true) |
| + svc.GenerationsFile = release.GenerationsPath(dd) |
| + _, pub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready", |
| + Hibernate: true, Version: "v2", GoodVersion: "v1", GoodDigest: "sha256:v1"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + leaveIntent(t, dd, "a.example.com", rollout.Intent{Account: "bab", App: "blog", Build: "v2", Mode: "hold", ID: "a3"}) |
| + store := release.NewDirStore(t.TempDir()) |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + t.Cleanup(cancel) |
| + m := armRollout(ctx, overlapEnv(pub), svc, reg, dd, nil, func(context.Context) (release.Store, error) { return store, nil }) |
| + if m == nil || rolloutUnavailable() != "" { |
| + t.Fatalf("not armed (%q)", rolloutUnavailable()) |
| + } |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b.Reconciled() != 0 { |
| + t.Fatal("the reconciled book was not saved: the intent is still on disk") |
| + } |
| + if !b.HeldDown("bab", "blog", "v2", "a.example.com") { |
| + t.Fatal("the hold intent's failure was not kept") |
| + } |
| + if p := svc.Planner(ctx, *mustGet(t, reg, "a.example.com")); p.Refuse != rollout.HeldDownRefusal("v2") { |
| + t.Fatalf("plan %+v: want the held-down refusal", p) |
| + } |
| +} |
| + |
| +// Task 5 rule 5: a reconciled book the box cannot save refuses every |
| +// managed start, as an unreadable one does: the scheduler must not run while |
| +// a restart would find the intents again with no way to keep their records. |
| +func TestAnArmedBoxThatCannotSaveItsReconciledBookRefusesManagedStarts(t *testing.T) { |
| + if os.Geteuid() == 0 { |
| + t.Skip("running as root: a read-only directory does not fail a save") |
| + } |
| + svc, reg, dd := rolloutFixture(t, true) |
| + _, pub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + leaveIntent(t, dd, "a.example.com", rollout.Intent{Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: "w4"}) |
| + if err := os.Chmod(dd, 0o500); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { os.Chmod(dd, 0o700) }) |
| + m := armRollout(context.Background(), overlapEnv(pub), svc, reg, dd, nil, |
| + func(context.Context) (release.Store, error) { return release.NewDirStore(t.TempDir()), nil }) |
| + if m != nil || armedRollout() != nil { |
| + t.Fatal("armed over a book it could not save") |
| + } |
| + if rolloutUnavailable() != rolloutBookUnreadable { |
| + t.Fatalf("unavailable %q, want %q", rolloutUnavailable(), rolloutBookUnreadable) |
| + } |
| + managed := registry.Route{Host: "a.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready", Hibernate: true, Version: "v2"} |
| + if p := svc.Planner(context.Background(), managed); p.Refuse != refuseTexts[rolloutBookUnreadable] { |
| + t.Fatalf("plan %+v: want the unreadable-book refusal", p) |
| + } |
| +} |
| + |
| +// The unreadable-book refusal is one approved text, shared with the |
| +// scheduler's refusal of a wake whose intent it could not write (Task 5). |
| +func TestTheUnreadableBookRefusalIsShared(t *testing.T) { |
| + if refuseTexts[rolloutBookUnreadable] != rollout.BookUnreadableRefusal { |
| + t.Fatalf("host %q, rollout %q", refuseTexts[rolloutBookUnreadable], rollout.BookUnreadableRefusal) |
| + } |
| +} |
| + |
| +func mustGet(t *testing.T, reg *registry.Registry, host string) *registry.Route { |
| + t.Helper() |
| + rt, err := reg.Get(host) |
| + if err != nil || rt == nil { |
| + t.Fatalf("get %s: %v", host, err) |
| + } |
| + return rt |
| +} |
| + |
| // PF15: with the flag on but no delegated cgroup subtree, the flag is |
| // refused before the activator exists. Generations and containment stay |
| // off, no manager is armed, no hook is set, every leg is today's, and the |
diff --git a/internal/host/rolloutdeclined.go b/internal/host/rolloutdeclined.go| index af7b06f68..2390885c6 100644 |
| --- a/internal/host/rolloutdeclined.go |
| +++ b/internal/host/rolloutdeclined.go |
| @@ -35,9 +35,10 @@ const refuseNoSwaps = "this box cannot run two builds side by side, which this i |
| // restart leave such an instance running rather than stop it into a refused |
| // wake. |
| // |
| -// The book is read once, at arming, and never written: nothing on a declined |
| -// box records, retries or prunes, so what was read stays the truth until the |
| -// agent restarts. |
| +// The book is read once, at arming, and written at most once then: when an |
| +// armed run left intents in it, the records reconciled from them are saved |
| +// (honourDeclinedFailures). Nothing on a declined box records, retries or |
| +// prunes, so what was read stays the truth until the agent restarts. |
| type failureHolds struct { |
| book *rollout.FailureBook |
| // get reads a route by host, for the legs that are handed only a host. |
| @@ -72,6 +73,16 @@ func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, |
| refuseManaged(svc, rolloutBookUnreadable) |
| return |
| } |
| + if n := book.Reconciled(); n > 0 { |
| + // An armed run's executions left unrecorded (Task 5): their records |
| + // are honoured below whether or not this save lands, since the book |
| + // in memory holds them either way, and the next restart reconciles |
| + // the intents again if it did not. |
| + log.Printf("agent: %s=overlap declined: %d execution(s) an armed run left unrecorded are now failure records", rollout.SwapEnv, n) |
| + if err := book.Flush(); err != nil { |
| + log.Printf("agent: %s=overlap declined: save the failure book after reconciling it: %v", rollout.SwapEnv, err) |
| + } |
| + } |
| recs := book.List() |
| if len(recs) == 0 { |
| return |
diff --git a/internal/host/rolloutdeclined_test.go b/internal/host/rolloutdeclined_test.go| index 5cda71769..1fdcaeb08 100644 |
| --- a/internal/host/rolloutdeclined_test.go |
| +++ b/internal/host/rolloutdeclined_test.go |
| @@ -392,6 +392,44 @@ func TestDeclinedBoxKeepsAHeldBuildOffARealActivator(t *testing.T) { |
| } |
| } |
| +// Task 5 rule 5: a declined box honours the records reconciled from an |
| +// intent an armed run left, and saves them; a save that fails is logged and |
| +// the records are honoured all the same. |
| +func TestDeclinedBoxHonoursAReconciledIntent(t *testing.T) { |
| + for _, canSave := range []bool{true, false} { |
| + t.Run(map[bool]string{true: "saved", false: "not saved"}[canSave], func(t *testing.T) { |
| + if !canSave && os.Geteuid() == 0 { |
| + t.Skip("running as root: a read-only directory does not fail a save") |
| + } |
| + t.Setenv("CARLOS_OPT_DIR", t.TempDir()) |
| + svc, reg, dd := rolloutFixture(t, false) |
| + acct := release.RouteAccount(managedRoute("x", "")) |
| + leaveIntent(t, dd, "down.example.com", rollout.Intent{Account: acct, App: "blog", Build: "v2", Mode: "hold", ID: "a9"}) |
| + if !canSave { |
| + if err := os.Chmod(dd, 0o500); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { os.Chmod(dd, 0o700) }) |
| + } |
| + declineWithBook(t, svc, reg, dd) |
| + if heldHolds() == nil || svc.Planner == nil || rolloutUnavailable() != "" { |
| + t.Fatalf("the reconciled record was not honoured (unavailable %q)", rolloutUnavailable()) |
| + } |
| + if p := svc.Planner(context.Background(), managedRoute("down.example.com", "v2")); p.Refuse == "" { |
| + t.Fatalf("plan %+v: want the held-down instance refused", p) |
| + } |
| + os.Chmod(dd, 0o700) |
| + b, err := rollout.OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if saved := b.Reconciled() == 0; saved != canSave { |
| + t.Fatalf("book saved %v, want %v", saved, canSave) |
| + } |
| + }) |
| + } |
| +} |
| + |
| // Final review F6 (d): a declined box whose failure book cannot be read |
| // fails closed, as refuseManaged does on an armed box: every managed wake is |
| // refused with the fixed text, and the bounce legs stand down. The status |
diff --git a/internal/rollout/bookseq_test.go b/internal/rollout/bookseq_test.go| index 3451d82c5..aa44f86f1 100644 |
| --- a/internal/rollout/bookseq_test.go |
| +++ b/internal/rollout/bookseq_test.go |
| @@ -291,6 +291,13 @@ func TestASettleDecidedBeforeAHoldRanKeepsItsMarker(t *testing.T) { |
| } |
| g, _ := h.act.Current("h") |
| h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("h"), Gen: g, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| + // Since Task 5 the wake's intent, which stands until its settle, refuses |
| + // an adoption of h (ErrIntentStands), so this ordering needs the intent |
| + // gone. It is removed by hand: the marker rule must hold whatever lets a |
| + // hold run between a plan and its settle. |
| + if err := h.book.End("h", wakeExecID(p.Token)); err != nil { |
| + t.Fatal(err) |
| + } |
| gate := h.act.hold("h") |
| h.install("v2", "h") |
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go| index 4aef326ae..32cdf2450 100644 |
| --- a/internal/rollout/failures.go |
| +++ b/internal/rollout/failures.go |
| @@ -7,9 +7,11 @@ import ( |
| "os" |
| "path/filepath" |
| "slices" |
| + "strings" |
| "sync" |
| "time" |
| + "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -142,6 +144,53 @@ func (h HoldRan) same(o HoldRan) bool { |
| return h.Account == o.Account && h.App == o.App && h.Build == o.Build |
| } |
| +// Intent is an execution that may leave a failure behind, written to the |
| +// book before the execution runs anything (Astra R1 #5, R3 §5 item 2): an |
| +// adoption attempt, or a planned first wake. Its outcome's own book write |
| +// (Record, MarkHoldRan) comes later, and End removes the intent only after |
| +// it. A failure held only in memory because its save failed, followed by an |
| +// agent restart, therefore still finds the intent on disk, and |
| +// OpenFailureBook turns it into the failure record it may have been. |
| +type Intent struct { |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + Build string `json:"build"` |
| + // Mode is the mode the execution runs in: the effective one, after the |
| + // activator's own overrides (an overlap attempt on a foreign-layout row |
| + // runs as a hold, plan review finding 2), so a reconciled hold intent |
| + // holds the instance down. |
| + Mode string `json:"mode"` |
| + // ID is the execution's identity (HoldRan.Exec): "a<n>" for an attempt |
| + // (dispatch.ExecID), "w<token>" for a planned wake (wakeExecID). A |
| + // reconciled hold intent's marker carries it. |
| + ID string `json:"id"` |
| + // Start is when the execution started (dispatch.Started, a wake's plan |
| + // time): a reconciled hold intent's marker carries it as its At (ruling |
| + // 205). Zero when not known; reconciliation then uses the open time. |
| + Start time.Time `json:"start"` |
| +} |
| + |
| +// wakeIntent reports whether id is a planned wake's intent ("w<token>"). |
| +func wakeIntent(id string) bool { return strings.HasPrefix(id, "w") } |
| + |
| +// ErrIntentStands is Begin's refusal while another execution's intent stands |
| +// for the host: an attempt's, or a wake's that an attempt would replace. Its |
| +// outcome is not yet written, so replacing its intent would lose it across a |
| +// failed save and a restart. |
| +var ErrIntentStands = errors.New("another execution on this instance has not recorded its outcome yet") |
| + |
| +// reconciledReason is a reconciled intent's Reason. reasonWord reads it as |
| +// a failed health check (status.ReasonHealth): the execution's verdict was |
| +// never recorded, which is all the box knows. |
| +const reconciledReason = "the agent restarted before this execution recorded its outcome" |
| + |
| +// failureSchema is the schema the book is written as. Schema 2 added the |
| +// intents (Task 5). It is written as 2 whatever the file was, so an older |
| +// agent, which accepts only schema 1, refuses the file rather than read it |
| +// and drop its markers and intents (an agent downgrade with the flag on |
| +// refuses managed starts until the file is upgraded or removed). |
| +const failureSchema = 2 |
| + |
| type failureFile struct { |
| Schema int `json:"schema"` |
| Records []FailureRecord `json:"records"` |
| @@ -152,6 +201,8 @@ type failureFile struct { |
| // Seq is the book's sequence (FailureBook.seq), additive like HoldRan, |
| // so a snapshot after a restart is newer than every marker before it. |
| Seq uint64 `json:"seq,omitempty"` |
| + // Intents are the executions in flight by host (Intent), schema 2. |
| + Intents map[string]Intent `json:"intents,omitempty"` |
| } |
| // FailureBook is the box's failure records. An empty path keeps them in |
| @@ -166,9 +217,17 @@ type FailureBook struct { |
| recs []FailureRecord |
| // held is the hold-ran markers by host (HoldRan), saved with the records. |
| held map[string]HoldRan |
| + // intents are the executions in flight by host (Intent), saved with the |
| + // records. reconciled is how many intents OpenFailureBook turned into |
| + // records. |
| + intents map[string]Intent |
| + reconciled int |
| // dirty is set when a save failed. Memory is the truth meanwhile, and |
| - // every later mutating call and Prune retries the save until it lands, |
| - // so the record still survives an agent restart (spec §8.4). |
| + // every later mutating call and Prune retries the save until it lands. |
| + // What keeps a failure across an agent restart before that is the |
| + // execution's intent (Begin), saved before the execution ran anything and |
| + // removed only after its outcome was written (End): a restart finds it |
| + // on disk and records the failure from it (spec §8.4, Astra R1 #5). |
| dirty bool |
| // now stamps a marker whose caller gave no At (markLocked). Nil means |
| // time.Now; tests and the simulation set their own clock. |
| @@ -207,7 +266,9 @@ func cloneRec(r FailureRecord) FailureRecord { |
| // OpenFailureBook loads the book from dataDir. A file that exists but cannot |
| // be read is an error, never an empty book: an empty book would move every |
| -// held instance straight back onto the build that failed. |
| +// held instance straight back onto the build that failed. Intents left in |
| +// the file become failure records (reconcileLocked, Reconciled), which the |
| +// caller saves with Flush. |
| func OpenFailureBook(dataDir string) (*FailureBook, error) { |
| b := &FailureBook{path: filepath.Join(dataDir, FailureFile)} |
| raw, err := os.ReadFile(b.path) |
| @@ -221,28 +282,148 @@ func OpenFailureBook(dataDir string) (*FailureBook, error) { |
| if err := json.Unmarshal(raw, &f); err != nil { |
| return nil, fmt.Errorf("%s: %w", b.path, err) |
| } |
| - if f.Schema != 1 { |
| + // Schema 1 is an agent before Task 5 (no intents); 2 is written now. |
| + if f.Schema != 1 && f.Schema != failureSchema { |
| return nil, fmt.Errorf("%s: unknown schema %d", b.path, f.Schema) |
| } |
| b.recs = f.Records |
| b.held = f.HoldRan |
| b.seq = f.Seq |
| + b.intents = f.Intents |
| for _, h := range b.held { |
| // A marker's sequence is saved with the book's, so this changes |
| // nothing in a file this book wrote; it keeps a hand-edited one from |
| // handing out a snapshot a standing marker is newer than. |
| b.seq = max(b.seq, h.Seq) |
| } |
| + b.mu.Lock() |
| + b.reconcileLocked() |
| + b.mu.Unlock() |
| return b, nil |
| } |
| +// reconcileLocked turns each intent read from the file into the failure |
| +// record its execution may have left (rule 3; spec §8.4, Astra R1 #5): its |
| +// build on its host, cause release, the intent's mode, stamped with the open |
| +// time, so only a pointer signed after the restart clears it. A hold intent |
| +// holds the instance down and sets its hold-ran marker too, with the |
| +// intent's identity and start (Task 4), as a hold that went down would have: |
| +// the build may have migrated the database, so nothing falls back there |
| +// automatically. An agent crash mid-execution therefore halts that build's |
| +// cohort until a retry, even when the execution would have passed (the |
| +// plan's ruling). The intents then go from memory and the book is owed a |
| +// save, which the caller makes (Flush); nothing is written here, so a |
| +// read-only open of a book never changes the file. The caller holds b.mu. |
| +func (b *FailureBook) reconcileLocked() { |
| + if len(b.intents) == 0 { |
| + b.intents = nil |
| + return |
| + } |
| + now := b.clock() |
| + hosts := make([]string, 0, len(b.intents)) |
| + for host := range b.intents { |
| + hosts = append(hosts, host) |
| + } |
| + slices.Sort(hosts) // one order for every open of the same file |
| + for _, host := range hosts { |
| + in := b.intents[host] |
| + r := FailureRecord{Account: in.Account, App: in.App, Build: in.Build, Host: host, |
| + Cause: string(activator.CauseRelease), Mode: in.Mode, Reason: reconciledReason, PromotedAt: now, At: now} |
| + if in.Mode == string(activator.ModeHold) { |
| + // A zero start leaves the marker's At to markLocked, which stamps |
| + // it with the book's clock: the open time. |
| + r.HeldDown, r.MarkHoldRan, r.HoldRanExec, r.HoldRanAt = []string{host}, true, in.ID, in.Start |
| + } |
| + b.recordLocked(r) |
| + } |
| + b.reconciled = len(hosts) |
| + b.intents = nil |
| + b.dirty = true |
| +} |
| + |
| +// Reconciled is how many intents OpenFailureBook turned into failure |
| +// records: executions an earlier agent began and never saw recorded. The |
| +// caller logs them and flushes the book (Flush). |
| +func (b *FailureBook) Reconciled() int { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + return b.reconciled |
| +} |
| + |
| +// Flush saves the book now: after reconciliation, so the records it made |
| +// and the intents it removed reach the disk before anything runs. |
| +func (b *FailureBook) Flush() error { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + return b.saveLocked() |
| +} |
| + |
| +// Begin writes host's intent and saves it before the execution runs |
| +// anything (rule 1). On a save error nothing of it stays in memory, the host's |
| +// earlier intent (if this one replaced it) is put back, and the error is |
| +// returned: the caller must not execute, since its failure could then be |
| +// lost across a failed save and a restart. |
| +// |
| +// One intent stands per host. A planned wake's intent replaces an older |
| +// wake's intent for the host: the activator delivers a planned wake's |
| +// outcome before a newer wake of the host plans (its outcome fence, |
| +// internal/activator awaitOutcomes, final review F3 and wave 2 N1), so the |
| +// older wake's intent is one whose outcome is written, or one whose build |
| +// never ran (GenCancelled, not executed), which an event on its way ends |
| +// anyway. Two plans for one host never race: the activator plans a host |
| +// only from its one waking entry. Any other standing intent refuses the |
| +// Begin (ErrIntentStands): an attempt's, whose outcome done() has yet to |
| +// write, or, for an attempt, a wake's, whose outcome may still be on its way |
| +// (a fallen-back wake's entry serves before its GenFellBack is delivered). |
| +// The refused execution is retried as any refused one is. |
| +func (b *FailureBook) Begin(host string, in Intent) error { |
| + if in.ID == "" { |
| + return errors.New("an intent needs its execution's identity") |
| + } |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + prev, had := b.intents[host] |
| + if had && prev.ID != in.ID && !(wakeIntent(prev.ID) && wakeIntent(in.ID)) { |
| + return fmt.Errorf("%s: %w (%s)", host, ErrIntentStands, prev.ID) |
| + } |
| + if b.intents == nil { |
| + b.intents = map[string]Intent{} |
| + } |
| + b.intents[host] = in |
| + if err := b.saveLocked(); err != nil { |
| + if had { |
| + b.intents[host] = prev |
| + } else { |
| + delete(b.intents, host) |
| + } |
| + return err |
| + } |
| + return nil |
| +} |
| + |
| +// End removes host's intent when it is id's, and saves (rule 2). It runs |
| +// after the execution's outcome is written to the book (Record, |
| +// MarkHoldRan), never before. Its save failing leaves the intent on disk, |
| +// which is the safe direction: a restart then records the failure the |
| +// outcome may also have written. An intent of another execution (a newer |
| +// wake that replaced this one's) is left alone. |
| +func (b *FailureBook) End(host, id string) error { |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + if cur, ok := b.intents[host]; !ok || id == "" || cur.ID != id { |
| + return b.retryLocked() |
| + } |
| + delete(b.intents, host) |
| + return b.saveLocked() |
| +} |
| + |
| // saveLocked writes the book and records whether the file is now behind |
| // memory, so a failure is retried by retryLocked rather than forgotten. |
| func (b *FailureBook) saveLocked() error { |
| if b.path == "" { |
| return nil |
| } |
| - raw, err := json.MarshalIndent(failureFile{Schema: 1, Records: b.recs, HoldRan: b.held, Seq: b.seq}, "", " ") |
| + raw, err := json.MarshalIndent(failureFile{Schema: failureSchema, Records: b.recs, HoldRan: b.held, Seq: b.seq, Intents: b.intents}, "", " ") |
| if err == nil { |
| err = release.WriteFileAtomic(b.path, raw, 0o644) |
| } |
| @@ -315,8 +496,11 @@ func (b *FailureBook) clearedLocked(acct, app, build, host string) { |
| // after the latest failure clears it. The in-memory record is kept even when |
| // the file write fails, so the caller's transition still blocks the build; |
| // the error is for the journal, and the next call on the book (this one |
| -// again, if nothing new, or Prune) retries the save. r.MarkHoldRan sets r's |
| -// host's hold-ran marker in the same save. |
| +// again, if nothing new, or Prune) retries the save. Until a save lands, what |
| +// keeps the failure across an agent restart is the execution's intent |
| +// (Begin), still on disk because its End comes after this call: the restart |
| +// records the failure from it. r.MarkHoldRan sets r's host's hold-ran marker |
| +// in the same save. |
| // |
| // A failure of an execution that began before a retry of its build (r.Seq, |
| // ruling 191) is stale: the record is neither created nor changed, the |
| @@ -331,21 +515,30 @@ func (b *FailureBook) clearedLocked(acct, app, build, host string) { |
| func (b *FailureBook) Record(r FailureRecord) (RecordResult, error) { |
| b.mu.Lock() |
| defer b.mu.Unlock() |
| + res, changed := b.recordLocked(r) |
| + if changed { |
| + return res, b.saveLocked() |
| + } |
| + return res, b.retryLocked() |
| +} |
| + |
| +// recordLocked is Record's change to memory, and reports whether it changed |
| +// anything to save. Reconciliation (reconcileLocked) shares it, so a |
| +// reconciled intent merges into a standing record as a live failure does. |
| +// The caller holds b.mu and saves. |
| +func (b *FailureBook) recordLocked(r FailureRecord) (RecordResult, bool) { |
| stale := b.preRetryLocked(r) |
| mark := r.MarkHoldRan || (stale && slices.Contains(r.HeldDown, r.Host)) |
| marked := mark && b.markLocked(r.Host, HoldRan{Account: r.Account, App: r.App, Build: r.Build, Exec: r.HoldRanExec, At: r.HoldRanAt}) |
| r.MarkHoldRan, r.HoldRanAt, r.HoldRanExec, r.Seq = false, time.Time{}, "", 0 |
| if stale { |
| - if marked { |
| - return PreRetry, b.saveLocked() |
| - } |
| - return PreRetry, b.retryLocked() |
| + return PreRetry, marked |
| } |
| i := b.find(r.Account, r.App, r.Build) |
| if i < 0 { |
| r.Retried = nil |
| b.recs = append(b.recs, cloneRec(r)) |
| - return Recorded, b.saveLocked() |
| + return Recorded, true |
| } |
| cur := &b.recs[i] |
| changed := false |
| @@ -363,13 +556,10 @@ func (b *FailureBook) Record(r FailureRecord) (RecordResult, error) { |
| changed = true |
| } |
| } |
| - switch { |
| - case changed: |
| - return Merged, b.saveLocked() |
| - case marked: |
| - return Unchanged, b.saveLocked() |
| + if changed { |
| + return Merged, true |
| } |
| - return Unchanged, b.retryLocked() |
| + return Unchanged, marked |
| } |
| // markLocked sets host's hold-ran marker to h and reports whether that |
diff --git a/internal/rollout/fake_test.go b/internal/rollout/fake_test.go| index 2e5918e17..2a4173ab5 100644 |
| --- a/internal/rollout/fake_test.go |
| +++ b/internal/rollout/fake_test.go |
| @@ -66,6 +66,17 @@ func (f *fakeAct) PublishHeldServing(host string) { |
| } |
| } |
| +// EffectiveMode is the mode Swap will run m in: mode when a test set it (the |
| +// foreign-layout override it models), m otherwise. |
| +func (f *fakeAct) EffectiveMode(_ string, m activator.SwapMode) activator.SwapMode { |
| + f.mu.Lock() |
| + defer f.mu.Unlock() |
| + if f.mode != "" { |
| + return f.mode |
| + } |
| + return m |
| +} |
| + |
| func newFakeAct() *fakeAct { |
| return &fakeAct{gens: map[string]activator.Gen{}, owned: map[string]bool{}, gates: map[string]chan struct{}{}} |
| } |
diff --git a/internal/rollout/holdran_test.go b/internal/rollout/holdran_test.go| index fba7cfd68..6d2fd08ac 100644 |
| --- a/internal/rollout/holdran_test.go |
| +++ b/internal/rollout/holdran_test.go |
| @@ -451,8 +451,11 @@ func TestTheHoldRanMarkerSurvivesReopen(t *testing.T) { |
| // Final review wave 2, N2: the markers are additive to schema 1. A file an |
| // older agent wrote (no hold_ran key) opens with no markers. A file with |
| -// markers still parses as an older agent reads it: plain encoding/json into |
| -// the old shape, which skips the unknown key, and every record intact. |
| +// markers still parses into the old shape (plain encoding/json skips the |
| +// unknown key) with every record intact. Since Task 5 it is written as |
| +// schema 2, which an older agent's reader refuses on purpose rather than |
| +// read it and drop the markers and intents |
| +// (TestASchema2BookIsUnreadableToASchema1Reader). |
| func TestTheHoldRanMarkerIsSchemaAdditive(t *testing.T) { |
| dir := t.TempDir() |
| old := `{"schema": 1, "records": [{"account": "bab", "app": "blog", "build": "v2", "host": "a", "cause": "release", |
| @@ -485,8 +488,8 @@ func TestTheHoldRanMarkerIsSchemaAdditive(t *testing.T) { |
| if err := json.Unmarshal(raw, &older); err != nil { |
| t.Fatalf("an older agent cannot parse the file with markers: %v", err) |
| } |
| - if older.Schema != 1 || len(older.Records) != 1 || older.Records[0].Build != "v2" { |
| - t.Fatalf("older agent's view = %+v: want schema 1 and the one record", older) |
| + if older.Schema != 2 || len(older.Records) != 1 || older.Records[0].Build != "v2" { |
| + t.Fatalf("older agent's view = %+v: want schema 2 and the one record", older) |
| } |
| if !strings.Contains(string(raw), `"hold_ran"`) { |
| t.Fatalf("the marker is not in the file:\n%s", raw) |
diff --git a/internal/rollout/intent_test.go b/internal/rollout/intent_test.go| new file mode 100644 |
| index 000000000..45b0d3927 |
| --- /dev/null |
| +++ b/internal/rollout/intent_test.go |
| @@ -0,0 +1,506 @@ |
| +package rollout |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/status" |
| +) |
| + |
| +// Task 5 (Astra F5; R1 #5, R3 §5 item 2): failure state survives a failed |
| +// book write followed by an agent restart. Every execution that may leave a |
| +// failure writes an intent first; a restart turns an intent left on disk |
| +// into the failure it may have been. |
| +// |
| +// A save fails by making the data dir read-only (WriteFileAtomic writes its |
| +// temp file beside the target). Running as root defeats it: the tests skip. |
| + |
| +func readOnly(t *testing.T, dir string) { |
| + t.Helper() |
| + if os.Geteuid() == 0 { |
| + t.Skip("running as root: a read-only directory does not fail a save") |
| + } |
| + if err := os.Chmod(dir, 0o500); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { os.Chmod(dir, 0o700) }) |
| +} |
| + |
| +func writable(t *testing.T, dir string) { |
| + t.Helper() |
| + if err := os.Chmod(dir, 0o700); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// onDisk gives the harness a book saved under a data dir of its own, before |
| +// anything has run, and returns the dir. |
| +func (h *harness) onDisk() string { |
| + h.t.Helper() |
| + dir := h.t.TempDir() |
| + b, err := OpenFailureBook(dir) |
| + if err != nil { |
| + h.t.Fatal(err) |
| + } |
| + b.now = h.clock |
| + h.book, h.m.d.Book = b, b |
| + return dir |
| +} |
| + |
| +// fileOf reads the book file in dir as it stands on disk. |
| +func fileOf(t *testing.T, dir string) failureFile { |
| + t.Helper() |
| + raw, err := os.ReadFile(filepath.Join(dir, FailureFile)) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return failureFile{} |
| + } |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var f failureFile |
| + if err := json.Unmarshal(raw, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + return f |
| +} |
| + |
| +// Review Focus 1: B's failure is held only in memory because the save |
| +// failed; the agent restarts. The intent written before B ran is still on |
| +// disk and comes back as B's failure. |
| +func TestAFailedSaveThenRestartKeepsTheFailure(t *testing.T) { |
| + dd := t.TempDir() |
| + b, _ := OpenFailureBook(dd) |
| + start := t0.Add(-time.Minute) |
| + if err := b.Begin("h", Intent{Account: "bab", App: "blog", Build: "v2", Mode: "hold", ID: "a7", Start: start}); err != nil { |
| + t.Fatal(err) |
| + } |
| + readOnly(t, dd) |
| + if _, err := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: "hold", HeldDown: []string{"h"}}); err == nil { |
| + t.Skip("the save did not fail") |
| + } |
| + _ = b.End("h", "a7") // fails too: the intent stays on disk |
| + writable(t, dd) |
| + b2, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b2.Reconciled() != 1 || !b2.HeldDown("bab", "blog", "v2", "h") { |
| + t.Fatalf("after restart: reconciled %d, held down %v", b2.Reconciled(), b2.HeldDown("bab", "blog", "v2", "h")) |
| + } |
| + m, ok := b2.HoldRanOn("h") |
| + if !ok || m.Build != "v2" { |
| + t.Fatal("a hold intent must leave its marker") |
| + } |
| + if m.Exec != "a7" || !m.At.Equal(start) { |
| + t.Fatalf("marker %+v: want the intent's execution (a7) and its start %v", m, start) |
| + } |
| +} |
| + |
| +func TestABeginThatCannotSaveRefusesAndLeavesNothing(t *testing.T) { |
| + dd := t.TempDir() |
| + b, _ := OpenFailureBook(dd) |
| + readOnly(t, dd) |
| + if err := b.Begin("h", Intent{Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: "a1"}); err == nil { |
| + t.Skip("the save did not fail") |
| + } |
| + if _, ok := b.intents["h"]; ok { |
| + t.Fatal("a refused begin kept its intent in memory") |
| + } |
| + writable(t, dd) |
| + b2, _ := OpenFailureBook(dd) |
| + if b2.Reconciled() != 0 { |
| + t.Fatal("a refused begin left an intent") |
| + } |
| + // The memory a later save writes has no intent either. |
| + if err := b.Flush(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if b3, _ := OpenFailureBook(dd); b3.Reconciled() != 0 { |
| + t.Fatal("a refused begin's intent reached the file at the next save") |
| + } |
| +} |
| + |
| +func TestASchema1BookReadsAndIsWrittenAsSchema2(t *testing.T) { |
| + dd := t.TempDir() |
| + raw := `{"schema":1,"records":[{"account":"bab","app":"blog","build":"v2","host":"h","cause":"release","mode":"overlap","reason":"x","promoted_at":"2026-10-01T00:00:00Z","at":"2026-10-01T00:00:00Z"}]}` |
| + if err := os.WriteFile(filepath.Join(dd, FailureFile), []byte(raw), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + b, err := OpenFailureBook(dd) |
| + if err != nil || !b.Blocks("bab", "blog", "v2", "x", false) { |
| + t.Fatalf("schema 1: %v", err) |
| + } |
| + if err := b.Flush(); err != nil { |
| + t.Fatal(err) |
| + } |
| + out, _ := os.ReadFile(filepath.Join(dd, FailureFile)) |
| + if !strings.Contains(string(out), `"schema": 2`) { |
| + t.Fatalf("written as %s", out) |
| + } |
| + if _, err := OpenFailureBook(dd); err != nil { |
| + t.Fatalf("schema 2 written, then refused: %v", err) |
| + } |
| +} |
| + |
| +// Review Focus 2: what an older agent sees. Its reader refused anything |
| +// but schema 1 (OpenFailureBook before Task 5). |
| +func TestASchema2BookIsUnreadableToASchema1Reader(t *testing.T) { |
| + var f struct { |
| + Schema int `json:"schema"` |
| + } |
| + dd := t.TempDir() |
| + b, _ := OpenFailureBook(dd) |
| + if err := b.Flush(); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _ := os.ReadFile(filepath.Join(dd, FailureFile)) |
| + if err := json.Unmarshal(raw, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f.Schema == 1 { |
| + t.Fatal("an old reader would accept the file and drop markers and intents") |
| + } |
| +} |
| + |
| +func TestABookOfAnUnknownSchemaIsRefused(t *testing.T) { |
| + for _, schema := range []string{"0", "3"} { |
| + dd := t.TempDir() |
| + if err := os.WriteFile(filepath.Join(dd, FailureFile), []byte(`{"schema":`+schema+`,"records":[]}`), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := OpenFailureBook(dd); err == nil { |
| + t.Fatalf("schema %s was read", schema) |
| + } |
| + } |
| +} |
| + |
| +// Rule 3: each intent left in the file becomes a failure record for its |
| +// build on that host: cause release, the intent's mode, held down with a |
| +// marker in hold mode, stamped with the open time, and a reason the member |
| +// surfaces word as a failed health check. The intents go, and the book is |
| +// owed a save (Flush), which leaves none on disk. |
| +func TestAnIntentLeftOnDiskBecomesItsFailure(t *testing.T) { |
| + dd := t.TempDir() |
| + b, _ := OpenFailureBook(dd) |
| + start := t0.Add(-time.Hour) |
| + for host, in := range map[string]Intent{ |
| + "a": {Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: "a3", Start: start}, |
| + "b": {Account: "bab", App: "blog", Build: "v2", Mode: "hold", ID: "w9", Start: start}, |
| + "c": {Account: "bab", App: "blog", Build: "v3", Mode: "hold", ID: "a4"}, |
| + } { |
| + if err := b.Begin(host, in); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + before := time.Now() |
| + b2, err := OpenFailureBook(dd) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + after := time.Now() |
| + if b2.Reconciled() != 3 { |
| + t.Fatalf("reconciled %d, want 3", b2.Reconciled()) |
| + } |
| + r, ok := b2.Lookup("bab", "blog", "v2") |
| + if !ok || r.Cause != "release" || len(r.HeldDown) != 1 || r.HeldDown[0] != "b" { |
| + t.Fatalf("v2's record %+v: want a release failure holding down b (the hold intent) alone", r) |
| + } |
| + if r.At.Before(before) || r.At.After(after) || !r.PromotedAt.Equal(r.At) { |
| + t.Fatalf("v2's record stamped at %v, promoted at %v: want the open time for both", r.At, r.PromotedAt) |
| + } |
| + if reasonWord(r.Reason) != status.ReasonHealth { |
| + t.Fatalf("reason %q words as %q", r.Reason, reasonWord(r.Reason)) |
| + } |
| + if !b2.Blocks("bab", "blog", "v2", "a", false) || b2.HeldDown("bab", "blog", "v2", "a") { |
| + t.Fatal("the overlap intent's host must be blocked, not held down") |
| + } |
| + if _, ok := b2.HoldRanOn("a"); ok { |
| + t.Fatal("an overlap intent left a hold-ran marker") |
| + } |
| + if m, ok := b2.HoldRanOn("b"); !ok || m.Build != "v2" || m.Exec != "w9" || !m.At.Equal(start) { |
| + t.Fatalf("b's marker %+v, %v: want v2's, from w9, at its start", m, ok) |
| + } |
| + rc, ok := b2.Lookup("bab", "blog", "v3") |
| + if !ok || rc.Mode != "hold" || !b2.HeldDown("bab", "blog", "v3", "c") { |
| + t.Fatalf("v3's record %+v: want c held down", rc) |
| + } |
| + if m, ok := b2.HoldRanOn("c"); !ok || m.Exec != "a4" || m.At.Before(before) || m.At.After(after) { |
| + t.Fatalf("c's marker %+v: an intent with no start is stamped with the open time", m) |
| + } |
| + if len(b2.intents) != 0 { |
| + t.Fatalf("intents %v stayed in memory", b2.intents) |
| + } |
| + // Nothing is written at open: the caller flushes. |
| + if f := fileOf(t, dd); len(f.Intents) != 3 { |
| + t.Fatalf("the open rewrote the file: %d intents on disk", len(f.Intents)) |
| + } |
| + if err := b2.Flush(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f := fileOf(t, dd); len(f.Intents) != 0 || len(f.Records) != 2 || f.Schema != 2 { |
| + t.Fatalf("after the flush: %+v", f) |
| + } |
| + if b3, _ := OpenFailureBook(dd); b3.Reconciled() != 0 || !b3.HeldDown("bab", "blog", "v2", "b") { |
| + t.Fatal("the flushed book did not keep the reconciled records") |
| + } |
| +} |
| + |
| +// Rule 1 and 2: a newer wake's intent replaces an older wake's for the host; |
| +// an attempt's intent is replaced by nothing, and an attempt never replaces |
| +// a wake's; End removes only its own intent. |
| +func TestBeginAndEndKeepOneIntentPerHost(t *testing.T) { |
| + dd := t.TempDir() |
| + b, _ := OpenFailureBook(dd) |
| + in := func(id string) Intent { |
| + return Intent{Account: "bab", App: "blog", Build: "v2", Mode: "overlap", ID: id} |
| + } |
| + if err := b.Begin("h", in("w1")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := b.Begin("h", in("w2")); err != nil { |
| + t.Fatalf("a newer wake's intent was refused: %v", err) |
| + } |
| + if err := b.Begin("h", in("a5")); !errors.Is(err, ErrIntentStands) { |
| + t.Fatalf("an attempt replaced a wake's intent: %v", err) |
| + } |
| + if f := fileOf(t, dd); f.Intents["h"].ID != "w2" { |
| + t.Fatalf("on disk %+v, want w2's intent", f.Intents) |
| + } |
| + if err := b.End("h", "w1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f := fileOf(t, dd); f.Intents["h"].ID != "w2" { |
| + t.Fatal("an older wake's End removed the newer wake's intent") |
| + } |
| + if err := b.End("h", "w2"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if f := fileOf(t, dd); len(f.Intents) != 0 { |
| + t.Fatalf("End left %+v on disk", f.Intents) |
| + } |
| + if err := b.Begin("h", in("a6")); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, id := range []string{"w3", "a7"} { |
| + if err := b.Begin("h", in(id)); !errors.Is(err, ErrIntentStands) { |
| + t.Fatalf("%s replaced an attempt's intent: %v", id, err) |
| + } |
| + } |
| + if err := b.Begin("other", in("w4")); err != nil { |
| + t.Fatalf("another host's intent was refused: %v", err) |
| + } |
| +} |
| + |
| +// Manager (a): an adoption whose intent cannot be written never reaches |
| +// Swap. It is abandoned unexecuted, waits out the abandon deferral, and runs |
| +// once the book can be written. |
| +func TestAnAdoptionWhoseIntentCannotBeWrittenNeverRuns(t *testing.T) { |
| + h := newHarness(t, "a") |
| + dir := h.onDisk() |
| + h.stage("v2") |
| + readOnly(t, dir) |
| + h.install("v2", "a") |
| + h.pass() |
| + waitFor(t, "the attempt to be abandoned and deferred", func() bool { |
| + s := h.m.schedulerOf("a") |
| + if s == nil { |
| + return false |
| + } |
| + s.mu.Lock() |
| + defer s.mu.Unlock() |
| + hs := s.q.hosts["a"] |
| + return hs != nil && hs.running == nil && hs.notBefore.After(h.clock()) |
| + }) |
| + if calls, started, _ := h.act.snapshot(); len(calls) != 0 || len(started) != 0 { |
| + t.Fatalf("Swap ran (%+v, %v) with no intent written", calls, started) |
| + } |
| + if len(h.book.List()) != 0 { |
| + t.Fatal("a refused attempt wrote a failure") |
| + } |
| + writable(t, dir) |
| + h.advance(abandonDeferral + time.Second) |
| + h.pass() |
| + waitFor(t, "the attempt after the wait", func() bool { return h.servingAll("v2", "a") }) |
| + waitFor(t, "the intent to end", func() bool { return len(fileOf(t, dir).Intents) == 0 }) |
| +} |
| + |
| +// An adoption's intent is on disk while its Swap runs, with its execution's |
| +// identity, mode and start, and is gone once its outcome is written. |
| +func TestAnAdoptionsIntentStandsWhileItRuns(t *testing.T) { |
| + for _, fail := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "done", true: "failed"}[fail], func(t *testing.T) { |
| + h := newHarness(t, "a") |
| + dir := h.onDisk() |
| + h.stage("v2") |
| + if fail { |
| + h.act.outcome = func(activator.Attempt) activator.SwapOutcome { return activator.SwapFailed } |
| + } |
| + gate := h.act.hold("a") |
| + h.install("v2", "a") |
| + h.pass() |
| + waitFor(t, "the swap to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 }) |
| + in, ok := fileOf(t, dir).Intents["a"] |
| + if !ok || !strings.HasPrefix(in.ID, "a") || in.Build != "v2" || in.Mode != "overlap" || !in.Start.Equal(t0) { |
| + t.Fatalf("intent on disk during the swap: %+v, %v", in, ok) |
| + } |
| + close(gate) |
| + waitFor(t, "the intent to end", func() bool { return len(fileOf(t, dir).Intents) == 0 }) |
| + recorded := false |
| + for _, r := range fileOf(t, dir).Records { |
| + recorded = recorded || r.Build == "v2" |
| + } |
| + if recorded != fail { |
| + t.Fatalf("failed %v, but v2 recorded %v", fail, recorded) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Plan review #2: Swap runs an overlap attempt on a foreign-layout row as a |
| +// hold. The intent carries the mode the activator will run (EffectiveMode), |
| +// so a crash mid-swap reopens the book with the instance held down and |
| +// marked, as a hold that went down would have left it. |
| +func TestAForeignLayoutHoldCrashReopensHeldDown(t *testing.T) { |
| + h := newHarness(t, "a") |
| + dir := h.onDisk() |
| + h.stage("v2") |
| + h.act.mode = activator.ModeHold // the foreign row's override |
| + gate := h.act.hold("a") |
| + t.Cleanup(func() { close(gate) }) |
| + h.install("v2", "a") |
| + h.pass() |
| + waitFor(t, "the swap to start", func() bool { _, started, _ := h.act.snapshot(); return len(started) == 1 }) |
| + if in := fileOf(t, dir).Intents["a"]; in.Mode != "hold" { |
| + t.Fatalf("intent %+v: want the effective mode, hold", in) |
| + } |
| + // The agent dies here; the next one opens the book. |
| + b, err := OpenFailureBook(dir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b.Reconciled() != 1 || !b.HeldDown("bab", "blog", "v2", "a") { |
| + t.Fatalf("reconciled %d: want a held down on v2", b.Reconciled()) |
| + } |
| + m, ok := b.HoldRanOn("a") |
| + if !ok || m.Build != "v2" || !strings.HasPrefix(m.Exec, "a") || !m.At.Equal(t0) { |
| + t.Fatalf("marker %+v, %v: want v2's, from the attempt, at its dispatch", m, ok) |
| + } |
| +} |
| + |
| +// Manager (b): a first wake whose intent cannot be written is refused with |
| +// the approved text, and nothing of it is registered. |
| +func TestAFirstWakeWhoseIntentCannotBeWrittenIsRefused(t *testing.T) { |
| + h := newHarness(t, "a") |
| + dir := h.onDisk() |
| + h.stage("v2") |
| + h.registerThenInstall("v2", "a") |
| + h.wakeHost("a") |
| + readOnly(t, dir) |
| + p := h.m.Plan(context.Background(), h.rows.get("a")) |
| + if p.Refuse != BookUnreadableRefusal || p.Boot.StagedPath != "" { |
| + t.Fatalf("plan %+v: want the wake refused with %q", p, BookUnreadableRefusal) |
| + } |
| + h.m.mu.Lock() |
| + _, planned := h.m.planned["a"] |
| + h.m.mu.Unlock() |
| + if planned { |
| + t.Fatal("a refused wake stayed registered as planned") |
| + } |
| +} |
| + |
| +// A planned wake does not take over an attempt's intent: its plan is |
| +// refused as the attempt's own planning refusal is, and nothing is lost. |
| +func TestAFirstWakeNeverReplacesAnAttemptsIntent(t *testing.T) { |
| + h := newHarness(t, "a") |
| + h.onDisk() |
| + h.stage("v2") |
| + h.registerThenInstall("v2", "a") |
| + h.wakeHost("a") |
| + if err := h.book.Begin("a", Intent{Account: "bab", App: "blog", Build: "v3", Mode: "hold", ID: "a1"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + p := h.m.Plan(context.Background(), h.rows.get("a")) |
| + if p.Refuse != "this instance is still moving to another build" { |
| + t.Fatalf("plan %+v: want the moving refusal", p) |
| + } |
| + if in := h.book.intents["a"]; in.ID != "a1" { |
| + t.Fatalf("intent %+v: the attempt's was replaced", in) |
| + } |
| +} |
| + |
| +// Manager (c), with plan review #1: a planned first wake's intent is written |
| +// at its plan, with the wake's identity and start, and ends with the first |
| +// event that settles its build's fate, never with the activator letting the |
| +// host go (an entry deleted before its failure is delivered). |
| +func TestAPlannedWakesIntentEndsWithItsOutcome(t *testing.T) { |
| + ctx := context.Background() |
| + setup := func(t *testing.T) (*harness, string, activator.WakePlan) { |
| + h := newHarness(t, "a") |
| + dir := h.onDisk() |
| + h.stage("v2") |
| + h.registerThenInstall("v2", "a") |
| + h.wakeHost("a") |
| + p := h.m.Plan(ctx, h.rows.get("a")) |
| + if !p.FirstWake || p.Boot.Version != "v2" { |
| + t.Fatalf("plan %+v: want a first wake of v2", p) |
| + } |
| + in, ok := fileOf(t, dir).Intents["a"] |
| + if !ok || in.ID != wakeExecID(p.Token) || in.Build != "v2" || in.Mode != string(p.Mode) || !in.Start.Equal(t0) { |
| + t.Fatalf("intent on disk %+v, %v: want the plan's", in, ok) |
| + } |
| + return h, dir, p |
| + } |
| + ev := func(p activator.WakePlan, h *harness, kind activator.GenEventKind, first bool) activator.GenEvent { |
| + return activator.GenEvent{Kind: kind, Route: h.rows.get("a"), Gen: activator.Gen{N: 3, Build: "v2", Digest: "sha-v2"}, |
| + Mode: p.Mode, FirstWake: first, Token: p.Token, ConfigRev: p.ConfigRev} |
| + } |
| + for _, tc := range []struct { |
| + name string |
| + kinds []activator.GenEventKind |
| + first bool |
| + ends bool |
| + }{ |
| + {"healthy first wake, under observation", []activator.GenEventKind{activator.GenHealthy}, true, false}, |
| + {"settled", []activator.GenEventKind{activator.GenHealthy, activator.GenSettled}, true, true}, |
| + {"failed", []activator.GenEventKind{activator.GenFailed}, true, true}, |
| + {"cancelled with nothing run", []activator.GenEventKind{activator.GenCancelled}, true, true}, |
| + {"the good build booted untried", []activator.GenEventKind{activator.GenHealthy}, false, true}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + h, dir, p := setup(t) |
| + for _, k := range tc.kinds { |
| + e := ev(p, h, k, tc.first) |
| + if k == activator.GenFailed { |
| + e.Failed = p.Boot |
| + } |
| + h.m.OnGen(e) |
| + } |
| + if _, stands := fileOf(t, dir).Intents["a"]; stands == tc.ends { |
| + t.Fatalf("after %v: intent stands %v", tc.kinds, stands) |
| + } |
| + }) |
| + } |
| + t.Run("an older wake's event leaves a newer wake's intent", func(t *testing.T) { |
| + h, dir, old := setup(t) |
| + cur := h.m.Plan(ctx, h.rows.get("a")) |
| + h.m.OnGen(ev(old, h, activator.GenCancelled, true)) |
| + if in := fileOf(t, dir).Intents["a"]; in.ID != wakeExecID(cur.Token) { |
| + t.Fatalf("intent %+v, want the newer wake's", in) |
| + } |
| + }) |
| + t.Run("not by the activator letting the host go", func(t *testing.T) { |
| + h, dir, _ := setup(t) |
| + h.sleepHost("a") // Owns false: the failed wake's entry is gone |
| + h.pass() |
| + if _, stands := fileOf(t, dir).Intents["a"]; !stands { |
| + t.Fatal("dropUnownedWakes ended the intent of a wake whose outcome is not delivered") |
| + } |
| + }) |
| +} |
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go| index 78efcedc3..fa06d268c 100644 |
| --- a/internal/rollout/manager.go |
| +++ b/internal/rollout/manager.go |
| @@ -22,12 +22,16 @@ import ( |
| // stop-then-start for workers and native sidecar members (spec §7.9), and |
| // PublishHeldServing releases a worker fallback's serving identity once |
| // done() has recorded the failure it falls back from (spec §8.1). |
| +// EffectiveMode is the mode Swap will run an attempt of mode m on host in |
| +// (an overlap on a foreign-layout row runs as a hold, spec §2.9), which the |
| +// attempt's intent records before Swap runs (plan review finding 2). |
| type Activator interface { |
| Current(host string) (activator.Gen, bool) |
| Owns(host string) bool |
| Swap(ctx context.Context, a activator.Attempt, newest func(host string) uint64) activator.SwapResult |
| ReplaceWorker(ctx context.Context, a activator.Attempt, fallback *release.StagedIdentity, newest func(host string) uint64) activator.SwapResult |
| PublishHeldServing(host string) |
| + EffectiveMode(host string, m activator.SwapMode) activator.SwapMode |
| } |
| // Routes is the part of *registry.Registry the scheduler reads and writes. |
| @@ -700,11 +704,12 @@ type wakeStart struct { |
| snap uint64 |
| } |
| -// wakeStartsKept bounds Manager.wakeAt per host. A wake that ends with no |
| -// event (refused at its lease, a failed ordinary wake) leaves its entry |
| -// until this many newer plans push it out; a live wake is never that far |
| -// behind its host's newest plan, because the activator waits for a planned |
| -// wake's outcome before it plans again (outcomeFenceWait). |
| +// wakeStartsKept bounds Manager.wakeAt per host. Every planned wake ends |
| +// with a terminal event (Task 5), which drops its entry; a plan the |
| +// activator never ran (one Plan returned to a caller that then gave up) |
| +// leaves its entry until this many newer plans push it out; a live wake is |
| +// never that far behind its host's newest plan, because the activator waits |
| +// for a planned wake's outcome before it plans again (outcomeFenceWait). |
| const wakeStartsKept = 4 |
| // noteWakeStart keeps the start of a planned wake that boots a build (tok): |
| @@ -796,13 +801,17 @@ func (m *Manager) notePlanning(rt registry.Route, tok uint64) { |
| } |
| // dropUnownedWakes forgets the planned wakes of hosts the activator no |
| -// longer owns: a wake that ended with no event (refused at its lease, |
| -// abandoned at its exec) sends no event carrying its token and leaves its |
| -// entry behind, and only the entry's own end may let the record go. (A |
| -// replan's first plan gets no event either; the replan, a newer Plan, |
| -// replaces its entry.) Owns is asked with no lock held (the lock order |
| -// on Manager), and an entry is dropped only if no newer plan replaced it |
| -// meanwhile. |
| +// longer owns: a planned wake of an older activator, or one whose terminal |
| +// event is still on its way, leaves its entry behind, and only the entry's |
| +// own end may let the record go. Owns is asked with no lock held (the lock |
| +// order on Manager), and an entry is dropped only if no newer plan replaced |
| +// it meanwhile. |
| +// |
| +// It never ends a wake's intent (plan review #1): a failed wake's entry goes |
| +// before its GenFailed is delivered (activator's wake, then afterWake), so |
| +// Owns false is no proof the outcome is written, and ending the intent there |
| +// could lose a failure across a failed save and a restart. The intent ends |
| +// on the wake's own terminal event (OnGen); every planned wake sends one. |
| func (m *Manager) dropUnownedWakes() { |
| m.mu.Lock() |
| seen := make(map[string]uint64, len(m.planned)) |
diff --git a/internal/rollout/scheduler.go b/internal/rollout/scheduler.go| index de5b7ea8b..37e442a3c 100644 |
| --- a/internal/rollout/scheduler.go |
| +++ b/internal/rollout/scheduler.go |
| @@ -2,6 +2,7 @@ package rollout |
| import ( |
| "context" |
| + "errors" |
| "strconv" |
| "sync" |
| "time" |
| @@ -144,6 +145,32 @@ func (s *scheduler) execute(ctx context.Context, d dispatch) { |
| d.Started = m.d.Now() |
| d.BookSeq = m.d.Book.Snapshot() |
| d.ExecID = "a" + strconv.FormatUint(m.nextExec(), 10) |
| + if d.Class == classAdopt { |
| + // Write-ahead (Task 5, Astra R1 #5): the adoption's intent is on disk |
| + // before the activator stops or starts anything, so a failure whose |
| + // own save fails is still found by an agent that restarts before a |
| + // save lands (OpenFailureBook). Its mode is the one the activator |
| + // will run (plan review finding 2): Swap runs an overlap on a |
| + // foreign-layout row as a hold, and a reconciled hold holds the |
| + // instance down. A worker's replacement runs its attempt's mode as |
| + // given. No intent, no execution: an attempt whose intent could not |
| + // be written is abandoned unexecuted, and done() defers and retries |
| + // it as any refusal before the exec. Not under scheduler.mu. |
| + mode := a.Mode |
| + if !rt.IsSidecar() { |
| + mode = m.d.Act.EffectiveMode(rt.Host, a.Mode) |
| + } |
| + in := Intent{Account: s.key.acct, App: s.key.app, Build: d.ID.Build, Mode: string(mode), ID: d.ExecID, Start: d.Started} |
| + if err := m.d.Book.Begin(d.ID.Host, in); err != nil { |
| + m.d.Logf("rollout: %s: write the intent of its %s attempt onto %s: %v; not running it", d.ID.Host, d.ID.Cause, d.ID.Build, err) |
| + reason := "failure records could not be written" |
| + if errors.Is(err, ErrIntentStands) { |
| + reason = "an earlier execution on this instance has not recorded its outcome" |
| + } |
| + s.done(d, activator.SwapResult{Outcome: activator.SwapAbandoned, Reason: reason}, "") |
| + return |
| + } |
| + } |
| if rt.IsSidecar() { |
| // Workers and native sidecar members keep stop-then-start on the |
| // same entry (spec §7.9). done() runs the cohort-failure transition |
| @@ -290,6 +317,16 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw |
| serving = h.serving |
| } |
| s.mu.Unlock() |
| + // The adoption's intent ends now (Task 5 rule 2), after every book write |
| + // its outcome makes (the record, the hold-ran marker, served's marker |
| + // clear above), never before, and off scheduler.mu. If those writes |
| + // failed to save, this save fails too and the intent stays on disk, where |
| + // a restart finds it. Ended before the poke below, so the next admission |
| + // for the host does not meet it. |
| + var endErr error |
| + if d.Class == classAdopt && d.ExecID != "" { |
| + endErr = m.d.Book.End(d.ID.Host, d.ExecID) |
| + } |
| if mark || (stale && heldDown) { |
| // A marker was set: the requested one, or one the book set in place |
| // of a stale hold-down (Task 4 fix round 1). |
| @@ -306,6 +343,9 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw |
| if markErr != nil { |
| m.d.Logf("rollout: %s: write hold-ran marker for %s: %v (held in memory)", d.ID.Host, d.ID.Build, markErr) |
| } |
| + if endErr != nil { |
| + m.d.Logf("rollout: %s: end the intent of its attempt onto %s: %v (it stays on disk until a save lands)", d.ID.Host, d.ID.Build, endErr) |
| + } |
| if len(superseded) > 0 { |
| m.d.Logf("rollout: %s/%s build %s failed: superseding attempts past spawn on %v", rec.Account, rec.App, rec.Build, superseded) |
| } |
diff --git a/internal/rollout/sim_test.go b/internal/rollout/sim_test.go| index d3d0e1bbc..9fd86b94d 100644 |
| --- a/internal/rollout/sim_test.go |
| +++ b/internal/rollout/sim_test.go |
| @@ -3,6 +3,7 @@ package rollout |
| import ( |
| "context" |
| "fmt" |
| + "maps" |
| "math/rand" |
| "os" |
| "runtime/debug" |
| @@ -74,6 +75,9 @@ import ( |
| // see what the Pass cleared; the evidence is the model's.) When the |
| // marked build is itself the installed good build, it boots under the |
| // marked-build rule, and I1 requires its wake to boot. |
| +// - I6: every execution's intent (Task 5) ends with its outcome: once the |
| +// run has drained, an intent stands only for a first wake still in |
| +// flight, and it is that wake's. |
| // |
| // Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000). |
| // ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some |
| @@ -507,6 +511,9 @@ func (f simAct) ReplaceWorker(context.Context, activator.Attempt, *release.Stage |
| func (f simAct) PublishHeldServing(string) {} |
| +// EffectiveMode: the simulation has no foreign-layout rows. |
| +func (f simAct) EffectiveMode(_ string, m activator.SwapMode) activator.SwapMode { return m } |
| + |
| // finishSwap is the modelled Swap's end (plan 3), run on the attempt's |
| // goroutine while the simulation waits: the re-check against newest, then |
| // the build's health, then the switch. It is one transition: nothing else |
| @@ -670,6 +677,7 @@ func (s *sim) run(steps int) { |
| close(sw.release) |
| s.await() |
| } |
| + s.checkIntents() |
| } |
| func sortedKeys[V any](m map[string]V) []string { |
| @@ -1007,12 +1015,12 @@ func (s *sim) wakeExec() { |
| rt, plan := h.planRoute, h.plan |
| if s.r.Intn(20) == 0 { |
| // The lease was refused (a previous incarnation still holds it): the |
| - // wake ends before any exec and reports no outcome, so only a Pass |
| - // that finds the activator no longer owns the host lets go of the |
| - // build its plan carried (Manager.planned). Not a build failure, so |
| - // no I1 check. |
| + // wake ends before any exec. The activator reports it as cancelled |
| + // with nothing run (Task 5), which ends the plan's registration and |
| + // its intent. Not a build failure, so no I1 check. |
| h.phase = simAsleep |
| - s.logf("wake %s: refused at its lease, no outcome reported", h.name) |
| + s.logf("wake %s: refused at its lease, cancelled with nothing run", h.name) |
| + s.cancelUnrun(rt, plan) |
| return |
| } |
| if plan.Check != nil && plan.Check(plan.Boot.Version) != nil { |
| @@ -1020,10 +1028,18 @@ func (s *sim) wakeExec() { |
| if next.Refuse != "" || next.Boot.StagedPath == "" || (next.Check != nil && next.Check(next.Boot.Version) != nil) { |
| h.phase = simAsleep |
| s.logf("wake %s: abandoned at exec (plan stale, replan %q)", h.name, next.Refuse) |
| + // The activator's checkPlan cancels a replan its own check |
| + // refused, then afterWake the wake's plan (Task 5): neither ran. |
| + if next.Refuse == "" && next.Boot.StagedPath != "" { |
| + s.cancelUnrun(rt, next) |
| + } |
| + s.cancelUnrun(rt, plan) |
| // The replan read the row afresh (Plan replans a moved row). |
| s.checkWakeRefused(h, s.rows.get(h.name).Version, "the plan went stale and the replan refused") |
| return |
| } |
| + // The replan replaces the plan, which checkPlan cancels: it never ran. |
| + s.cancelUnrun(rt, plan) |
| plan, h.plan, h.planAt, h.stamp = next, next, s.now, s.stampFor(h, next.Boot.Version) |
| h.underMark = s.holdRan[h.name] != "" |
| s.logf("wake %s replanned: boot %s first %v", h.name, plan.Boot.Version, plan.FirstWake) |
| @@ -1039,6 +1055,7 @@ func (s *sim) wakeExec() { |
| // The good build's own gate refused it at the exec: nothing ran. |
| h.phase = simAsleep |
| s.logf("wake %s: %s untried, and the good build %s was refused at its exec: %v", h.name, plan.Boot.Version, good.Version, err) |
| + s.cancelUnrun(rt, plan) |
| s.checkWakeRefused(h, rt.Version, "the untried first wake's good build was refused at its exec") |
| return |
| } |
| @@ -1069,15 +1086,16 @@ func (s *sim) wakeExec() { |
| if s.r.Intn(12) == 0 { |
| // The wake's lifecycle ended it after the build's exec (a stop, a |
| // fence, a step after the spawn): no verdict. The activator reports |
| - // a hold first wake's run (GenCancelled, Executed); an overlap one |
| - // reports nothing, as before (N2). |
| + // it as cancelled after a run (GenCancelled, Executed): a hold one |
| + // leaves the hold-ran restriction (N2), an overlap one only ends the |
| + // wake (Task 5). |
| h.phase, h.gen = simAsleep, activator.Gen{} |
| s.logf("wake %s: first wake of %s ended after its exec, no verdict (mode %s)", h.name, boot.Version, plan.Mode) |
| if plan.Mode == activator.ModeHold { |
| s.holdRanOn(h, boot.Version, h.planAt) |
| - s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Gen: activator.Gen{Build: boot.Version}, Mode: plan.Mode, |
| - FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev, Executed: true}) |
| } |
| + s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Gen: activator.Gen{Build: boot.Version}, Mode: plan.Mode, |
| + FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev, Executed: true}) |
| return |
| } |
| b := s.builds[boot.Version] |
| @@ -1124,6 +1142,32 @@ func (s *sim) wakeExec() { |
| } |
| } |
| +// cancelUnrun is the activator's report of a planned wake that ended before |
| +// its build ran (Task 5): refused at its lease or its exec, or replaced by a |
| +// replan. It records nothing, and ends the plan's registration and intent. |
| +func (s *sim) cancelUnrun(rt registry.Route, plan activator.WakePlan) { |
| + s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Mode: plan.Mode, FirstWake: plan.FirstWake, At: s.now, |
| + Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| +} |
| + |
| +// checkIntents is I6 once the run has drained (Task 5): every execution's |
| +// intent ends with its outcome, so the only intents left are those of first |
| +// wakes still in flight (planned or under observation), each the host's |
| +// current plan's. A leaked intent would refuse the host's adoptions, and an |
| +// agent restart would turn it into a failure nothing failed. |
| +func (s *sim) checkIntents() { |
| + s.book.mu.Lock() |
| + intents := maps.Clone(s.book.intents) |
| + s.book.mu.Unlock() |
| + for _, host := range sortedKeys(intents) { |
| + in, h := intents[host], s.byName[host] |
| + inFlight := h != nil && (h.phase == simWaking || h.phase == simObserving) && h.plan.FirstWake && in.ID == wakeExecID(h.plan.Token) |
| + if !inFlight { |
| + violate("I6", "%s's intent %+v outlived its execution", host, in) |
| + } |
| + } |
| +} |
| + |
| // observeEnd ends a first wake's observation: it settles, or the build exits. |
| func (s *sim) observeEnd() { |
| h := s.pick(func(h *simHost) bool { return h.phase == simObserving }) |
diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go| index 401064810..3500693e3 100644 |
| --- a/internal/rollout/wakeplan.go |
| +++ b/internal/rollout/wakeplan.go |
| @@ -4,6 +4,7 @@ import ( |
| "context" |
| "errors" |
| "fmt" |
| + "time" |
| "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -47,6 +48,12 @@ func HeldDownRefusal(build string) string { |
| return fmt.Sprintf("build %s failed in hold mode on this instance, so it stays down until you roll back or retry the build", build) |
| } |
| +// BookUnreadableRefusal is the wake refusal when this box's failure records |
| +// cannot be read or written: an unreadable book at arming (internal/host's |
| +// refuseTexts), or a planned first wake whose intent could not be saved |
| +// (Task 5). Approved text, shared rather than copied. |
| +const BookUnreadableRefusal = "this box cannot start instances right now: its record of failed builds cannot be read" |
| + |
| // decideWake is spec §8.2's "when the wake picks a build", as a pure |
| // function. The installed build (root's stamp) boots unless a failure record |
| // holds it for this instance, in which case the good build boots from its |
| @@ -207,6 +214,13 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W |
| tok, planAt := m.newWakeToken(rt.Host) |
| defer func() { |
| plan.Token = tok |
| + // Write-ahead (Task 5, Astra R1 #5): a first wake's intent is on disk |
| + // before the activator takes the lease, with no activator lock held |
| + // here. It ends on the wake's terminal event (OnGen). A plan whose |
| + // intent could not be written boots nothing, with the approved |
| + // refusal for a failure book this box cannot keep, and is |
| + // registered as the refusal it now is. |
| + plan = m.beginWake(rt, plan, tok, planAt) |
| m.notePlanned(rt, plan, tok) |
| // The book's snapshot is taken here, at the plan's end, after every |
| // book read the planner made (decideWake, the replan): the wake's |
| @@ -223,7 +237,7 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W |
| // hold that went down, or a swap whose entry stopped under it, is |
| // recorded by done() before anything wakes the instance, so no wake |
| // can boot the good build ahead of a hold-down record. |
| - return activator.WakePlan{Refuse: "this instance is still moving to another build"} |
| + return activator.WakePlan{Refuse: movingRefusal} |
| } |
| if rt.Version == "" { |
| // A cold managed host (spec §6.4, §8.4): its build is installed and |
| @@ -271,6 +285,37 @@ func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.W |
| } |
| } |
| +// movingRefusal is a wake's refusal while the instance's attempt has not |
| +// reported (Plan's attemptRunning check, and beginWake meeting the |
| +// attempt's intent between its slot freeing and its End). |
| +const movingRefusal = "this instance is still moving to another build" |
| + |
| +// beginWake writes the intent of plan, a first wake planned with token tok |
| +// at planAt (Task 5 rule 1), and returns the plan to hand the activator: |
| +// plan itself, or a refusal when the intent could not be written. Any |
| +// other plan writes nothing. The intent's mode is the plan's: the activator |
| +// runs a planned wake in the mode it was planned in (a foreign-layout row |
| +// gets one generation at a time, so no override applies; its events carry |
| +// plan.Mode), and that is the mode a failure of the wake records. |
| +func (m *Manager) beginWake(rt registry.Route, plan activator.WakePlan, tok uint64, planAt time.Time) activator.WakePlan { |
| + if !plan.FirstWake || plan.Boot.Version == "" || plan.Refuse != "" { |
| + return plan |
| + } |
| + in := Intent{Account: release.RouteAccount(rt), App: rt.App, Build: plan.Boot.Version, Mode: string(plan.Mode), |
| + ID: wakeExecID(tok), Start: planAt} |
| + err := m.d.Book.Begin(rt.Host, in) |
| + if err == nil { |
| + return plan |
| + } |
| + m.d.Logf("rollout: %s: write the intent of its first wake of %s: %v; refusing the wake", rt.Host, plan.Boot.Version, err) |
| + if errors.Is(err, ErrIntentStands) { |
| + // The instance's attempt finished and has not yet ended its intent: |
| + // it is still, for a moment, moving. Not a book fault. |
| + return activator.WakePlan{Refuse: movingRefusal, Token: tok} |
| + } |
| + return activator.WakePlan{Refuse: BookUnreadableRefusal, Token: tok} |
| +} |
| + |
| // planRow is one plan against rt (Plan's body, after the cold install), for |
| // the plan whose token is tok. It registers rt's installed build as the |
| // host's in-flight planned build before any slow read (the policy, the |
| @@ -403,7 +448,8 @@ func (m *Manager) OnGen(ev activator.GenEvent) { |
| // Every kind but a first wake's healthy is the wake's last event, which |
| // lets the entry go: an ordinary wake (an untried first wake's good |
| // build included) reports healthy once and nothing after. |
| - started := m.wakeStarted(rt.Host, ev.Token, ev.Kind != activator.GenHealthy || !ev.FirstWake) |
| + ended := ev.Kind != activator.GenHealthy || !ev.FirstWake |
| + started := m.wakeStarted(rt.Host, ev.Token, ended) |
| m.mu.Lock() |
| newest := ev.Token != 0 && m.wakeTok[rt.Host] == ev.Token |
| // The wake has an outcome: its build is no longer only planned, so its |
| @@ -472,6 +518,20 @@ func (m *Manager) OnGen(ev activator.GenEvent) { |
| m.pointerReadDue(appKey{mark.Account, mark.App}) |
| } |
| } |
| + // The wake's intent ends with the first event that settles its planned |
| + // build's fate (Task 5 rule 2, plan review #1), after that event's own |
| + // book write above (buildFailed's record, the hold-ran marker): settled, |
| + // failed, fell back, cancelled (with or without a run), and healthy when |
| + // it is not a first wake's, which is the good build booted under the |
| + // verdict floor with the planned build never run. A first wake's healthy |
| + // event opens its observation and is not the end. An older wake's event |
| + // carries its own token, so it never ends a newer wake's intent. Not |
| + // under activator.Service.mu (emitGen's rule). |
| + if ended { |
| + if err := m.d.Book.End(rt.Host, wakeExecID(ev.Token)); err != nil { |
| + m.d.Logf("rollout: %s: end the intent of its wake: %v (it stays on disk until a save lands)", rt.Host, err) |
| + } |
| + } |
| } |
| // wakeConfigActivated is configActivated for a planned wake's event (final |