amadan

carlos / platform Public

zdd-5 final review w2: hold-ran marker, one-lock config activation, re-arm on proof (N2, N3, N4)

N2: the failure book keeps a per-host hold-ran marker (schema-additive
hold_ran key). A hold adoption that ran and neither passed nor was held
down sets it in done(), in the same save as any record; a hold first wake
ended with no verdict after its exec sets it through GenCancelled, which
now carries Executed (the activator reports the lifecycle-ended case too,
under the outcome fence). While it stands, decideWake and planCheck never
boot the good build in place of the installed one. A good-build write or
`carlos instances retry` clears it; status shows the host as held.

N3: the wake-token check and the config activation are one critical
section of Manager.mu; the ack write stays outside it.

N4: re-arming a retry allowance stays bound to proof that nothing ran;
a test covers a worker attempt whose execution is unknown.

The simulation models the marker (ground truth, new I5, I1 exemptions).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 1cc74fe5d3d80bda6b904766bce5ff292239894a parent 4c71a7f
13 files changed, +1159 −51
  • internal/activator/activator.go +10 −3
  • internal/activator/holdran_event_test.go +97 −0
  • internal/activator/wakeplan.go +33 −7
  • internal/rollout/config_race_test.go +87 −0
  • internal/rollout/failures.go +81 −3
  • internal/rollout/holdran_seq_test.go +78 −0
  • internal/rollout/holdran_test.go +379 −0
  • internal/rollout/manager.go +36 −11
  • internal/rollout/retry_rearm_test.go +55 −0
  • internal/rollout/scheduler.go +26 −5
  • internal/rollout/sim_test.go +142 −13
  • internal/rollout/status.go +15 −0
  • internal/rollout/wakeplan.go +120 −9
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index 625bbba7f..b18ffdb82 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -1393,9 +1393,11 @@ func (a *Service) wake(e *actEntry) {
releaseOutcome func()
)
a.mu.Lock()
- if a.outcomeOwedLocked(e, w) {
+ if a.outcomeOwedLocked(e, w, err) {
// A failed or fallen-back first wake (final review F3; spec §8.1,
- // O7): its failure is recorded only by afterWake's OnGen, after this
+ // O7), or a hold first wake that ran and was ended with no verdict
+ // (final review wave 2, N2): its failure or hold-ran marker is
+ // recorded only by afterWake's OnGen, after this
// critical section and after e.ready closes. The host's outcome
// fence is taken here, before the entry goes and before anyone
// waiting is released, so a new wake of the host does not plan
@@ -1615,7 +1617,12 @@ type wokenInstance struct {
// first wake's B was never exec'd because too little of its health
// window was left (firstWakeMinVerdict), so the good build was booted
// in its place as an ordinary wake.
- planned Gen
+ planned Gen
+ // bootRan: the plan's own build (Boot) ran, or may have (a process came
+ // back, or startMayHaveRun says its start is unproven or uncertain).
+ // afterWake reads it for a hold first wake that ended with no verdict
+ // (holdRanUnjudged; final review wave 2, N2).
+ bootRan bool
failed *release.StagedIdentity
failReason string
fellBack bool
diff --git a/internal/activator/holdran_event_test.go b/internal/activator/holdran_event_test.go
new file mode 100644
index 000000000..0368db5d5
--- /dev/null
+++ b/internal/activator/holdran_event_test.go
@@ -0,0 +1,97 @@
+package activator
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "testing"
+ "time"
+)
+
+// Final review wave 2, N2: a planned first wake that ran its build and was
+// ended with no verdict tells the scheduler the build ran (GenCancelled,
+// Executed, Gen.Build naming it), so a hold build's run holds the instance
+// off its good build. Two ways it happens: an intentional stop cancels the
+// observation of a healthy first wake, or the wake's lifecycle ends it after
+// the exec (here Shutdown during the health check). The second, which
+// reported nothing before, is reported for a hold first wake only, under the
+// host's outcome fence, so no new wake plans before the scheduler has it. A
+// wake that ran nothing reports nothing, and an overlap wake ended the same
+// way is unchanged (nothing reported).
+func TestAFirstWakeEndedAfterItsExecReportsThatItRan(t *testing.T) {
+ const host = "one.example.com"
+ endInHealth := func(t *testing.T, mode SwapMode, refuse bool) (*Service, *genLog, *planGenBackend, chan bool) {
+ dir := t.TempDir()
+ good, next := stagedID(t, dir, "v1"), stagedID(t, dir, "v2")
+ plan := WakePlan{Boot: next, Mode: mode, FirstWake: true, Token: 41, ConfigRev: "sha256:c1"}
+ if mode == ModeOverlap {
+ plan.Fallback = &good
+ }
+ if refuse {
+ plan.Check = func(string) error { return errors.New("held") }
+ }
+ a, b, evs := planActivator(t, plan)
+ fenced := make(chan bool, 4)
+ a.OnGen = func(ev GenEvent) {
+ if ev.Kind == GenCancelled {
+ a.mu.Lock()
+ _, held := a.outcomeFences[host]
+ a.mu.Unlock()
+ fenced <- held
+ }
+ evs.add(ev)
+ }
+ entered := make(chan struct{})
+ var once sync.Once
+ a.genHealth = func(ctx context.Context, _ Gen) error {
+ once.Do(func() { close(entered) })
+ <-ctx.Done()
+ return ctx.Err()
+ }
+ go a.beginRequest(context.Background(), testRoute(host)) //nolint:errcheck
+ if !refuse {
+ <-entered
+ } else {
+ time.Sleep(100 * time.Millisecond)
+ }
+ a.Shutdown(context.Background())
+ return a, evs, b, fenced
+ }
+
+ t.Run("hold, ended in its health check", func(t *testing.T) {
+ _, evs, _, fenced := endInHealth(t, ModeHold, false)
+ ev := evs.wait(t, GenCancelled)
+ if !ev.Executed || ev.Gen.Build != "v2" || !ev.FirstWake || ev.Mode != ModeHold || ev.Token != 41 {
+ t.Fatalf("event %+v: want GenCancelled, Executed, naming v2, with the plan's token", ev)
+ }
+ if !<-fenced {
+ t.Fatal("the host's outcome fence was not held while the event was delivered")
+ }
+ })
+ t.Run("overlap, ended in its health check", func(t *testing.T) {
+ _, evs, _, _ := endInHealth(t, ModeOverlap, false)
+ time.Sleep(100 * time.Millisecond)
+ if k := evs.kinds(); len(k) != 0 {
+ t.Fatalf("events %v: an overlap first wake ended by its lifecycle reports nothing, as before", k)
+ }
+ })
+ t.Run("hold, refused before its exec", func(t *testing.T) {
+ _, evs, b, _ := endInHealth(t, ModeHold, true)
+ time.Sleep(100 * time.Millisecond)
+ if k := evs.kinds(); len(k) != 0 || len(b.started()) != 0 {
+ t.Fatalf("events %v, started %+v: a wake that ran nothing reports nothing", k, b.started())
+ }
+ })
+ t.Run("hold, its observation cancelled", func(t *testing.T) {
+ shrinkFirstWake(t, 55*time.Second, 25*time.Second, 2*time.Second)
+ next := stagedID(t, t.TempDir(), "v2")
+ a, _, evs := planActivator(t, WakePlan{Boot: next, Mode: ModeHold, FirstWake: true, Token: 42})
+ mustServe(t, a)
+ evs.wait(t, GenHealthy)
+ a.BounceFor(host, CauseConfig)
+ ev := evs.wait(t, GenCancelled)
+ if !ev.Executed || ev.Gen.Build != "v2" || ev.Mode != ModeHold || ev.Token != 42 {
+ t.Fatalf("event %+v: want the cancelled observation reported as executed, naming v2", ev)
+ }
+ })
+}
diff --git a/internal/activator/wakeplan.go b/internal/activator/wakeplan.go
index 227cb5480..25dae3664 100644
--- a/internal/activator/wakeplan.go
+++ b/internal/activator/wakeplan.go
@@ -40,8 +40,9 @@ type WakePlan struct {
// build may still boot. Planning runs before the lease and the restore,
// so a sibling's failure recorded since can hold the build it picked.
// The activator asks it at every planned exec, after the restore; the
- // good build always passes. Nil: nothing to re-check. It is called
- // without a.mu held.
+ // good build passes unless a hold build has run on the instance since it
+ // last passed (the scheduler's hold-ran marker), where nothing falls
+ // back. Nil: nothing to re-check. It is called without a.mu held.
Check func(build string) error
// Token identifies this planned wake (final review F4). The planner
// assigns it, unique per plan it makes (a replan, checkPlan's, is a new
@@ -201,6 +202,14 @@ type GenEvent struct {
// one planned wake carries the same pair, on every kind.
Token uint64
ConfigRev string
+ // Executed (cancelled only): the first wake's build ran, or may have,
+ // before the wake ended with no verdict, and Gen.Build names it (final
+ // review wave 2, N2). Always so for a cancelled observation, which
+ // follows a healthy wake. A hold first wake that its lifecycle ended
+ // after the exec (a stop, a fence, a failure after the spawn) is
+ // reported the same way: the build may have migrated the database, so
+ // the scheduler must not boot the good build there next.
+ Executed bool
}
// The first-wake budget (spec §8.2). Vars so tests can shrink them.
@@ -347,6 +356,7 @@ func (a *Service) spawnPlanned(ctx context.Context, e *actEntry, rt registry.Rou
gate = verdictFloorGate(gate, healthBy, &left)
}
proc, ver, g, err := a.startPinnedAt(ctx, rt, genOf(boot), gate, healthBy)
+ w.bootRan = err == nil || startMayHaveRun(proc, err)
if proc == nil && errors.Is(err, errNoTimeForVerdict) {
// B never ran: too little of its window was left for a verdict at
// the exec. The good build boots in its place through the same
@@ -463,6 +473,7 @@ func (a *Service) spawnPlannedWorker(ctx context.Context, e *actEntry, rt regist
canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == ModeOverlap
boot := plan.Boot
proc, id, err := start(boot)
+ w.bootRan = err == nil || startMayHaveRun(proc, err)
if err == nil || proc == nil {
// Ready, or nothing of the build ran: a refusal at the gate, an
// unpublished build (ErrBuildNotPublished), a spawn error with no
@@ -600,8 +611,19 @@ func (a *Service) awaitOutcomes(ctx context.Context, host string) error {
// outcomeOwedLocked reports whether wake w of e ends with a failure that
// afterWake will deliver (GenFailed, GenFellBack): the outcome fence covers
// it. The caller holds a.mu.
-func (a *Service) outcomeOwedLocked(e *actEntry, w *wokenInstance) bool {
- return a.OnGen != nil && w != nil && w.failed != nil && e.plan.planned()
+func (a *Service) outcomeOwedLocked(e *actEntry, w *wokenInstance, err error) bool {
+ return a.OnGen != nil && w != nil && e.plan.planned() && (w.failed != nil || (err != nil && w.holdRanUnjudged(e.plan)))
+}
+
+// holdRanUnjudged reports whether a failed wake of plan ran (or may have
+// run) its first wake's hold build without a verdict on it (final review
+// wave 2, N2): its lifecycle ended it after the exec (a stop, a fence, a
+// step after the spawn), or its start's outcome is uncertain. afterWake
+// reports it as GenCancelled with Executed, and the outcome fence covers
+// the delivery, so the next wake plans only once the scheduler holds the
+// instance off its good build.
+func (w *wokenInstance) holdRanUnjudged(plan WakePlan) bool {
+ return w.bootRan && w.failed == nil && plan.FirstWake && plan.Mode == ModeHold
}
// emitPlanned is emitGen for an event of a wake that ran plan: it carries
@@ -632,8 +654,11 @@ func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) {
now := time.Now()
switch {
case err != nil:
- if w.failed != nil {
+ switch {
+ case w.failed != nil:
a.emitPlanned(plan, GenEvent{Kind: GenFailed, Route: e.route, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder})
+ case w.holdRanUnjudged(plan):
+ a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: e.route, Gen: genOf(plan.Boot), Mode: plan.Mode, FirstWake: true, At: now, Executed: true})
}
case w.fellBack:
a.emitPlanned(plan, GenEvent{Kind: GenFellBack, Route: e.route, Gen: w.planned, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder})
@@ -685,8 +710,9 @@ func (a *Service) watchFirstWake(e *actEntry, g Gen, plan WakePlan) {
cancelled := func(rt registry.Route) {
// Every observation GenHealthy opened is closed by an event; this
// one says "no verdict", so the scheduler stops showing the host
- // as settling and records nothing (spec §6.4, §9.1).
- a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: rt, Gen: g, Mode: plan.Mode, FirstWake: true, At: time.Now()})
+ // as settling and records nothing (spec §6.4, §9.1). The build was
+ // healthy, so it ran (Executed).
+ a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: rt, Gen: g, Mode: plan.Mode, FirstWake: true, At: time.Now(), Executed: true})
}
// The observation was registered in the awake transition
// (armFirstWatchLocked), so a stop that began meanwhile has already
diff --git a/internal/rollout/config_race_test.go b/internal/rollout/config_race_test.go
new file mode 100644
index 000000000..f3c7b9488
--- /dev/null
+++ b/internal/rollout/config_race_test.go
@@ -0,0 +1,87 @@
+package rollout
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+)
+
+// Final review wave 2, N3: an older wake W1's healthy event passes the token
+// check (it is still the host's newest wake), then pauses in served's
+// registry write. Meanwhile a newer wake W2 is planned, with config c2
+// delivered, and its healthy event activates c2. W1 resumes. Its c1 must
+// not be activated: active stays c2, with nothing left pending. Checked
+// before the pause and acted on after it, W1 put c1 back, and the host was
+// shown with config pending for a revision whose attempt had already
+// cleared.
+func TestAStaleWakeCannotActivateItsConfigAfterANewerWake(t *testing.T) {
+ ctx := context.Background()
+ h := newHarness(t, "a")
+ h.stage("v2")
+ h.pass()
+ // The queue stays out of it: no config attempt runs beside the wakes,
+ // so only their events activate anything.
+ s := h.m.schedulerOf("a")
+ s.mu.Lock()
+ s.q.deferHost("a", t0.Add(time.Hour))
+ s.mu.Unlock()
+
+ // W1: an ordinary wake of the good v1, planned with c1 delivered.
+ h.wakeHost("a")
+ h.m.ConfigDelivered("a", "sha256:c1")
+ w1 := h.m.Plan(ctx, h.rows.get("a"))
+ if w1.FirstWake || w1.Boot.Version != "v1" || w1.ConfigRev != "sha256:c1" {
+ t.Fatalf("W1 = %+v: want an ordinary wake of v1 carrying c1", w1)
+ }
+ h.act.serve("a", "v1")
+ g1, _ := h.act.Current("a")
+
+ paused, resume := make(chan struct{}), make(chan struct{})
+ h.rows.mu.Lock()
+ h.rows.setServingHook = func(string) {
+ h.rows.mu.Lock()
+ h.rows.setServingHook = nil
+ h.rows.mu.Unlock()
+ close(paused)
+ <-resume
+ }
+ h.rows.mu.Unlock()
+ w1Done := make(chan struct{})
+ go func() {
+ defer close(w1Done)
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("a"), Gen: g1,
+ Token: w1.Token, ConfigRev: w1.ConfigRev})
+ }()
+ <-paused // W1 is past its token check, inside served
+
+ // W2: v2 installed, c2 delivered, a first wake planned and healthy.
+ h.install("v2", "a")
+ h.m.ConfigDelivered("a", "sha256:c2")
+ w2 := h.m.Plan(ctx, h.rows.get("a"))
+ if !w2.FirstWake || w2.Boot.Version != "v2" || w2.ConfigRev != "sha256:c2" {
+ t.Fatalf("W2 = %+v: want a first wake of v2 carrying c2", w2)
+ }
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("a"), Gen: activator.Gen{N: 99, Build: "v2"},
+ FirstWake: true, Token: w2.Token, ConfigRev: w2.ConfigRev})
+ if got := h.activeConfig("a"); got != "sha256:c2" {
+ t.Fatalf("active = %q after W2's healthy event, want c2", got)
+ }
+
+ close(resume)
+ <-w1Done
+ if got := h.activeConfig("a"); got != "sha256:c2" {
+ t.Fatalf("active = %q: the older wake activated its revision after the newer wake's", got)
+ }
+ if got := h.m.Status().Hosts["a"]; got.ConfigPending {
+ t.Fatalf("a = %+v: config pending after the newer wake activated the delivered revision", got)
+ }
+}
+
+// activeConfig is host's active config revision.
+func (h *harness) activeConfig(host string) string {
+ h.m.mu.Lock()
+ defer h.m.mu.Unlock()
+ return h.m.track(host).active
+}
diff --git a/internal/rollout/failures.go b/internal/rollout/failures.go
index b1caaea41..a2fcbffe3 100644
--- a/internal/rollout/failures.go
+++ b/internal/rollout/failures.go
@@ -42,11 +42,36 @@ type FailureRecord struct {
// Retried are instances `carlos instances retry` cleared: the record no
// longer holds them, and their next attempt is a canary.
Retried []string `json:"retried,omitempty"`
+ // MarkHoldRan asks Record to set Host's hold-ran marker onto this
+ // record's build in the same save (final review wave 2, N2): an attempt
+ // that failed in hold mode, ran its build, and was not held down. It is
+ // an instruction to Record, never stored in the record.
+ MarkHoldRan bool `json:"-"`
+}
+
+// HoldRan is an instance's "hold ran" marker (final review wave 2, N2): a
+// build that executed on it in hold mode and neither passed nor was held
+// down there (an attempt superseded or stopped after its exec, a first wake
+// whose observation was cancelled). It may have migrated the instance's
+// database, so nothing falls back on that instance automatically (spec
+// §6.4): while the marker stands the planner never boots the good build in
+// place of the installed one (decideWake, planCheck). It is a restriction on
+// the instance, not a failure of the build: it blocks no sibling and halts
+// no cohort. Its own good-build write (a pass) or `carlos instances retry`
+// clears it.
+type HoldRan struct {
+ Account string `json:"account"`
+ App string `json:"app"`
+ Build string `json:"build"`
}
type failureFile struct {
Schema int `json:"schema"`
Records []FailureRecord `json:"records"`
+ // HoldRan is the hold-ran markers by host. It is additive to schema 1: an
+ // agent that predates it ignores the key (encoding/json skips unknown
+ // fields), and a file without it has no markers.
+ HoldRan map[string]HoldRan `json:"hold_ran,omitempty"`
}
// FailureBook is the box's failure records. An empty path keeps them in
@@ -59,6 +84,8 @@ type FailureBook struct {
path string
mu sync.Mutex
recs []FailureRecord
+ // held is the hold-ran markers by host (HoldRan), saved with the records.
+ held map[string]HoldRan
// dirty is set when a save failed. Memory is the truth meanwhile, and
// every later mutating call and Prune retries the save until it lands,
// so the record still survives an agent restart (spec §8.4).
@@ -92,6 +119,7 @@ func OpenFailureBook(dataDir string) (*FailureBook, error) {
return nil, fmt.Errorf("%s: unknown schema %d", b.path, f.Schema)
}
b.recs = f.Records
+ b.held = f.HoldRan
return b, nil
}
@@ -101,7 +129,7 @@ func (b *FailureBook) saveLocked() error {
if b.path == "" {
return nil
}
- raw, err := json.MarshalIndent(failureFile{Schema: 1, Records: b.recs}, "", " ")
+ raw, err := json.MarshalIndent(failureFile{Schema: 1, Records: b.recs, HoldRan: b.held}, "", " ")
if err == nil {
err = release.WriteFileAtomic(b.path, raw, 0o644)
}
@@ -131,10 +159,13 @@ func (b *FailureBook) find(acct, app, build string) int {
// after the latest failure clears it. The in-memory record is kept even when
// the file write fails, so the caller's transition still blocks the build;
// the error is for the journal, and the next call on the book (this one
-// again, if nothing new, or Prune) retries the save.
+// again, if nothing new, or Prune) retries the save. r.MarkHoldRan sets r's
+// host's hold-ran marker in the same save.
func (b *FailureBook) Record(r FailureRecord) (bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
+ marked := r.MarkHoldRan && b.markLocked(r.Host, HoldRan{Account: r.Account, App: r.App, Build: r.Build})
+ r.MarkHoldRan = false
i := b.find(r.Account, r.App, r.Build)
if i < 0 {
r.Retried = nil
@@ -142,7 +173,7 @@ func (b *FailureBook) Record(r FailureRecord) (bool, error) {
return true, b.saveLocked()
}
cur := &b.recs[i]
- changed := false
+ changed := marked
if j := slices.Index(cur.Retried, r.Host); j >= 0 {
cur.Retried = slices.Delete(slices.Clone(cur.Retried), j, j+1)
changed = true
@@ -163,6 +194,53 @@ func (b *FailureBook) Record(r FailureRecord) (bool, error) {
return false, b.saveLocked()
}
+// markLocked sets host's hold-ran marker to h and reports whether that
+// changed anything. The caller holds b.mu and saves.
+func (b *FailureBook) markLocked(host string, h HoldRan) bool {
+ if cur, ok := b.held[host]; ok && cur == h {
+ return false
+ }
+ if b.held == nil {
+ b.held = map[string]HoldRan{}
+ }
+ b.held[host] = h
+ return true
+}
+
+// MarkHoldRan sets host's hold-ran marker (HoldRan) and saves it: the
+// outcome that set it changed nothing else in the book. A marker already
+// standing for another build is replaced: the newest hold build to run on
+// the instance is the one its database may now need.
+func (b *FailureBook) MarkHoldRan(host string, h HoldRan) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if !b.markLocked(host, h) {
+ return b.retryLocked()
+ }
+ return b.saveLocked()
+}
+
+// HoldRanOn returns host's hold-ran marker, if one stands.
+func (b *FailureBook) HoldRanOn(host string) (HoldRan, bool) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ h, ok := b.held[host]
+ return h, ok
+}
+
+// ClearHoldRan drops host's hold-ran marker: the instance recorded a new
+// good build, or `carlos instances retry` named it. It reports whether a
+// marker stood.
+func (b *FailureBook) ClearHoldRan(host string) (bool, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if _, ok := b.held[host]; !ok {
+ return false, b.retryLocked()
+ }
+ delete(b.held, host)
+ return true, b.saveLocked()
+}
+
// Lookup returns the record for (acct, app, build), if one stands.
func (b *FailureBook) Lookup(acct, app, build string) (FailureRecord, bool) {
b.mu.Lock()
diff --git a/internal/rollout/holdran_seq_test.go b/internal/rollout/holdran_seq_test.go
new file mode 100644
index 000000000..6efb41f6f
--- /dev/null
+++ b/internal/rollout/holdran_seq_test.go
@@ -0,0 +1,78 @@
+package rollout
+
+import (
+ "context"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/appconfig"
+)
+
+// sleepHost makes host a sleeper: no current generation, no entry.
+func (h *harness) sleepHost(host string) {
+ h.act.mu.Lock()
+ defer h.act.mu.Unlock()
+ delete(h.act.gens, host)
+ delete(h.act.owned, host)
+}
+
+// Final review wave 2, N2, the reviewer's sequence: instance h serves its
+// good build v1, and its hold attempt onto v2 runs v2's process. A sibling's
+// first wake then fails v2, and the cohort-failure transition supersedes h's
+// attempt, which the activator abandons after v2 ran (Executed). Nothing is
+// held down on h: it was superseded, not seen to fail. h stops. Its next
+// wake must not boot v1: v2 may have migrated h's database (spec §6.4). The
+// sibling's record blocks v2 for h, so before the marker the planner sent
+// the wake to the good build. It boots v2 again instead, in hold mode, with
+// nothing behind it.
+func TestASupersededHoldThatRanKeepsTheInstanceOffItsGoodBuild(t *testing.T) {
+ ctx := context.Background()
+ h := newHarness(t, "h", "s")
+ h.swap = appconfig.SwapHold
+ h.stage("v2")
+ h.act.mode = activator.ModeHold
+ h.act.executed = func(a activator.Attempt, _ activator.SwapOutcome) bool { return a.Build == "v2" }
+ h.sleepHost("s")
+ h.pass()
+
+ gate := h.act.hold("h")
+ h.install("v2", "h", "s")
+ h.pass()
+ waitFor(t, "h's hold attempt onto v2", func() bool {
+ _, started, _ := h.act.snapshot()
+ return len(started) == 1 && started[0] == "h:v2"
+ })
+
+ // The sibling's first wake of v2 fails, in hold: a record for v2 (held
+ // down on s), and the halt supersedes h's attempt.
+ h.wakeHost("s")
+ sp := h.m.Plan(ctx, h.rows.get("s"))
+ if !sp.FirstWake || sp.Boot.Version != "v2" || sp.Mode != activator.ModeHold {
+ t.Fatalf("s's plan %+v: want a hold first wake of v2", sp)
+ }
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: h.rows.get("s"), Failed: sp.Boot, Mode: sp.Mode,
+ FirstWake: true, Reason: "health check failed", Token: sp.Token, ConfigRev: sp.ConfigRev})
+ close(gate)
+ waitFor(t, "h's attempt to finish", func() bool { return !h.m.attemptRunning("h") })
+ calls, _, _ := h.act.snapshot()
+ if len(calls) != 1 || calls[0].outcome != activator.SwapAbandoned {
+ t.Fatalf("calls %+v: want h's attempt abandoned (superseded by the halt)", calls)
+ }
+ if h.book.HeldDown("bab", "blog", "v2", "h") {
+ t.Fatal("h is held down: a superseded attempt is no failure of the build")
+ }
+
+ // h stops, and wakes again.
+ h.sleepHost("h")
+ h.wakeHost("h")
+ p := h.m.Plan(ctx, h.rows.get("h"))
+ if p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold {
+ t.Fatalf("h's plan %+v: want v2 again, a hold first wake with no fallback, never the good v1", p)
+ }
+ if p.Check == nil || p.Check("v2") != nil {
+ t.Fatal("h's check refuses v2, the build its database may now need")
+ }
+ if p.Check("v1") == nil {
+ t.Fatal("h's check lets the good v1 boot after the hold build ran")
+ }
+}
diff --git a/internal/rollout/holdran_test.go b/internal/rollout/holdran_test.go
new file mode 100644
index 000000000..f7c2948b5
--- /dev/null
+++ b/internal/rollout/holdran_test.go
@@ -0,0 +1,379 @@
+package rollout
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+var markV2 = HoldRan{Account: "bab", App: "blog", Build: "v2"}
+
+// Final review wave 2, N2: decideWake's rule for an instance with a
+// hold-ran marker. The good build never boots in place of the installed
+// build.
+func TestDecideWakeWithAHoldRanMarker(t *testing.T) {
+ overlap := DefaultPolicy()
+ book := func(recs ...FailureRecord) *FailureBook {
+ b := &FailureBook{}
+ for _, r := range recs {
+ b.Record(r)
+ }
+ b.MarkHoldRan("a", markV2)
+ return b
+ }
+ sibV2 := FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "x", At: t0}
+ sibV3 := FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "x", At: t0}
+
+ // The marked build, blocked for the siblings: it boots, hold, no fallback.
+ p, err := decideWake(inputs(wakeRoute("v2", "v1"), book(sibV2), overlap))
+ if err != nil || p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold {
+ t.Fatalf("installed is the marked build: want a hold first wake of v2 with no fallback, got %+v, %v", p, err)
+ }
+ // Unblocked, and its release allows overlap: still hold, no fallback.
+ if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), book(), overlap)); p.Boot.Version != "v2" || p.Fallback != nil || p.Mode != activator.ModeHold {
+ t.Fatalf("installed is the marked build, unblocked: want hold with no fallback, got %+v", p)
+ }
+ // Held down on the instance: the hold-down refusal still comes first.
+ down := book(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}, At: t0})
+ if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), down, overlap)); p.Refuse != HeldDownRefusal("v2") {
+ t.Fatalf("marked build held down: want the hold-down refusal, got %+v", p)
+ }
+ // Another installed build held for the instance: refused, never v1.
+ if p, _ := decideWake(inputs(wakeRoute("v3", "v1"), book(sibV3), overlap)); p.Refuse != HeldDownRefusal("v2") || p.Boot.StagedPath != "" {
+ t.Fatalf("another held build: want the wake refused in the hold-down words, got %+v", p)
+ }
+ // Another, unblocked build: a first wake with no fallback, in its own mode.
+ if p, _ := decideWake(inputs(wakeRoute("v3", "v1"), book(), overlap)); p.Refuse != "" || p.Boot.Version != "v3" || !p.FirstWake ||
+ p.Fallback != nil || p.Mode != activator.ModeOverlap {
+ t.Fatalf("another unblocked build: want an overlap first wake of v3 with no fallback, got %+v", p)
+ }
+ // The good build installed by hand (a rollback): what was installed boots.
+ if p, _ := decideWake(inputs(wakeRoute("v1", "v1"), book(), overlap)); p.Boot.Version != "v1" || p.FirstWake || p.Refuse != "" {
+ t.Fatalf("the good build installed: want its ordinary wake, got %+v", p)
+ }
+ // A staging refusal never boots the good build instead.
+ in := inputs(wakeRoute("v2", "v1"), book(), overlap)
+ in.Demand = func(string, string) (string, string, error) { return "", "", errors.New("digest mismatch") }
+ if p, err := decideWake(in); err == nil || p.Refuse == "" || p.Boot.StagedPath != "" {
+ t.Fatalf("a staging refusal under the marker: want the wake refused, got %+v, %v", p, err)
+ }
+ // Another app's marker under the same host name changes nothing.
+ other := &FailureBook{}
+ other.Record(sibV2)
+ other.MarkHoldRan("a", HoldRan{Account: "bab", App: "other", Build: "v2"})
+ if p, _ := decideWake(inputs(wakeRoute("v2", "v1"), other, overlap)); p.Boot.Version != "v1" || p.FirstWake {
+ t.Fatalf("another app's marker: want the usual recovery onto v1, got %+v", p)
+ }
+}
+
+// Final review wave 2, N2: a planned wake onto the good build made before
+// the marker existed (a recovery for the blocked v2) is refused at its exec
+// once the marker stands. The marker comes from a hold first wake whose
+// observation was cancelled after it ran. The refusal uses existing words.
+func TestAHoldRanMarkerRefusesTheGoodBuildAtTheExec(t *testing.T) {
+ ctx := context.Background()
+ h := newHarness(t, "h")
+ h.stage("v2")
+ h.install("v2", "h")
+ h.sleepHost("h")
+ h.pass()
+ h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0})
+ h.wakeHost("h")
+ recovery := h.m.Plan(ctx, h.rows.get("h"))
+ if recovery.Boot.Version != "v1" || recovery.FirstWake || recovery.Check == nil {
+ t.Fatalf("plan %+v: want the recovery onto the good v1 with its check (v2 is blocked, no marker yet)", recovery)
+ }
+ if err := recovery.Check("v1"); err != nil {
+ t.Fatalf("before the marker the good build passes the check: %v", err)
+ }
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("h"), Gen: activator.Gen{Build: "v2"},
+ Mode: activator.ModeHold, FirstWake: true, Executed: true})
+ err := recovery.Check("v1")
+ if err == nil {
+ t.Fatal("the recovery onto v1 passed its exec check after a hold build ran on h")
+ }
+ if want := HeldDownRefusal("v2"); err.Error() != want {
+ t.Fatalf("refusal %q, want the existing words %q", err, want)
+ }
+}
+
+// Final review wave 2, N2: a planned hold first wake whose observation was
+// cancelled after its build ran (the activator's GenCancelled, Executed)
+// leaves the marker, so a sibling's failure of the build afterwards does not
+// send the instance's next wake to the good build. The marker is a
+// restriction on the instance, not a failure: nothing is recorded or halted.
+// The instance passing on the build (a settled first wake records it good)
+// clears the marker, and with it the restriction.
+func TestAFirstWakeCancelledAfterItsHoldBuildRanAndItsClearingByAPass(t *testing.T) {
+ ctx := context.Background()
+ h := newHarness(t, "s")
+ h.swap = appconfig.SwapHold
+ h.stage("v2", "v3")
+ h.install("v2", "s")
+ h.sleepHost("s")
+ h.pass()
+ h.wakeHost("s")
+ p := h.m.Plan(ctx, h.rows.get("s"))
+ if !p.FirstWake || p.Boot.Version != "v2" || p.Mode != activator.ModeHold {
+ t.Fatalf("plan %+v: want a hold first wake of v2", p)
+ }
+ g := activator.Gen{N: 7, Build: "v2", StagedPath: p.Boot.StagedPath, Digest: p.Boot.Digest}
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true, Token: p.Token})
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("s"), Gen: g, Mode: p.Mode, FirstWake: true,
+ Token: p.Token, Executed: true})
+ if got, ok := h.book.HoldRanOn("s"); !ok || got != markV2 {
+ t.Fatalf("marker = %+v, %v: want v2's", got, ok)
+ }
+ if _, ok := h.book.Lookup("bab", "blog", "v2"); ok {
+ t.Fatal("a cancelled observation recorded a failure")
+ }
+
+ h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0})
+ h.sleepHost("s")
+ h.wakeHost("s")
+ p = h.m.Plan(ctx, h.rows.get("s"))
+ if p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake || p.Fallback != nil || p.Mode != activator.ModeHold {
+ t.Fatalf("plan %+v: want v2 again, never the good v1", p)
+ }
+
+ // It runs and settles: v2 is s's good build now, and the marker goes.
+ h.act.serve("s", "v2")
+ cur, _ := h.act.Current("s")
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("s"), Gen: cur, Mode: p.Mode, FirstWake: true, Token: p.Token})
+ h.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: h.rows.get("s"), Gen: cur, Mode: p.Mode, FirstWake: true, Token: p.Token})
+ if r := h.rows.get("s"); r.GoodVersion != "v2" {
+ t.Fatalf("good build %q after the settle, want v2", r.GoodVersion)
+ }
+ if got, ok := h.book.HoldRanOn("s"); ok {
+ t.Fatalf("marker %+v survived the instance passing on a build", got)
+ }
+ // The usual rule again: a held v3 sends the wake to the good build.
+ h.install("v3", "s")
+ h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", At: t0})
+ h.sleepHost("s")
+ h.wakeHost("s")
+ if p := h.m.Plan(ctx, h.rows.get("s")); p.Boot.Version != "v2" || p.FirstWake {
+ t.Fatalf("plan %+v after the marker cleared: want the recovery onto the good v2", p)
+ }
+}
+
+// Final review wave 2, N2: `carlos instances retry` clears the host's
+// marker, with or without a failure record to clear beside it.
+func TestRetryHostClearsTheHoldRanMarker(t *testing.T) {
+ t.Run("with a record", func(t *testing.T) {
+ h := newHarness(t, "a")
+ h.stage("v2")
+ h.install("v2", "a")
+ h.sleepHost("a")
+ h.pass()
+ h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0})
+ h.book.MarkHoldRan("a", markV2)
+ if !h.m.RetryHost("a") {
+ t.Fatal("RetryHost reported nothing cleared")
+ }
+ if _, ok := h.book.HoldRanOn("a"); ok {
+ t.Fatal("the marker survived the host's retry")
+ }
+ })
+ t.Run("marker only", func(t *testing.T) {
+ h := newHarness(t, "a", "b")
+ h.stage("v2")
+ h.install("v2", "a")
+ h.sleepHost("a")
+ h.pass()
+ h.book.MarkHoldRan("a", markV2)
+ h.book.MarkHoldRan("b", markV2)
+ if !h.m.RetryHost("a") {
+ t.Fatal("RetryHost reported nothing cleared with a marker standing")
+ }
+ if _, ok := h.book.HoldRanOn("a"); ok {
+ t.Fatal("the marker survived the host's retry")
+ }
+ if _, ok := h.book.HoldRanOn("b"); !ok {
+ t.Fatal("a's retry cleared b's marker")
+ }
+ })
+}
+
+// Final review wave 2, N2, the scheduler's half: which attempt outcomes set
+// the marker. A hold adoption that ran and did not pass, and was not held
+// down, sets it; a failed one sets it in the same save as its record. One
+// that ran nothing, an overlap attempt, a restart, a hold-down and a pass do
+// not.
+func TestWhichAttemptOutcomesLeaveTheHoldRanMarker(t *testing.T) {
+ adopt := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRelease, Build: "v2"}, SchedRev: 1, Class: classAdopt, Cohort: cohortKey("v2", classAdopt)}
+ restart := dispatch{ID: Identity{Host: "a", Cause: activator.CauseRestart, Build: "v1", RestartID: "r1"}, SchedRev: 1, Class: classRecover, Cohort: cohortKey("v1", classRecover)}
+ for _, tc := range []struct {
+ name string
+ d dispatch
+ res activator.SwapResult
+ mode activator.SwapMode
+ want bool
+ }{
+ {"hold, superseded after it ran", adopt, activator.SwapResult{Outcome: activator.SwapAbandoned, Executed: true}, activator.ModeHold, true},
+ {"hold, failed after it ran, not held down", adopt, activator.SwapResult{Outcome: activator.SwapFailed, Reason: "health check failed", Executed: true}, activator.ModeHold, true},
+ {"hold, abandoned before it ran", adopt, activator.SwapResult{Outcome: activator.SwapAbandoned}, activator.ModeHold, false},
+ {"hold, down: held down instead", adopt, activator.SwapResult{Outcome: activator.SwapDown, Reason: "health check failed", Executed: true}, activator.ModeHold, false},
+ {"overlap, superseded after it ran", adopt, activator.SwapResult{Outcome: activator.SwapAbandoned, Executed: true}, activator.ModeOverlap, false},
+ {"hold restart, superseded after it ran", restart, activator.SwapResult{Outcome: activator.SwapAbandoned, Executed: true}, activator.ModeHold, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ h := newHarness(t, "a")
+ dir := t.TempDir()
+ b, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h.book, h.m.d.Book = b, b
+ s := h.m.scheduler(appKey{"bab", "blog"})
+ s.mu.Lock()
+ slot := s.q.slot("a", t0)
+ slot.serving, slot.good = "v1", "v1"
+ d := tc.d
+ slot.running = &d
+ s.q.running++
+ s.mu.Unlock()
+ s.done(tc.d, tc.res, tc.mode)
+
+ again, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, ok := again.HoldRanOn("a")
+ if ok != tc.want || (ok && got != markV2) {
+ t.Fatalf("marker on disk = %+v, %v; want set %v", got, ok, tc.want)
+ }
+ if tc.res.Outcome == activator.SwapFailed {
+ if _, ok := again.Lookup("bab", "blog", "v2"); !ok {
+ t.Fatal("the failure record is not on disk beside the marker")
+ }
+ }
+ })
+ }
+}
+
+// Final review wave 2, N2: status shows an instance with a marker as held,
+// in the existing words (HeldBy, AttemptDown), never new ones.
+func TestStatusShowsAHoldRanInstanceAsHeld(t *testing.T) {
+ h := newHarness(t, "a", "b")
+ h.stage("v2", "v3")
+ h.install("v2", "a")
+ h.install("v3", "b")
+ h.sleepHost("a")
+ h.sleepHost("b")
+ h.pass()
+ h.book.MarkHoldRan("a", markV2)
+ h.book.MarkHoldRan("b", markV2)
+ h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "sib", Reason: "exited early", At: t0})
+ r := h.m.Status()
+ if got := r.Hosts["a"]; got.HeldBy != "v2" || got.Attempt != "" {
+ t.Fatalf("a = %+v: want held by v2 (its wake boots v2, nothing else)", got)
+ }
+ if got := r.Hosts["b"]; got.HeldBy != "v3" || got.Attempt != status.AttemptDown || got.Reason != status.ReasonExit {
+ t.Fatalf("b = %+v: want down, held by v3 (its wakes are refused, not fallen back)", got)
+ }
+}
+
+// Final review wave 2, N2: the markers survive a reopen; a record written
+// with MarkHoldRan carries its marker in the same file and never stores the
+// instruction; a clear is saved too.
+func TestTheHoldRanMarkerSurvivesReopen(t *testing.T) {
+ dir := t.TempDir()
+ b, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := b.MarkHoldRan("a.example.com", markV2); err != nil {
+ t.Fatal(err)
+ }
+ r := rec("v3", "c.example.com")
+ r.MarkHoldRan = true
+ if _, err := b.Record(r); err != nil {
+ t.Fatal(err)
+ }
+ raw, err := os.ReadFile(filepath.Join(dir, FailureFile))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(strings.ToLower(string(raw)), "markholdran") || strings.Contains(string(raw), "mark_hold_ran") {
+ t.Fatalf("the instruction was stored:\n%s", raw)
+ }
+ again, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, ok := again.HoldRanOn("a.example.com"); !ok || got != markV2 {
+ t.Fatalf("a's marker after reopen = %+v, %v", got, ok)
+ }
+ if got, ok := again.HoldRanOn("c.example.com"); !ok || got.Build != "v3" {
+ t.Fatalf("c's marker (set with its record) after reopen = %+v, %v", got, ok)
+ }
+ if rec, _ := again.Lookup("bab", "blog", "v3"); rec.MarkHoldRan {
+ t.Fatal("a record came back carrying the instruction")
+ }
+ if cleared, err := again.ClearHoldRan("a.example.com"); !cleared || err != nil {
+ t.Fatalf("ClearHoldRan = %v, %v", cleared, err)
+ }
+ third, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := third.HoldRanOn("a.example.com"); ok {
+ t.Fatal("the clear was not saved")
+ }
+ if _, ok := third.HoldRanOn("c.example.com"); !ok {
+ t.Fatal("clearing a's marker dropped c's")
+ }
+}
+
+// Final review wave 2, N2: the markers are additive to schema 1. A file an
+// older agent wrote (no hold_ran key) opens with no markers. A file with
+// markers still parses as an older agent reads it: plain encoding/json into
+// the old shape, which skips the unknown key, and every record intact.
+func TestTheHoldRanMarkerIsSchemaAdditive(t *testing.T) {
+ dir := t.TempDir()
+ old := `{"schema": 1, "records": [{"account": "bab", "app": "blog", "build": "v2", "host": "a", "cause": "release",
+ "mode": "hold", "reason": "health check failed", "promoted_at": "2023-11-14T22:13:20Z", "at": "2023-11-14T22:13:20Z"}]}`
+ if err := os.WriteFile(filepath.Join(dir, FailureFile), []byte(old), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ b, err := OpenFailureBook(dir)
+ if err != nil {
+ t.Fatalf("an older agent's file: %v", err)
+ }
+ if _, ok := b.HoldRanOn("a"); ok {
+ t.Fatal("a file without markers came back with one")
+ }
+ if !b.Blocks("bab", "blog", "v2", "x", false) {
+ t.Fatal("the older file's record was lost")
+ }
+ if err := b.MarkHoldRan("a", markV2); err != nil {
+ t.Fatal(err)
+ }
+ raw, err := os.ReadFile(filepath.Join(dir, FailureFile))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The schema 1 shape an agent before the markers decodes into.
+ var older struct {
+ Schema int `json:"schema"`
+ Records []FailureRecord `json:"records"`
+ }
+ if err := json.Unmarshal(raw, &older); err != nil {
+ t.Fatalf("an older agent cannot parse the file with markers: %v", err)
+ }
+ if older.Schema != 1 || len(older.Records) != 1 || older.Records[0].Build != "v2" {
+ t.Fatalf("older agent's view = %+v: want schema 1 and the one record", older)
+ }
+ if !strings.Contains(string(raw), `"hold_ran"`) {
+ t.Fatalf("the marker is not in the file:\n%s", raw)
+ }
+}
diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go
index 5b0ba5ee1..154719138 100644
--- a/internal/rollout/manager.go
+++ b/internal/rollout/manager.go
@@ -507,6 +507,11 @@ func (m *Manager) retryBuild(k appKey, build, host string) {
// Admission cannot run between the two, so the adoption onto the cleared
// build is the attempt the flag lets through, as a canary, and a restart or
// config attempt cannot start in between and spend the flag.
+//
+// It also clears host's hold-ran marker (final review wave 2, N2), whatever
+// build the marker names and whether or not a record held the host: the
+// retry is the explicit recovery spec §6.4 asks for. True means either was
+// cleared.
func (m *Manager) RetryHost(host string) bool {
m.mu.Lock()
k, ok := m.hostApp[host]
@@ -531,14 +536,20 @@ func (m *Manager) RetryHost(host string) bool {
s.q.wantBuild(host, activator.CauseRetry, build, m.d.Now())
s.q.retryHost(host)
}
+ unmarked, merr := m.d.Book.ClearHoldRan(host)
s.mu.Unlock()
if err != nil {
m.d.Logf("rollout: retry %s on %s: %v (cleared in memory)", build, host, err)
}
+ if merr != nil {
+ m.d.Logf("rollout: retry on %s: clear hold-ran marker: %v (cleared in memory)", host, merr)
+ }
if cleared {
m.retryBuild(k, build, host)
+ } else if unmarked {
+ m.changed()
}
- return cleared
+ return cleared || unmarked
}
// failedUnder is the PromotedAt a failure record keeps (spec rev 14 §8.4):
@@ -951,6 +962,10 @@ func (m *Manager) writeGood(host string, g activator.Gen) {
// serialises every writer and each has just proven g current, so a retry
// only covers a write from outside this manager; a second loss is logged
// and the next serve tries again. The caller holds m.goodMu.
+//
+// A good build recorded (written, or found already written) clears the
+// host's hold-ran marker (final review wave 2, N2): the instance passed on
+// g, so g is what a later wake may fall back to.
func (m *Manager) writeGoodLocked(host string, g activator.Gen) {
if g.Build == "" || g.Digest == "" {
return
@@ -962,12 +977,12 @@ func (m *Manager) writeGoodLocked(host string, g activator.Gen) {
return
}
if cur.Good() == next {
- m.observeGood(host, next.Version)
+ m.goodRecorded(host, next.Version)
return
}
err = m.d.Routes.SetGoodBuild(host, cur.Good(), next)
if err == nil {
- m.observeGood(host, next.Version)
+ m.goodRecorded(host, next.Version)
return
}
if !errors.Is(err, registry.ErrGoodBuildChanged) {
@@ -978,6 +993,15 @@ func (m *Manager) writeGoodLocked(host string, g activator.Gen) {
m.d.Logf("rollout: %s: good build moved twice under this write, leaving it", host)
}
+// goodRecorded follows a good-build write that landed (writeGoodLocked):
+// the hold-ran marker goes, and the queue learns the build.
+func (m *Manager) goodRecorded(host, ver string) {
+ if _, err := m.d.Book.ClearHoldRan(host); err != nil {
+ m.d.Logf("rollout: %s: clear hold-ran marker after good build %s: %v (cleared in memory)", host, ver, err)
+ }
+ m.observeGood(host, ver)
+}
+
func (m *Manager) observeGood(host, ver string) {
s := m.schedulerOf(host)
if s == nil {
@@ -997,6 +1021,14 @@ func (m *Manager) configActivated(host, rev string) {
m.mu.Lock()
write, _ := m.track(host).activate(rev)
m.mu.Unlock()
+ m.activationDone(host, rev, write)
+}
+
+// activationDone is the rest of an activation of rev, after the config
+// track has recorded it under m.mu: the queue's pending config attempt for
+// rev is cleared, and the held ack (write, nil when none) is written. Called
+// with no lock held: the ack is I/O.
+func (m *Manager) activationDone(host, rev string, write func()) {
if s := m.schedulerOf(host); s != nil {
s.mu.Lock()
s.q.configActive(host, rev)
@@ -1021,14 +1053,7 @@ func (m *Manager) configSlept(host, rev string) {
}
write, _ := t.activate(rev)
m.mu.Unlock()
- if s := m.schedulerOf(host); s != nil {
- s.mu.Lock()
- s.q.configActive(host, rev)
- s.mu.Unlock()
- }
- if write != nil {
- write()
- }
+ m.activationDone(host, rev, write)
}
// configFailure counts a failed attempt that carried config revision rev
diff --git a/internal/rollout/retry_rearm_test.go b/internal/rollout/retry_rearm_test.go
index 182329a21..b0b12db16 100644
--- a/internal/rollout/retry_rearm_test.go
+++ b/internal/rollout/retry_rearm_test.go
@@ -6,6 +6,7 @@ import (
"time"
"github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/release"
)
// Final review F8: `carlos instances retry <host>` promises that host's next
@@ -87,3 +88,57 @@ func TestARetryAllowanceComesBackOnlyWhenNothingRan(t *testing.T) {
})
}
}
+
+// Final review wave 2, N4: only proof that nothing ran gives the allowance
+// back. A worker attempt abandoned when its caller left can still have had
+// its respawn exec the build (the activator joins the respawn, and past its
+// bound reports Executed true: unknown). Executed true spends the
+// allowance, whether the build surely ran or only may have; Executed false,
+// the activator's proof that nothing ran, gives it back.
+func TestARetryAllowanceIsSpentWhenAWorkerAttemptMayHaveRun(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ executed bool
+ }{
+ {"execution unknown", true},
+ {"proven unexecuted", false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ h := workerHarness(t, "a", "b")
+ h.stage("v2")
+ var bTries atomic.Int32
+ h.act.mu.Lock()
+ h.act.worker = func(a activator.Attempt, _ *release.StagedIdentity) activator.SwapResult {
+ switch {
+ case a.Build != "v2":
+ return activator.SwapResult{Outcome: activator.SwapDone}
+ case a.Host == "a":
+ return activator.SwapResult{Outcome: activator.SwapFailed, Reason: "exited within 10s of starting", Executed: true}
+ case bTries.Add(1) == 1:
+ return activator.SwapResult{Outcome: activator.SwapAbandoned, Reason: "swap: an intentional stop cancelled the attempt", Executed: tc.executed}
+ }
+ return activator.SwapResult{Outcome: activator.SwapDone, Executed: true}
+ }
+ h.act.mu.Unlock()
+ h.install("v2", "a", "b")
+ h.pass()
+ waitFor(t, "the halt", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok })
+ waitFor(t, "a's slot to free", func() bool { return !h.m.attemptRunning("a") })
+ if !h.m.RetryHost("b") {
+ t.Fatal("RetryHost did not clear b")
+ }
+ waitFor(t, "b's first retry attempt", func() bool { return bTries.Load() >= 1 })
+ waitFor(t, "b's slot to free", func() bool { return !h.m.attemptRunning("b") })
+ h.advance(abandonDeferral + time.Second)
+ h.pass()
+ if tc.executed {
+ time.Sleep(100 * time.Millisecond)
+ if n := bTries.Load(); n != 1 {
+ t.Fatalf("b ran %d attempts: an attempt that may have run its build spent the retry, yet b ran again through the halt", n)
+ }
+ return
+ }
+ waitFor(t, "b's re-armed retry canary", func() bool { return bTries.Load() == 2 })
+ })
+ }
+}
diff --git a/internal/rollout/scheduler.go b/internal/rollout/scheduler.go
index 0eeda9aec..d7cfa4028 100644
--- a/internal/rollout/scheduler.go
+++ b/internal/rollout/scheduler.go
@@ -199,17 +199,35 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw
added bool
superseded []string
recErr error
+ markErr error
)
+ // A hold build that ran here and neither passed nor was held down (an
+ // attempt superseded or stopped after its exec) leaves the instance's
+ // hold-ran marker (final review wave 2, N2; spec §6.4): the build may
+ // have migrated the database, so no later wake may boot the good build
+ // in its place. Only an adoption sets it: a restart or config attempt
+ // re-runs a build the instance has already served. Set in the step that
+ // frees the slot, with the failure record when there is one (one save),
+ // so no wake planned after the slot frees misses it.
+ heldDown := transition && rec != nil && len(rec.HeldDown) > 0
+ mark := mode == activator.ModeHold && d.Class == classAdopt && res.Executed &&
+ res.Outcome != activator.SwapDone && !heldDown
if transition && rec != nil {
// The whole cohort-failure transition (spec §8.1 steps 1 and 2): the
// record (in memory at once, on disk as it can) and the halt.
- added, superseded, recErr = failCohort(m.d.Book, s.q, *rec)
+ r := *rec
+ r.MarkHoldRan = mark
+ added, superseded, recErr = failCohort(m.d.Book, s.q, r)
+ } else if mark {
+ markErr = m.d.Book.MarkHoldRan(d.ID.Host, HoldRan{Account: s.key.acct, App: s.key.app, Build: d.ID.Build})
}
if res.Outcome == activator.SwapAbandoned && !res.Executed {
- // Nothing of the build ran (a refusal before its exec, a staging
- // refusal, an unreadable row): a retried host's allowance is not
- // spent, and its promised canary still runs after the deferral
- // (final review F8).
+ // It is proven that nothing of the build ran (a refusal before its
+ // exec, a staging refusal, an unreadable row): a retried host's
+ // allowance is not spent, and its promised canary still runs after
+ // the deferral (final review F8). Executed true means the build ran
+ // or may have (final review wave 2, N4: an uncertain start, a respawn
+ // whose end the attempt could not learn), and that spends it.
s.q.rearmRetry(d)
}
h := s.q.hosts[d.ID.Host]
@@ -232,6 +250,9 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw
if recErr != nil {
m.d.Logf("rollout: write failure record %s/%s %s: %v (held in memory)", rec.Account, rec.App, rec.Build, recErr)
}
+ if markErr != nil {
+ m.d.Logf("rollout: %s: write hold-ran marker for %s: %v (held in memory)", d.ID.Host, d.ID.Build, markErr)
+ }
if len(superseded) > 0 {
m.d.Logf("rollout: %s/%s build %s failed: superseding attempts past spawn on %v", rec.Account, rec.App, rec.Build, superseded)
}
diff --git a/internal/rollout/sim_test.go b/internal/rollout/sim_test.go
index 1a54d18f1..d3541d442 100644
--- a/internal/rollout/sim_test.go
+++ b/internal/rollout/sim_test.go
@@ -42,7 +42,10 @@ import (
// has not served it, a build whose failure on this box stands. Spec
// §6.4's hold-down (a hold build that ran and failed on the instance) is
// the one refusal it allows, and spec §8.5's bootstrap rule exempts an
-// instance with no good build.
+// instance with no good build. An instance where a hold build ran
+// without passing (the hold-ran marker, final review wave 2, N2) is
+// restricted, not failed: its wakes may boot that build whatever record
+// blocks it, may be refused, and carry no fallback (I5 is its rule).
// - I2: swap_concurrency is never exceeded.
// - I3: a failure record never blocks an instance already serving that
// build. With capacity free, its restart and config attempts onto that
@@ -52,6 +55,12 @@ import (
// - I4: a swap never switches to a superseded build: at the switch, an
// adoption's build is the instance's installed build, and no failure of
// it stands against the instance.
+// - I5: once a hold build has run on an instance without passing there
+// (an adoption abandoned or failed after its exec, not held down; a hold
+// first wake ended with no verdict after its exec), no wake of that
+// instance boots its good build in place of another installed build,
+// as a recovery or a fallback, until the instance records a good build
+// or a host retry names it (spec §6.4).
//
// Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000).
// ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some
@@ -173,6 +182,10 @@ type simHost struct {
wakeStart time.Time
stamp time.Time
failLater bool // observing: the build exits inside its observation
+ // underMark: the wake's plan was made while a hold-ran restriction
+ // stood (N2), so it carries no fallback even if a retry lifts the
+ // restriction before its exec.
+ underMark bool
}
type simBuild struct {
@@ -238,6 +251,10 @@ type sim struct {
failedEver map[string]bool
exempt map[string]map[string]bool
heldDown map[string]map[string]bool
+ // holdRan is the hold-ran restriction per host: the hold build that ran
+ // there without passing, until the host records a good build (goodAt)
+ // or a host retry names it.
+ holdRan map[string]string
swaps map[string]*simSwap
events chan simEvent
@@ -250,7 +267,7 @@ func newSim(seed int64, opt string, staged map[string]bool) *sim {
byName: map[string]*simHost{}, channels: [2]string{"", "beta"},
pointers: map[string]release.ChannelPointer{}, builds: map[string]*simBuild{},
lastStamp: map[string]time.Time{}, lastSigned: map[string]time.Time{}, failedEver: map[string]bool{},
- exempt: map[string]map[string]bool{}, heldDown: map[string]map[string]bool{},
+ exempt: map[string]map[string]bool{}, heldDown: map[string]map[string]bool{}, holdRan: map[string]string{},
swaps: map[string]*simSwap{}, events: make(chan simEvent)}
s.hold = s.r.Intn(4) == 0
s.concurrency = 1 + s.r.Intn(3)
@@ -452,9 +469,23 @@ func (s *sim) finishSwap(sw *simSwap) activator.SwapResult {
if s.draining || sw.newest(a.Host) != a.SchedRev || s.r.Intn(30) == 0 {
// Superseded, or a re-check that refused (a lost lease, a stop).
res.Outcome, res.Reason = activator.SwapAbandoned, "superseded"
- s.logf("swap %s %s %s: abandoned", a.Host, a.Cause, a.Build)
+ if !s.draining && s.r.Intn(3) == 0 {
+ // After the build's exec (SwapResult.Executed). A hold stopped the
+ // old generation first, and the abandoned build is stopped too:
+ // nothing serves. Not seen to fail, so nothing is recorded, but an
+ // adoption's hold build ran against the database (N2).
+ res.Executed = true
+ if a.Mode == activator.ModeHold {
+ h.phase, h.gen = simAsleep, activator.Gen{}
+ if !sw.serves {
+ s.holdRanOn(h, a.Build)
+ }
+ }
+ }
+ s.logf("swap %s %s %s: abandoned (executed %v)", a.Host, a.Cause, a.Build, res.Executed)
return res
}
+ res.Executed = true
fresh := !h.served[a.Build]
b := s.builds[a.Build]
fails := false
@@ -477,6 +508,10 @@ func (s *sim) finishSwap(sw *simSwap) activator.SwapResult {
if fresh {
s.failure(a.Build, a.Host, sw.stamp, res.Outcome == activator.SwapDown)
}
+ if res.Outcome == activator.SwapFailed && a.Mode == activator.ModeHold && !sw.serves {
+ // A hold adoption that ran, failed and was not held down.
+ s.holdRanOn(h, a.Build)
+ }
return res
}
if a.Cause == activator.CauseRelease || a.Cause == activator.CauseRetry {
@@ -489,7 +524,8 @@ func (s *sim) finishSwap(sw *simSwap) activator.SwapResult {
}
g := s.newGen(release.StagedIdentity{Version: a.Build, StagedPath: a.StagedPath, Digest: a.Digest, Demand: a.Demand})
h.phase, h.gen = simAwake, g
- h.good, h.served[a.Build] = a.Build, true
+ s.goodAt(h, a.Build)
+ h.served[a.Build] = true
res.Outcome, res.Gen = activator.SwapDone, g
s.logf("swap %s %s %s (%s): done", a.Host, a.Cause, a.Build, a.Mode)
return res
@@ -728,6 +764,36 @@ func (s *sim) heldDownOn(host, build string) bool {
return s.heldDown[host][build]
}
+// holdRanOn is the ground truth of a hold build that ran on h without
+// passing or being held down there (N2).
+func (s *sim) holdRanOn(h *simHost, build string) {
+ s.holdRan[h.name] = build
+ s.logf("ground truth: hold build %s ran on %s without a verdict", build, h.name)
+}
+
+// goodAt is h recording build as its good build (the code's good-build
+// write), which lifts any hold-ran restriction.
+func (s *sim) goodAt(h *simHost, build string) {
+ h.good = build
+ delete(s.holdRan, h.name)
+}
+
+// checkI5 is I5 at the exec of build on h, for a wake planned against rt.
+func (s *sim) checkI5(h *simHost, rt registry.Route, build string) {
+ if mark := s.holdRan[h.name]; mark != "" && build == h.good && build != rt.Version {
+ violate("I5", "%s boots its good build %s in place of the installed %s after hold build %s ran there", h.name, build, rt.Version, mark)
+ }
+}
+
+// bootable is the plan's Check for build at its exec (planGate): nil when
+// the exec may go ahead.
+func bootable(plan activator.WakePlan, build string) error {
+ if plan.Check == nil {
+ return nil
+ }
+ return plan.Check(build)
+}
+
// wakeBegin is a wake's planning (Manager.Plan, as the activator calls it
// before the lease).
func (s *sim) wakeBegin() {
@@ -745,18 +811,33 @@ func (s *sim) wakeBegin() {
return
}
h.plan, h.planRoute, h.wakeStart, h.stamp = plan, row, s.now, s.stampFor(h, plan.Boot.Version)
+ h.underMark = s.holdRan[h.name] != ""
s.logf("wake %s planned: boot %s first %v fallback %v mode %s", h.name, plan.Boot.Version, plan.FirstWake, plan.Fallback != nil, plan.Mode)
}
func (s *sim) checkWakeRefused(h *simHost, installed, why string) {
- if h.good != "" && !s.heldDownOn(h.name, installed) {
- violate("I1", "%s has good build %s but its wake boots nothing: %s", h.name, h.good, why)
+ if h.good == "" || s.heldDownOn(h.name, installed) {
+ return
}
+ if mark := s.holdRan[h.name]; mark != "" {
+ // Restricted (N2): refused only for another installed build held for
+ // the instance, or one not staged correctly. The marked build itself
+ // boots, whatever record blocks it. "Held" is read loosely here (a
+ // failure of it was recorded at some point and no host retry exempts
+ // the instance): the book clears a record only at its next due
+ // pointer read, after the model's re-sign.
+ held := s.failedEver[installed] && !s.exempt[h.name][installed]
+ if (installed != mark && held) || s.builds[installed].misstaged {
+ return
+ }
+ violate("I1", "%s (good build %s, hold build %s ran there) has %s installed but its wake boots nothing: %s", h.name, h.good, mark, installed, why)
+ }
+ violate("I1", "%s has good build %s but its wake boots nothing: %s", h.name, h.good, why)
}
// checkBoot is I1 at the exec of build on h.
func (s *sim) checkBoot(h *simHost, build string) {
- if h.good != "" && build != h.good && !h.served[build] && s.blockedFor(h.name, build) {
+ if h.good != "" && build != h.good && !h.served[build] && s.blockedFor(h.name, build) && build != s.holdRan[h.name] {
violate("I1", "%s (good build %s) boots %s, whose failure on this box stands", h.name, h.good, build)
}
}
@@ -785,10 +866,12 @@ func (s *sim) wakeExec() {
if next.Refuse != "" || next.Boot.StagedPath == "" || (next.Check != nil && next.Check(next.Boot.Version) != nil) {
h.phase = simAsleep
s.logf("wake %s: abandoned at exec (plan stale, replan %q)", h.name, next.Refuse)
- s.checkWakeRefused(h, rt.Version, "the plan went stale and the replan refused")
+ // The replan read the row afresh (Plan replans a moved row).
+ s.checkWakeRefused(h, s.rows.get(h.name).Version, "the plan went stale and the replan refused")
return
}
plan, h.plan, h.stamp = next, next, s.stampFor(h, next.Boot.Version)
+ h.underMark = s.holdRan[h.name] != ""
s.logf("wake %s replanned: boot %s first %v", h.name, plan.Boot.Version, plan.FirstWake)
}
canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == activator.ModeOverlap
@@ -798,25 +881,51 @@ func (s *sim) wakeExec() {
// and with less than 20 s of it left at the exec, the good build
// boots untried and nothing is recorded against B.
good := *plan.Fallback
+ if err := bootable(plan, good.Version); err != nil {
+ // The good build's own gate refused it at the exec: nothing ran.
+ h.phase = simAsleep
+ s.logf("wake %s: %s untried, and the good build %s was refused at its exec: %v", h.name, plan.Boot.Version, good.Version, err)
+ s.checkWakeRefused(h, rt.Version, "the untried first wake's good build was refused at its exec")
+ return
+ }
s.checkBoot(h, good.Version)
+ s.checkI5(h, rt, good.Version)
h.phase, h.gen = simAwake, s.newGen(good)
h.served[good.Version] = true
+ s.goodAt(h, good.Version) // served: the good build is written again
s.logf("wake %s: %s untried, booted the good build %s", h.name, plan.Boot.Version, good.Version)
s.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: h.gen, Mode: plan.Mode, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev})
return
}
boot := plan.Boot
s.checkBoot(h, boot.Version)
- if h.good != "" && plan.FirstWake && !s.expectHold(boot.Version) && (plan.Fallback == nil || plan.Fallback.Version != h.good) {
+ s.checkI5(h, rt, boot.Version)
+ marked := h.underMark || s.holdRan[h.name] != ""
+ if h.good != "" && plan.FirstWake && !s.expectHold(boot.Version) && (plan.Fallback == nil || plan.Fallback.Version != h.good) && !marked {
violate("I1", "%s (good build %s) first-wakes %s in overlap mode without its good build behind it", h.name, h.good, boot.Version)
}
if !plan.FirstWake {
h.phase, h.gen = simAwake, s.newGen(boot)
- h.good, h.served[boot.Version] = boot.Version, true
+ s.goodAt(h, boot.Version)
+ h.served[boot.Version] = true
s.logf("wake %s: booted %s", h.name, boot.Version)
s.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: h.gen, Mode: plan.Mode, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev})
return
}
+ if s.r.Intn(12) == 0 {
+ // The wake's lifecycle ended it after the build's exec (a stop, a
+ // fence, a step after the spawn): no verdict. The activator reports
+ // a hold first wake's run (GenCancelled, Executed); an overlap one
+ // reports nothing, as before (N2).
+ h.phase, h.gen = simAsleep, activator.Gen{}
+ s.logf("wake %s: first wake of %s ended after its exec, no verdict (mode %s)", h.name, boot.Version, plan.Mode)
+ if plan.Mode == activator.ModeHold {
+ s.holdRanOn(h, boot.Version)
+ s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Gen: activator.Gen{Build: boot.Version}, Mode: plan.Mode,
+ FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev, Executed: true})
+ }
+ return
+ }
b := s.builds[boot.Version]
fails := b.bad || (b.flaky && s.r.Intn(2) == 0)
if fails && s.r.Intn(2) == 0 {
@@ -830,7 +939,19 @@ func (s *sim) wakeExec() {
h.phase, h.gen = simObserving, s.newGen(boot)
s.logf("wake %s: first wake of %s healthy, observing", h.name, boot.Version)
s.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: h.gen, Mode: plan.Mode, FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev})
+ case canFall && bootable(plan, plan.Fallback.Version) != nil:
+ // B failed, and its fallback's own gate refused the good build at
+ // its exec: B's failure is recorded, nothing fell back.
+ s.failure(boot.Version, h.name, h.stamp, false)
+ h.phase, h.gen = simAsleep, activator.Gen{}
+ s.logf("wake %s: first wake of %s failed, its fallback %s refused at its exec", h.name, boot.Version, plan.Fallback.Version)
+ s.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: rt, Failed: boot, Mode: plan.Mode,
+ FirstWake: true, Reason: "health check failed", At: s.now, FailedUnder: plan.FailedUnder, Token: plan.Token, ConfigRev: plan.ConfigRev})
+ if h.good != "" && s.holdRan[h.name] == "" {
+ violate("I1", "%s (good build %s) failed a first wake of %s and its fallback was refused with no hold-ran restriction", h.name, h.good, boot.Version)
+ }
case canFall:
+ s.checkI5(h, rt, plan.Fallback.Version)
s.failure(boot.Version, h.name, h.stamp, false)
h.phase, h.gen = simAwake, s.newGen(*plan.Fallback)
s.logf("wake %s: first wake of %s failed, fell back to %s", h.name, boot.Version, plan.Fallback.Version)
@@ -843,7 +964,7 @@ func (s *sim) wakeExec() {
s.logf("wake %s: first wake of %s failed, nothing fell back (mode %s)", h.name, boot.Version, plan.Mode)
s.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: rt, Failed: boot, Mode: plan.Mode,
FirstWake: true, Reason: "health check failed", At: s.now, FailedUnder: plan.FailedUnder, Token: plan.Token, ConfigRev: plan.ConfigRev})
- if h.good != "" && !hold {
+ if h.good != "" && !hold && !marked {
violate("I1", "%s (good build %s) failed a first wake of %s in overlap mode and nothing fell back", h.name, h.good, boot.Version)
}
}
@@ -865,7 +986,8 @@ func (s *sim) observeEnd() {
return
}
h.phase = simAwake
- h.good, h.served[h.gen.Build] = h.gen.Build, true
+ s.goodAt(h, h.gen.Build)
+ h.served[h.gen.Build] = true
s.logf("wake %s: %s settled", h.name, h.gen.Build)
s.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: rt, Gen: h.gen, Mode: plan.Mode, FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev})
}
@@ -888,7 +1010,13 @@ func (s *sim) sleep() {
}
s.logf("sleep %s (stopping phase %v, observation cancelled %v)", h.name, h.phase == simStopping, observing)
if observing {
- s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.planRoute, Gen: g, Mode: h.plan.Mode, FirstWake: true, At: s.now, Token: h.plan.Token, ConfigRev: h.plan.ConfigRev})
+ // The observed build was healthy, so it ran: a hold one leaves the
+ // hold-ran restriction (N2).
+ if h.plan.Mode == activator.ModeHold {
+ s.holdRanOn(h, g.Build)
+ }
+ s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.planRoute, Gen: g, Mode: h.plan.Mode, FirstWake: true, At: s.now,
+ Token: h.plan.Token, ConfigRev: h.plan.ConfigRev, Executed: true})
}
}
@@ -941,6 +1069,7 @@ func (s *sim) hostRetry() {
}
s.exempt[h.name][b] = true
delete(s.heldDown[h.name], b)
+ delete(s.holdRan, h.name)
s.logf("instances retry %s onto %s: %v", h.name, b, ok)
}
diff --git a/internal/rollout/status.go b/internal/rollout/status.go
index 775b8e0e6..909e05109 100644
--- a/internal/rollout/status.go
+++ b/internal/rollout/status.go
@@ -77,6 +77,21 @@ func (m *Manager) Status() status.RolloutReport {
}
}
}
+ // A hold build ran here without passing (final review wave 2, N2):
+ // the instance is held to it, in the same words. Asleep on a
+ // different held build, its wakes are refused rather than fall
+ // back, so it is down. Awake, it serves what it serves.
+ if mark, ok := holdRanFor(m.d.Book, s.key.acct, s.key.app, host); ok && h.serving == "" {
+ switch {
+ case hr.HeldBy != "" && hr.HeldBy != mark.Build && h.good != "":
+ hr.Attempt = status.AttemptDown
+ case hr.Attempt == "":
+ hr.HeldBy = mark.Build
+ if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok {
+ hr.Reason = reasonWord(rec.Reason)
+ }
+ }
+ }
r.Hosts[host] = hr
}
s.mu.Unlock()
diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go
index 50d315379..1bcc8ecf2 100644
--- a/internal/rollout/wakeplan.go
+++ b/internal/rollout/wakeplan.go
@@ -2,6 +2,7 @@ package rollout
import (
"context"
+ "errors"
"fmt"
"github.com/carlosframework/platform/internal/activator"
@@ -13,6 +14,18 @@ import (
type wakeBook interface {
Blocks(acct, app, build, host string, served bool) bool
HeldDown(acct, app, build, host string) bool
+ HoldRanOn(host string) (HoldRan, bool)
+}
+
+// holdRanFor is host's hold-ran marker when it is one of (acct, app)'s: a
+// marker left by another app's route under the same host name says nothing
+// about this one's database.
+func holdRanFor(b wakeBook, acct, app, host string) (HoldRan, bool) {
+ h, ok := b.HoldRanOn(host)
+ if !ok || h.Account != acct || h.App != app {
+ return HoldRan{}, false
+ }
+ return h, true
}
type wakeInputs struct {
@@ -60,6 +73,9 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) {
if in.Book.HeldDown(acct, app, installed, host) {
return activator.WakePlan{Refuse: HeldDownRefusal(installed)}, nil
}
+ if mark, ok := holdRanFor(in.Book, acct, app, host); ok {
+ return decideHoldRan(in, mark, hasGood, recovery)
+ }
if hasGood && installed == rt.GoodVersion {
return recovery, nil
}
@@ -91,6 +107,50 @@ func decideWake(in wakeInputs) (activator.WakePlan, error) {
return plan, nil
}
+// decideHoldRan is decideWake for an instance with a hold-ran marker (final
+// review wave 2, N2; spec §6.4: once a hold build has executed against an
+// instance's database, nothing falls back automatically on that instance).
+// The good build never boots in place of the installed build, as a
+// recovery or as a fallback:
+// - the installed build is the marked build: it boots, in hold mode with
+// no fallback, even when a sibling's failure record blocks it for the
+// instance's siblings, and its failure here is a hold-down;
+// - the installed build is the good build (an explicit rollback): it is
+// what was installed, not a fallback, so it boots as an ordinary wake;
+// - the installed build is another build held for the instance (a record
+// that would send the wake to the good build): the wake is refused, and
+// the instance stays down until a retry or a newer build;
+// - any other installed build boots as a first wake with no fallback.
+//
+// The hold-down refusal above this call still comes first. The refusal
+// words are HeldDownRefusal's for the marked build: no new member text.
+func decideHoldRan(in wakeInputs, mark HoldRan, hasGood bool, recovery activator.WakePlan) (activator.WakePlan, error) {
+ rt := in.Route
+ acct, app, host := release.RouteAccount(rt), rt.App, rt.Host
+ installed := rt.Version
+ switch {
+ case installed == mark.Build:
+ // Planned below, whatever record blocks it for the siblings.
+ case hasGood && installed == rt.GoodVersion:
+ return recovery, nil
+ case hasGood && in.Book.Blocks(acct, app, installed, host, false):
+ return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil
+ }
+ path := release.VersionPath(in.OptDir, acct, installed, app)
+ demand, digest, err := in.Demand(installed, path)
+ if err != nil {
+ // A staging refusal cannot boot the good build instead here, as it
+ // does for an instance with no marker: that would be the fallback.
+ return activator.WakePlan{Refuse: "build " + installed + " is not staged correctly on this box"}, err
+ }
+ mode := AttemptMode(in.Policy, demand)
+ if installed == mark.Build {
+ mode = activator.ModeHold
+ }
+ return activator.WakePlan{Boot: release.StagedIdentity{Version: installed, StagedPath: path, Digest: digest, Demand: demand},
+ Mode: mode, FirstWake: true}, nil
+}
+
// Plan is the activator's Planner (spec §8.2). It is called without
// activator.Service.mu held, before the wake's lease. It also snapshots the
// instance's delivered config revision into the plan (WakePlan.ConfigRev).
@@ -230,17 +290,39 @@ func (m *Manager) planRow(ctx context.Context, rt registry.Route, tok uint64) ac
// An instance with no good build is refused only by a hold-down: a record
// alone would not move it onto anything, and it has no fallback by design
// until it first serves (spec §8.5), so refusing it would be an outage.
+//
+// A hold-ran marker (final review wave 2, N2), set since the plan or
+// before it, applies decideHoldRan's rule at the exec: the marked build may
+// boot unless it is held down there, whatever record blocks it for the
+// siblings, and the good build may not boot in place of another installed
+// build, as a recovery, a fallback or an untried first wake's stand-in.
// The book is in memory; nothing here blocks.
func (m *Manager) planCheck(rt registry.Route, plan activator.WakePlan) func(build string) error {
acct, app, host := release.RouteAccount(rt), rt.App, rt.Host
boot := plan.Boot.Version
hasGood := rt.GoodVersion != "" && rt.GoodDigest != ""
+ // The texts reach the member's "wake failed" event (memberSafeWakeReason
+ // passes a planned wake's error through), so they are existing words.
+ held := func(build string) error {
+ return fmt.Errorf("build %s is held by a failure recorded since the wake was planned", build)
+ }
return func(build string) error {
+ if mark, ok := holdRanFor(m.d.Book, acct, app, host); ok {
+ switch {
+ case build == mark.Build:
+ if m.d.Book.HeldDown(acct, app, build, host) {
+ return held(build)
+ }
+ return nil
+ case hasGood && build == rt.GoodVersion && rt.Version != rt.GoodVersion:
+ return errors.New(HeldDownRefusal(mark.Build))
+ }
+ }
if !plan.FirstWake || build != boot {
return nil
}
if m.d.Book.HeldDown(acct, app, build, host) || (hasGood && m.d.Book.Blocks(acct, app, build, host, false)) {
- return fmt.Errorf("build %s is held by a failure recorded since the wake was planned", build)
+ return held(build)
}
return nil
}
@@ -313,9 +395,7 @@ func (m *Manager) OnGen(ev activator.GenEvent) {
}
// A first wake is published when it settles: until then the
// activator hides it (Current), so it is not yet acknowledged.
- if newest {
- m.configActivated(rt.Host, ev.ConfigRev)
- }
+ m.wakeConfigActivated(rt.Host, ev.Token, ev.ConfigRev)
case activator.GenSettled:
m.served(rt.Host, ev.Gen)
case activator.GenFellBack:
@@ -324,16 +404,47 @@ func (m *Manager) OnGen(ev activator.GenEvent) {
// failed build is held.
m.buildFailed(rt, ev)
m.publishServing(rt.Host, ev.Gen)
- if newest {
- m.configActivated(rt.Host, ev.ConfigRev)
- }
+ m.wakeConfigActivated(rt.Host, ev.Token, ev.ConfigRev)
m.emit(rt, EvFallbackServed, "warn", textFallback(ev.Failed.Version, ev.Gen.Build))
case activator.GenFailed:
m.buildFailed(rt, ev)
case activator.GenCancelled:
- // An intentional stop ended the first wake's observation (spec
- // §6.4): no verdict on the build, so nothing is recorded or published.
+ // An intentional stop ended the first wake's observation, or the
+ // wake's lifecycle ended it after its build ran (spec §6.4): no
+ // verdict on the build, so nothing is recorded or published. A hold
+ // build that ran leaves the instance's hold-ran marker (final review
+ // wave 2, N2): its database may be the build's now, so the next wake
+ // must not boot the good build. A fact about the instance, like a
+ // failure, so it is not token-scoped.
+ if ev.Executed && ev.FirstWake && ev.Mode == activator.ModeHold && ev.Gen.Build != "" {
+ mark := HoldRan{Account: release.RouteAccount(rt), App: rt.App, Build: ev.Gen.Build}
+ if err := m.d.Book.MarkHoldRan(rt.Host, mark); err != nil {
+ m.d.Logf("rollout: %s: write hold-ran marker for %s: %v (held in memory)", rt.Host, mark.Build, err)
+ }
+ }
+ }
+}
+
+// wakeConfigActivated is configActivated for a planned wake's event (final
+// review wave 2, N3): the revision activates only while tok is still the
+// host's newest wake token, and that check and the activation are one
+// critical section of m.mu. A check made earlier (OnGen's newest) can go
+// stale while the event does its other work (served's registry write): a
+// newer wake could plan and activate its own revision meanwhile, and this
+// older one would then put its revision back. The held ack is written, and
+// the queue's pending config attempt cleared, after the lock is released.
+func (m *Manager) wakeConfigActivated(host string, tok uint64, rev string) {
+ if rev == "" || tok == 0 {
+ return
}
+ m.mu.Lock()
+ if m.wakeTok[host] != tok {
+ m.mu.Unlock()
+ return
+ }
+ write, _ := m.track(host).activate(rev)
+ m.mu.Unlock()
+ m.activationDone(host, rev, write)
}
// buildFailed is a failed first wake's cohort-failure transition (spec