amadan
rollout sim: stale grants, crashes and failed saves; I9, I10
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 file changed,
+485
−15
internal/rollout/sim_test.go+485 −15
diff --git a/internal/rollout/sim_test.go b/internal/rollout/sim_test.go| index f075fa81a..e1c45d8cc 100644 |
| --- a/internal/rollout/sim_test.go |
| +++ b/internal/rollout/sim_test.go |
| @@ -2,10 +2,13 @@ package rollout |
| import ( |
| "context" |
| + "encoding/json" |
| + "errors" |
| "fmt" |
| "maps" |
| "math/rand" |
| "os" |
| + "path/filepath" |
| "runtime/debug" |
| "slices" |
| "sort" |
| @@ -94,6 +97,28 @@ import ( |
| // lands (Task 5, F5): at every failure, before the code records it, an |
| // agent opening the book from disk holds the build for the instance, |
| // and holds it down when the failure does. |
| +// - I9: a stale grant never clears a later failure (the crash-epoch hard |
| +// limit, readiness round 2): when a signed `carlos instances retry` |
| +// grant issued at step t clears anything (RetryHostAt), no unanswered |
| +// failure fact of the grant's (build, host) has a start after t. The |
| +// facts are the model's (sim.facts), never the book's verdicts: every |
| +// failure, hold-down and hold run the model delivers that changes the |
| +// model's own restriction state (sim.grec, the hold-ran restriction), |
| +// and every intent a restart reconciles. A fact is answered by a clear |
| +// the spec allows to lift it, issued after its start: a grant for its |
| +// (build, host), a pointer naming its build (a record), or a lift of |
| +// its host's hold-ran restriction (a hold run). |
| +// - I10: durable before use: every non-zero generation PublishedEpoch |
| +// returns is the app's generation in the book's last landed file, |
| +// re-read from disk at that moment, under the identity the file holds. |
| +// |
| +// I9 and I10 run on the seeds whose book is on disk (seed%4 == 3, not as |
| +// root), with I8's read-only windows, and three things only those seeds |
| +// draw (sim.gr): grants, queued and delivered later, again and again, as |
| +// an agent re-reads its pending grants; agent restarts, which drop every |
| +// execution in flight and reopen the book from disk under a new Manager; |
| +// and box rebuilds, which remove the book's file and open a new book. The |
| +// grant queue and the facts outlive both. |
| // |
| // Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000). |
| // ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some |
| @@ -162,6 +187,9 @@ type simFailure struct { |
| func runSimSeed(seed int64, opt, books string, staged map[string]bool, steps int) (out *simFailure) { |
| var s *sim |
| defer func() { |
| + if s != nil && s.cancel != nil { |
| + s.cancel() |
| + } |
| if s != nil && s.dir != "" { |
| // Writable again, so the seed's book can go (and t.TempDir's |
| // own cleanup never meets a read-only directory). |
| @@ -262,6 +290,32 @@ type simLate struct { |
| start time.Time |
| } |
| +// simGrant is a signed `carlos instances retry` grant (I9): the build and |
| +// host it names, the book and generation status published for them, and |
| +// when it was issued (the step, and its time, which a fact's start is |
| +// compared with). |
| +type simGrant struct { |
| + build, host, book string |
| + epoch uint64 |
| + step int |
| + at time.Time |
| +} |
| + |
| +// simFact is one of I9's failure facts: something the model delivered that |
| +// restricts build on host (kind says what), by an execution that began at |
| +// start. |
| +type simFact struct { |
| + build, host, kind string |
| + start time.Time |
| + marker bool // a hold run: its host's hold-ran restriction |
| +} |
| + |
| +// simRec is the model's own failure record of a build (sim.grec): the hosts |
| +// a host retry cleared, and those held down. |
| +type simRec struct { |
| + retried, down map[string]bool |
| +} |
| + |
| type simEvent struct { |
| host string |
| swap *simSwap // the attempt reached Swap and is parked |
| @@ -339,6 +393,33 @@ type sim struct { |
| fault bool |
| fr *rand.Rand |
| + // The crash-epoch hard limit (I9, I10), on the seeds whose book is on |
| + // disk. gr draws the grants, their deliveries, the agent restarts and |
| + // the box rebuilds, apart from r and fr. cancel ends the running agent's |
| + // Manager (its ack writer). |
| + gr *rand.Rand |
| + cancel context.CancelFunc |
| + // grants are the signed host retries the console issued, oldest first, |
| + // each delivered again at every delivery step (an agent re-reads its |
| + // pending grants), as the agent does until a grant expires. They |
| + // survive restarts and rebuilds. |
| + grants []simGrant |
| + // facts are I9's failure facts, and issued, signed and lifted the clears |
| + // that answer them: when each (build, host) grant was issued, each |
| + // build's pointer signed, each host's hold-ran restriction lifted. |
| + // They survive restarts and rebuilds. |
| + facts []simFact |
| + issued map[[2]string][]time.Time |
| + signed map[string][]time.Time |
| + lifted map[string][]time.Time |
| + // grec is the model's own failure record per build (I9): which hosts a |
| + // retry clears and which are held down, so a failure is a fact only when |
| + // it changes what the spec says restricts its host (plan finding 7: an |
| + // unchanged record or a pre-retry outcome changes nothing). Kept from the |
| + // model's events; a record leaving the book (a pass) and a restart's |
| + // reread of the disk are the only things it takes from the book. |
| + grec map[string]*simRec |
| + |
| // late holds an ordinary wake's healthy event (GenHealthy, not a first |
| // wake) that the activator has not yet delivered, per host: its |
| // generation serves, and its good write waits for the event (Astra R3 |
| @@ -361,9 +442,11 @@ func newSim(seed int64, opt, books string, staged map[string]bool) *sim { |
| exempt: map[string]map[string]bool{}, heldDown: map[string]map[string]bool{}, holdRan: map[string]string{}, |
| holdRanAt: map[string]time.Time{}, holdRanSet: map[string]time.Time{}, retryAt: map[string]map[string]time.Time{}, |
| clearedAt: map[string]time.Time{}, failedAt: map[string]time.Time{}, late: map[string]simLate{}, swaps: map[string]*simSwap{}, |
| + issued: map[[2]string][]time.Time{}, signed: map[string][]time.Time{}, lifted: map[string][]time.Time{}, grec: map[string]*simRec{}, |
| events: make(chan simEvent)} |
| if seed%4 == 3 && os.Geteuid() != 0 { |
| - // Root writes into a read-only directory, so it gets no faults. |
| + // Root writes into a read-only directory, so it gets no faults, and |
| + // none of I8, I9 and I10: its book stays in memory. |
| dir, err := os.MkdirTemp(books, "book") |
| if err != nil { |
| panic(simCrash{msg: "make the book's directory: " + err.Error()}) |
| @@ -372,7 +455,12 @@ func newSim(seed int64, opt, books string, staged map[string]bool) *sim { |
| if s.book, err = OpenFailureBook(dir); err != nil { |
| panic(simCrash{msg: "open the book: " + err.Error()}) |
| } |
| + // The agent saves a new book before anything runs (internal/host). |
| + if err := s.book.Flush(); err != nil { |
| + panic(simCrash{msg: "save the new book: " + err.Error()}) |
| + } |
| s.fr = rand.New(rand.NewSource(seed*7919 + 1)) |
| + s.gr = rand.New(rand.NewSource(seed*7919 + 2)) |
| } |
| s.book.now = func() time.Time { return s.now } |
| s.hold = s.r.Intn(4) == 0 |
| @@ -403,7 +491,16 @@ func newSim(seed int64, opt, books string, staged map[string]bool) *sim { |
| s.hosts = append(s.hosts, h) |
| s.byName[h.name] = h |
| } |
| - s.m = NewManager(context.Background(), Deps{Act: simAct{s}, Routes: s.rows, Book: s.book, OptDir: opt, Box: "box-1", |
| + s.startAgent() |
| + return s |
| +} |
| + |
| +// startAgent is the agent's start with s.book (internal/host): a new |
| +// Manager, Bootstrap, and its first pass. |
| +func (s *sim) startAgent() { |
| + ctx, cancel := context.WithCancel(context.Background()) |
| + s.cancel = cancel |
| + s.m = NewManager(ctx, Deps{Act: simAct{s}, Routes: s.rows, Book: s.book, OptDir: s.opt, Box: "box-1", |
| Demand: s.demand, Policy: s.policy, Pointer: s.pointer, Now: func() time.Time { return s.now }, |
| Logf: func(string, ...any) {}}) |
| // The app's scheduler, registered by hand so its goroutine never starts: |
| @@ -419,7 +516,6 @@ func newSim(seed int64, opt, books string, staged map[string]bool) *sim { |
| rs, _ := s.rows.List() |
| s.m.Bootstrap(rs) |
| s.pass() |
| - return s |
| } |
| func (s *sim) logf(format string, args ...any) { |
| @@ -455,6 +551,7 @@ func (s *sim) newGen(id release.StagedIdentity) activator.Gen { |
| func (s *sim) sign(channel, b string) { |
| s.pointers[channel] = release.ChannelPointer{Schema: 1, Account: "bab", App: "blog", Channel: channel, Version: b, PromotedAt: s.now} |
| s.lastSigned[b] = s.now |
| + s.signed[b] = append(s.signed[b], s.now) |
| } |
| // demand is release.CanonicalDemand's contract, as the harness has it. |
| @@ -507,13 +604,31 @@ func (s *sim) expectHold(build string) bool { |
| // hold build ran against the database; Task 4 fix round 1). A restriction a |
| // run leaves without a hold-down is the caller's, as for any run. |
| func (s *sim) failure(build, host string, stamp time.Time, heldDown bool, start time.Time) { |
| - if !start.IsZero() && (start.Before(s.retryAt[host][build]) || start.Before(s.clearedAt[build])) { |
| + s.failureOf(build, host, stamp, heldDown, start, false) |
| +} |
| + |
| +// failureOf is failure, and with reconciled an intent a restart turned into |
| +// a failure (OpenFailureBook): never stale, since nothing of the retry |
| +// ordering survives a restart, and always a fact, since the record it makes |
| +// carries the open time, newer than any. |
| +func (s *sim) failureOf(build, host string, stamp time.Time, heldDown bool, start time.Time, reconciled bool) { |
| + if !reconciled && s.stale(build, host, start) { |
| s.logf("ground truth: %s failed on %s in an execution started at %s, before a retry of it: stale", build, host, start.Sub(t0)) |
| if heldDown { |
| s.holdRanOn(s.byName[host], build, start) |
| } |
| return |
| } |
| + kind := "a failure" |
| + if heldDown { |
| + kind = "a hold-down" |
| + } |
| + if reconciled { |
| + kind = "an intent the restart reconciled" |
| + } |
| + if s.recFailed(build, host, heldDown) || reconciled { |
| + s.facts = append(s.facts, simFact{build: build, host: host, kind: kind, start: start}) |
| + } |
| s.checkI8(build, host, heldDown) |
| if stamp.IsZero() { |
| stamp = s.now |
| @@ -533,6 +648,50 @@ func (s *sim) failure(build, host string, stamp time.Time, heldDown bool, start |
| s.logf("ground truth: %s failed on %s (stamp %s, held down %v)", build, host, stamp.Sub(t0), heldDown) |
| } |
| +// stale reports whether an execution of build on host that started at start |
| +// began before a retry of it since (ruling 191): the host's `carlos |
| +// instances retry`, or a Pass's clear on a re-signed pointer. |
| +func (s *sim) stale(build, host string, start time.Time) bool { |
| + return !start.IsZero() && (start.Before(s.retryAt[host][build]) || start.Before(s.clearedAt[build])) |
| +} |
| + |
| +// recFailed is a failure of build on host, not stale, in the model's own |
| +// record of build (grec), and reports whether it changed what restricts |
| +// host (I9): a new record (which restricts every host), host's retry spent, |
| +// or host newly held down. A failure that changes none of these moves no |
| +// generation (rule 1 in FailureBook's doc), so it is no fact; a newer |
| +// failure stamp alone is not counted either, the permissive direction. |
| +func (s *sim) recFailed(build, host string, heldDown bool) bool { |
| + r := s.grec[build] |
| + changed := r == nil || r.retried[host] || (heldDown && !r.down[host]) |
| + if r == nil { |
| + r = &simRec{retried: map[string]bool{}, down: map[string]bool{}} |
| + s.grec[build] = r |
| + } |
| + delete(r.retried, host) |
| + if heldDown { |
| + r.down[host] = true |
| + } |
| + return changed |
| +} |
| + |
| +// answered reports whether a clear the spec allows to lift f was issued |
| +// after f's execution began (I9): a grant for its (build, host), a pointer |
| +// naming its build for a record, a lift of its host's hold-ran restriction |
| +// (a good write, a signed rollback) for a hold run. |
| +func (s *sim) answered(f simFact) bool { |
| + after := func(ts []time.Time) bool { |
| + return slices.ContainsFunc(ts, func(t time.Time) bool { return t.After(f.start) }) |
| + } |
| + if after(s.issued[[2]string{f.build, f.host}]) { |
| + return true |
| + } |
| + if f.marker { |
| + return after(s.lifted[f.host]) |
| + } |
| + return after(s.signed[f.build]) |
| +} |
| + |
| // diskFault opens or closes a window in which every save of the book fails |
| // (I8), for a seed whose book is on disk: its directory is made read-only, |
| // so WriteFileAtomic cannot create its temp file there. |
| @@ -712,6 +871,12 @@ func (s *sim) finishSwap(sw *simSwap) activator.SwapResult { |
| s.logf("swap %s %s %s (%s): %s", a.Host, a.Cause, a.Build, a.Mode, res.Outcome) |
| if fresh { |
| s.failure(a.Build, a.Host, sw.stamp, res.Outcome == activator.SwapDown, sw.started) |
| + } else if !sw.serves && !s.stale(a.Build, a.Host, sw.started) { |
| + // An adoption onto a build the instance served once, and serves no |
| + // more: not a failure the invariants count, but the code records |
| + // it (done's transition), so the model's own record follows it |
| + // (I9), as no fact. |
| + s.recFailed(a.Build, a.Host, res.Outcome == activator.SwapDown) |
| } |
| if res.Outcome == activator.SwapFailed && a.Mode == activator.ModeHold && !sw.serves { |
| // A hold adoption that ran, failed and was not held down. |
| @@ -786,6 +951,23 @@ func (s *sim) run(steps int) { |
| for s.step = 0; s.step < steps; s.step++ { |
| s.now = s.now.Add(time.Duration(1+s.r.Intn(3)) * time.Second) |
| s.diskFault() |
| + if s.gr != nil { |
| + // Before the step's own action, so an execution it starts begins |
| + // after a restart's pass and after every grant delivered here, as |
| + // the code's snapshot sees it. |
| + switch { |
| + case s.gr.Intn(400) == 0: |
| + s.agentRestart(true) |
| + case s.gr.Intn(40) == 0: |
| + s.agentRestart(false) |
| + } |
| + if s.gr.Intn(3) == 0 { |
| + s.deliverGrants() |
| + } |
| + if s.gr.Intn(4) == 0 { |
| + s.issueGrant(s.hosts[s.gr.Intn(len(s.hosts))]) |
| + } |
| + } |
| if hosts := sortedKeys(s.late); len(hosts) > 0 && s.r.Intn(12) == 0 { |
| s.deliverLate(hosts[s.r.Intn(len(hosts))]) |
| } |
| @@ -821,6 +1003,7 @@ func (s *sim) run(steps int) { |
| } |
| s.checkI2() |
| s.checkI7() |
| + s.checkI10() |
| } |
| s.draining = true |
| for _, host := range sortedKeys(s.swaps) { |
| @@ -888,6 +1071,13 @@ func (s *sim) pass() { |
| for _, r := range s.book.List() { |
| still[r.Build] = true |
| } |
| + // The model's own records (I9) leave with the book's: a pointer's clear |
| + // or a prune. |
| + for _, b := range sortedKeys(s.grec) { |
| + if !still[b] { |
| + delete(s.grec, b) |
| + } |
| + } |
| for _, b := range sortedKeys(recorded) { |
| if still[b] { |
| continue |
| @@ -935,6 +1125,7 @@ func (s *sim) pass() { |
| h.name, mark, s.holdRanAt[h.name].Sub(t0), h.good, s.installed[h.ch], p) |
| } |
| delete(s.holdRan, h.name) |
| + s.lifted[h.name] = append(s.lifted[h.name], s.now) |
| s.logf("ground truth: a signed rollback onto %s lifted %s's hold-ran restriction (%s)", h.good, h.name, mark) |
| } |
| } |
| @@ -1070,6 +1261,8 @@ func (s *sim) holdRanOn(h *simHost, build string, start time.Time) { |
| } |
| s.holdRan[h.name] = build |
| s.holdRanSet[h.name] = s.now |
| + // Every run is a new execution, which sets or renews the marker (I9). |
| + s.facts = append(s.facts, simFact{build: build, host: h.name, kind: "a hold run", start: start, marker: true}) |
| s.logf("ground truth: hold build %s ran on %s without a verdict", build, h.name) |
| } |
| @@ -1081,6 +1274,9 @@ func (s *sim) holdRanOn(h *simHost, build string, start time.Time) { |
| func (s *sim) goodAt(h *simHost, build string, start time.Time) { |
| h.good = build |
| if s.holdRanSet[h.name].Before(start) { |
| + if s.holdRan[h.name] != "" { |
| + s.lifted[h.name] = append(s.lifted[h.name], s.now) |
| + } |
| delete(s.holdRan, h.name) |
| } |
| } |
| @@ -1497,6 +1693,12 @@ func (s *sim) pointerRetry() { |
| // build's failure stands. |
| func (s *sim) hostRetry() { |
| h := s.hosts[s.r.Intn(len(s.hosts))] |
| + if s.gr != nil { |
| + // A seed whose book is on disk queues the grant (I9): delivered at |
| + // later steps, again and again (deliverGrants). |
| + s.issueGrant(h) |
| + return |
| + } |
| b := s.installed[h.ch] |
| if !s.stands(b) { |
| return |
| @@ -1510,19 +1712,11 @@ func (s *sim) hostRetry() { |
| clears := recorded && !slices.Contains(rec.Retried, h.name) |
| // The console's grant names the build and the generation the box |
| // publishes for it (Task 8; readiness round 2 Task 1): the sim reads it |
| - // just as the console would. A book that owes a save publishes none, and |
| - // clears nothing for a grant (the crash-epoch hard limit, rule 2): the |
| - // retry then changes nothing, in the code or the model. |
| + // just as the console would, and delivers it at once. This book is in |
| + // memory, so it never owes a save; the seeds whose book is on disk queue |
| + // their grants instead (issueGrant, deliverGrants). |
| epoch := s.book.PublishedEpoch("bab", "blog", b, h.name) |
| - if epoch == 0 && s.book.SaveOwed() { |
| - s.logf("instances retry %s onto %s: no generation published, the book owes a save", h.name, b) |
| - return |
| - } |
| ok := s.m.RetryHost(h.name, b, s.book.ID(), epoch) |
| - if !ok && s.book.SaveOwed() { |
| - s.logf("instances retry %s onto %s: refused, the book owes a save", h.name, b) |
| - return |
| - } |
| // The retry exempts h from the record that stands now, and from no |
| // record written after that one goes (pass): with no record there is |
| // nothing to exempt it from. |
| @@ -1547,6 +1741,282 @@ func (s *sim) hostRetry() { |
| s.logf("instances retry %s onto %s: %v", h.name, b, ok) |
| } |
| +// simGrantsKept bounds the grant queue: the oldest goes first, as a real |
| +// grant expires (24 h; the whole run is well inside that). |
| +const simGrantsKept = 64 |
| + |
| +// issueGrant is the console signing `carlos instances retry` for h, on a |
| +// seed whose book is on disk: it reads, as status does, the (book, |
| +// generation) the box publishes for a build holding h (the installed build |
| +// of h's channel, or its hold-ran marker's build), and queues a grant for |
| +// it, stamped with the step. A build the box publishes no generation for |
| +// gets no grant (the console refuses the retry). |
| +func (s *sim) issueGrant(h *simHost) { |
| + builds := []string{s.installed[h.ch]} |
| + if m, ok := s.book.HoldRanOn(h.name); ok && m.Build != builds[0] { |
| + builds = append(builds, m.Build) |
| + } |
| + b := builds[s.gr.Intn(len(builds))] |
| + epoch := s.published(b, h.name) |
| + if epoch == 0 { |
| + return |
| + } |
| + g := simGrant{build: b, host: h.name, book: s.book.ID(), epoch: epoch, step: s.step, at: s.now} |
| + s.grants = append(s.grants, g) |
| + if len(s.grants) > simGrantsKept { |
| + s.grants = s.grants[1:] |
| + } |
| + k := [2]string{b, h.name} |
| + s.issued[k] = append(s.issued[k], s.now) |
| + s.logf("grant: instances retry %s onto %s issued (book %.8s, generation %d)", h.name, b, g.book, epoch) |
| +} |
| + |
| +// deliverGrants is the agent reading its pending grants (internal/host, |
| +// host-retry.json): each one, oldest first, through RetryHost, which applies |
| +// it with RetryHostAt. A grant spent or stale clears nothing (the plan's |
| +// ruling: grants are never skipped, only refused), so every pending one is |
| +// delivered again at every read, before and after restarts and rebuilds. |
| +func (s *sim) deliverGrants() { |
| + for _, g := range slices.Clone(s.grants) { |
| + s.deliver(g) |
| + } |
| +} |
| + |
| +// deliver applies g. When it clears anything, I9 is checked first, then the |
| +// model follows what it cleared: a record that now lists the host as |
| +// retried (exempt, and the retry's place in the ordering), a marker of g's |
| +// build gone (the hold-ran restriction lifted). |
| +func (s *sim) deliver(g simGrant) { |
| + h := s.byName[g.host] |
| + rec, recorded := s.book.Lookup("bab", "blog", g.build) |
| + wasRetried := recorded && slices.Contains(rec.Retried, g.host) |
| + mk, marked := s.book.HoldRanOn(g.host) |
| + marked = marked && mk.Build == g.build |
| + if !s.m.RetryHost(g.host, g.build, g.book, g.epoch) { |
| + return |
| + } |
| + s.checkI9(g) |
| + after, still := s.book.Lookup("bab", "blog", g.build) |
| + recCleared := recorded && still && !wasRetried && slices.Contains(after.Retried, g.host) |
| + _, markStill := s.book.HoldRanOn(g.host) |
| + if s.retryAt[g.host] == nil { |
| + s.retryAt[g.host] = map[string]time.Time{} |
| + } |
| + s.retryAt[g.host][g.build] = s.now |
| + if recCleared { |
| + if s.exempt[g.host] == nil { |
| + s.exempt[g.host] = map[string]bool{} |
| + } |
| + s.exempt[g.host][g.build] = true |
| + delete(s.heldDown[g.host], g.build) |
| + if r := s.grec[g.build]; r != nil { |
| + r.retried[g.host] = true |
| + delete(r.down, g.host) |
| + } |
| + } |
| + if marked && !markStill && s.holdRan[h.name] == g.build { |
| + delete(s.holdRan, h.name) |
| + } |
| + s.logf("grant: instances retry %s onto %s from step %d (generation %d) cleared it (record %v, marker %v)", |
| + g.host, g.build, g.step, g.epoch, recCleared, marked && !markStill) |
| +} |
| + |
| +// checkI9 is I9 at a grant that has just cleared something: no failure fact |
| +// of its (build, host) began after the grant was issued, unless a clear the |
| +// spec allows has answered it since. |
| +func (s *sim) checkI9(g simGrant) { |
| + for _, f := range s.facts { |
| + if f.build != g.build || f.host != g.host || !f.start.After(g.at) || s.answered(f) { |
| + continue |
| + } |
| + violate("I9", "a grant for %s on %s issued at step %d (%s, book %.8s, generation %d) cleared it after %s of %s on %s in an execution begun at %s, which nothing has answered", |
| + g.build, g.host, g.step, g.at.Sub(t0), g.book, g.epoch, f.kind, f.build, f.host, f.start.Sub(t0)) |
| + } |
| +} |
| + |
| +// published is PublishedEpoch for build on host, as status reads it, with |
| +// I10 checked on every non-zero value. |
| +func (s *sim) published(build, host string) uint64 { |
| + e := s.book.PublishedEpoch("bab", "blog", build, host) |
| + if e != 0 && s.dir != "" { |
| + s.checkI10At(build, host, e) |
| + } |
| + return e |
| +} |
| + |
| +// checkI10 is I10 after every step, on a seed whose book is on disk: what |
| +// status would publish now for each host (its installed build, and its |
| +// hold-ran marker's) is on disk. |
| +func (s *sim) checkI10() { |
| + if s.dir == "" { |
| + return |
| + } |
| + for _, h := range s.hosts { |
| + s.published(s.installed[h.ch], h.name) |
| + if m, ok := s.book.HoldRanOn(h.name); ok && m.Build != s.installed[h.ch] { |
| + s.published(m.Build, h.name) |
| + } |
| + } |
| +} |
| + |
| +// checkI10At is I10 at one published value: the book's file, read now, is |
| +// the last save that landed, and it must hold epoch as the app's generation, |
| +// under the identity a grant for it would name. |
| +func (s *sim) checkI10At(build, host string, epoch uint64) { |
| + f, err := s.diskBook() |
| + if err != nil { |
| + violate("I10", "%s on %s published generation %d, and the book's file cannot be read: %v", build, host, epoch, err) |
| + } |
| + id := s.book.ID() |
| + if got := f.Gens[genKey("bab", "blog")]; got != epoch || f.Book != id { |
| + violate("I10", "%s on %s published generation %d under book %.8s, and the last landed file holds generation %d under book %.8s (saves failing: %v)", |
| + build, host, epoch, id, got, f.Book, s.fault) |
| + } |
| +} |
| + |
| +// diskBook reads the book's file as the agent would at its start. A missing |
| +// file is an empty book. |
| +func (s *sim) diskBook() (failureFile, error) { |
| + var f failureFile |
| + raw, err := os.ReadFile(filepath.Join(s.dir, FailureFile)) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return f, nil |
| + } |
| + if err != nil { |
| + return f, err |
| + } |
| + err = json.Unmarshal(raw, &f) |
| + return f, err |
| +} |
| + |
| +// agentRestart is the agent process ending and starting again, on a seed |
| +// whose book is on disk (I9). Everything in flight goes with it: each |
| +// attempt parked in Swap is left there for good (its goroutine never runs |
| +// again, so nothing of the old Manager touches the book), a healthy event |
| +// on its way is lost, and every instance stops, the agent's children with |
| +// it. A window of failing saves closes first: an agent that cannot save |
| +// its book at startup refuses managed starts (internal/host), so the box |
| +// runs again only once the disk takes a save. The new agent opens the book |
| +// from disk (OpenFailureBook, which reconciles the intents left there), |
| +// saves it, and starts a Manager with it. |
| +// |
| +// rebuild is a box rebuilt: the book's file is removed first, so the new |
| +// book is empty under a new identity, and the model forgets what the old |
| +// book held (forgetBook). Grants and failure facts outlive both. |
| +func (s *sim) agentRestart(rebuild bool) { |
| + s.cancel() |
| + dropped := len(s.swaps) |
| + s.swaps = map[string]*simSwap{} |
| + s.late = map[string]simLate{} |
| + for _, h := range s.hosts { |
| + h.phase, h.gen, h.plan, h.planRoute, h.failLater, h.underMark = simAsleep, activator.Gen{}, activator.WakePlan{}, registry.Route{}, false, false |
| + } |
| + if s.fault { |
| + if err := os.Chmod(s.dir, 0o700); err != nil { |
| + panic(simCrash{msg: "the book's directory: " + err.Error()}) |
| + } |
| + s.fault = false |
| + } |
| + if rebuild { |
| + if err := os.Remove(filepath.Join(s.dir, FailureFile)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| + panic(simCrash{msg: "remove the book: " + err.Error()}) |
| + } |
| + s.forgetBook() |
| + } else { |
| + s.reread() |
| + } |
| + // The retry ordering is in memory only (FailureBook.cleared): no |
| + // execution from before the restart is left for it to order. |
| + s.retryAt, s.clearedAt = map[string]map[string]time.Time{}, map[string]time.Time{} |
| + b, err := OpenFailureBook(s.dir) |
| + if err != nil { |
| + panic(simCrash{msg: "reopen the book: " + err.Error()}) |
| + } |
| + b.now = func() time.Time { return s.now } |
| + if err := b.Flush(); err != nil { |
| + panic(simCrash{msg: "save the reopened book: " + err.Error()}) |
| + } |
| + s.book = b |
| + s.logf("agent restart (box rebuilt %v): %d attempt(s) dropped, book %.8s", rebuild, dropped, b.ID()) |
| + s.startAgent() |
| +} |
| + |
| +// reread is the model's side of a restart's open (agentRestart). A crash |
| +// rolls back what the book held in memory and never saved: a clear made |
| +// while saves failed (a host retry's, a good write's marker clear) is undone |
| +// on disk, and the restriction is back. So the model rereads, as the agent |
| +// does, the restrictions on disk: each record's retried and held-down hosts |
| +// (exempt, heldDown, its own record grec) and the hold-ran markers. Nothing |
| +// is lost the other way: a failure or a marker set while saves failed has |
| +// its execution's intent on disk still (its End failed too), and the open |
| +// turns each intent into the failure it may have been, which the model |
| +// takes as a failure here (failureOf, reconciled), and as a hold run when |
| +// the intent was a hold's. |
| +func (s *sim) reread() { |
| + f, err := s.diskBook() |
| + if err != nil { |
| + panic(simCrash{msg: "read the book's file: " + err.Error()}) |
| + } |
| + s.grec = map[string]*simRec{} |
| + for _, r := range f.Records { |
| + g := &simRec{retried: map[string]bool{}, down: map[string]bool{}} |
| + s.grec[r.Build] = g |
| + for _, h := range s.hosts { |
| + delete(s.exempt[h.name], r.Build) |
| + } |
| + for _, host := range r.Retried { |
| + g.retried[host] = true |
| + if s.exempt[host] == nil { |
| + s.exempt[host] = map[string]bool{} |
| + } |
| + s.exempt[host][r.Build] = true |
| + } |
| + for _, host := range r.HeldDown { |
| + g.down[host] = true |
| + if g.retried[host] { |
| + continue |
| + } |
| + if s.heldDown[host] == nil { |
| + s.heldDown[host] = map[string]bool{} |
| + } |
| + s.heldDown[host][r.Build] = true |
| + } |
| + } |
| + for _, h := range s.hosts { |
| + m, ok := f.HoldRan[h.name] |
| + switch { |
| + case !ok: |
| + delete(s.holdRan, h.name) |
| + case s.holdRan[h.name] != m.Build: |
| + s.holdRan[h.name], s.holdRanAt[h.name], s.holdRanSet[h.name] = m.Build, m.At, s.now |
| + } |
| + } |
| + for _, host := range sortedKeys(f.Intents) { |
| + in := f.Intents[host] |
| + h := s.byName[host] |
| + hold := in.Mode == string(activator.ModeHold) |
| + s.logf("restart: %s's intent %s (%s, %s) is a failure now", host, in.ID, in.Build, in.Mode) |
| + s.failureOf(in.Build, host, s.now, hold, in.Start, true) |
| + if hold { |
| + start := in.Start |
| + if start.IsZero() { |
| + start = s.now |
| + } |
| + s.holdRanOn(h, in.Build, start) |
| + } |
| + } |
| +} |
| + |
| +// forgetBook is the model's side of a box rebuild (agentRestart): the book |
| +// is gone, and with it every failure, retry, hold-down and hold-ran |
| +// restriction the model held for it. The facts and grants stay. |
| +func (s *sim) forgetBook() { |
| + s.failedEver, s.lastStamp, s.failedAt = map[string]bool{}, map[string]time.Time{}, map[string]time.Time{} |
| + s.exempt, s.heldDown = map[string]map[string]bool{}, map[string]map[string]bool{} |
| + s.holdRan, s.holdRanAt, s.holdRanSet = map[string]string{}, map[string]time.Time{}, map[string]time.Time{} |
| + s.grec = map[string]*simRec{} |
| +} |
| + |
| // restage is root restaging a build whose staged file was wrong. |
| func (s *sim) restage() { |
| for _, b := range sortedKeys(s.builds) { |