| index e1c45d8cc..b2c9435b9 100644 |
| --- a/internal/rollout/sim_test.go |
| +++ b/internal/rollout/sim_test.go |
| @@ -113,12 +113,15 @@ import ( |
| // re-read from disk at that moment, under the identity the file holds. |
| // |
| // I9 and I10 run on the seeds whose book is on disk (seed%4 == 3, not as |
| -// root), with I8's read-only windows, and three things only those seeds |
| -// draw (sim.gr): grants, queued and delivered later, again and again, as |
| -// an agent re-reads its pending grants; agent restarts, which drop every |
| -// execution in flight and reopen the book from disk under a new Manager; |
| -// and box rebuilds, which remove the book's file and open a new book. The |
| -// grant queue and the facts outlive both. |
| +// root, where these seeds keep their book in memory and check neither), |
| +// with I8's read-only windows, and what only those seeds draw (sim.gr): |
| +// grants, queued and delivered later, again and again, as an agent |
| +// re-reads its pending grants; agent restarts, which drop every execution |
| +// in flight and reopen the book from disk under a new Manager; box |
| +// rebuilds, which remove the book's file and open a new book; and a Retry |
| +// on an instance its hold-ran marker holds, followed by a wake stopped |
| +// before it settles (rewakeArm). The grant queue and the facts outlive |
| +// restarts and rebuilds. |
| // |
| // Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000). |
| // ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some |
| @@ -419,6 +422,9 @@ type sim struct { |
| // model's events; a record leaving the book (a pass) and a restart's |
| // reread of the disk are the only things it takes from the book. |
| grec map[string]*simRec |
| + // rewake is the host rewakeArm granted a retry to, which rewakeRun wakes |
| + // and stops at the next step. |
| + rewake string |
| |
| // late holds an ordinary wake's healthy event (GenHealthy, not a first |
| // wake) that the activator has not yet delivered, per host: its |
| @@ -956,16 +962,21 @@ func (s *sim) run(steps int) { |
| // after a restart's pass and after every grant delivered here, as |
| // the code's snapshot sees it. |
| switch { |
| - case s.gr.Intn(400) == 0: |
| + case s.gr.Intn(150) == 0: |
| s.agentRestart(true) |
| case s.gr.Intn(40) == 0: |
| s.agentRestart(false) |
| } |
| + if s.rewake != "" { |
| + s.rewakeRun() |
| + } else if s.gr.Intn(6) == 0 { |
| + s.rewakeArm() |
| + } |
| if s.gr.Intn(3) == 0 { |
| s.deliverGrants() |
| } |
| if s.gr.Intn(4) == 0 { |
| - s.issueGrant(s.hosts[s.gr.Intn(len(s.hosts))]) |
| + s.retryHeld() |
| } |
| } |
| if hosts := sortedKeys(s.late); len(hosts) > 0 && s.r.Intn(12) == 0 { |
| @@ -1305,10 +1316,13 @@ func bootable(plan activator.WakePlan, build string) error { |
| // wakeBegin is a wake's planning (Manager.Plan, as the activator calls it |
| // before the lease). |
| func (s *sim) wakeBegin() { |
| - h := s.pick(func(h *simHost) bool { return h.phase == simAsleep }) |
| - if h == nil { |
| - return |
| + if h := s.pick(func(h *simHost) bool { return h.phase == simAsleep }); h != nil { |
| + s.wakeBeginOn(h) |
| } |
| +} |
| + |
| +// wakeBeginOn is wakeBegin for h, which is asleep. |
| +func (s *sim) wakeBeginOn(h *simHost) { |
| // The activator's outcome fence (awaitOutcomes): an earlier wake's |
| // events reach the agent before the host plans again. |
| if _, ok := s.late[h.name]; ok { |
| @@ -1390,10 +1404,13 @@ func (s *sim) checkPlanned(h *simHost, build string) { |
| // plan's Check, one replan if it refuses, the verdict floor, and the build's |
| // outcome reported through the real OnGen. |
| func (s *sim) wakeExec() { |
| - h := s.pick(func(h *simHost) bool { return h.phase == simWaking }) |
| - if h == nil { |
| - return |
| + if h := s.pick(func(h *simHost) bool { return h.phase == simWaking }); h != nil { |
| + s.wakeExecOn(h) |
| } |
| +} |
| + |
| +// wakeExecOn is wakeExec for h, which is waking. |
| +func (s *sim) wakeExecOn(h *simHost) { |
| rt, plan := h.planRoute, h.plan |
| if s.r.Intn(20) == 0 { |
| // The lease was refused (a previous incarnation still holds it): the |
| @@ -1629,13 +1646,17 @@ func (s *sim) observeEnd() { |
| // sleep hibernates an instance with no attempt in Swap: at once, or through a |
| // stopping phase the view leaves out. An observation it ends is cancelled. |
| func (s *sim) sleep() { |
| - h := s.pick(func(h *simHost) bool { |
| + if h := s.pick(func(h *simHost) bool { |
| _, inSwap := s.swaps[h.name] |
| return !inSwap && (h.phase == simAwake || h.phase == simObserving) |
| - }) |
| - if h == nil { |
| - return |
| + }); h != nil { |
| + s.sleepOn(h) |
| } |
| +} |
| + |
| +// sleepOn is sleep for h, which is awake or observing with no attempt in |
| +// Swap. |
| +func (s *sim) sleepOn(h *simHost) { |
| observing := h.phase == simObserving |
| g := h.gen |
| h.phase, h.gen = simAsleep, activator.Gen{} |
| @@ -1696,7 +1717,7 @@ func (s *sim) hostRetry() { |
| if s.gr != nil { |
| // A seed whose book is on disk queues the grant (I9): delivered at |
| // later steps, again and again (deliverGrants). |
| - s.issueGrant(h) |
| + s.issueGrant(h, "") |
| return |
| } |
| b := s.installed[h.ch] |
| @@ -1741,31 +1762,104 @@ func (s *sim) hostRetry() { |
| s.logf("instances retry %s onto %s: %v", h.name, b, ok) |
| } |
| |
| -// simGrantsKept bounds the grant queue: the oldest goes first, as a real |
| -// grant expires (24 h; the whole run is well inside that). |
| -const simGrantsKept = 64 |
| - |
| -// issueGrant is the console signing `carlos instances retry` for h, on a |
| -// seed whose book is on disk: it reads, as status does, the (book, |
| -// generation) the box publishes for a build holding h (the installed build |
| -// of h's channel, or its hold-ran marker's build), and queues a grant for |
| -// it, stamped with the step. A build the box publishes no generation for |
| -// gets no grant (the console refuses the retry). |
| -func (s *sim) issueGrant(h *simHost) { |
| +// heldBuilds are the builds that may hold h as status shows it: the |
| +// installed build of h's channel, and its hold-ran marker's build. |
| +func (s *sim) heldBuilds(h *simHost) []string { |
| builds := []string{s.installed[h.ch]} |
| if m, ok := s.book.HoldRanOn(h.name); ok && m.Build != builds[0] { |
| builds = append(builds, m.Build) |
| } |
| - b := builds[s.gr.Intn(len(builds))] |
| + return builds |
| +} |
| + |
| +// retryHeld is an operator pressing Retry on a held instance, on a seed |
| +// whose book is on disk: one (host, build) the box publishes a generation |
| +// for, at random, gets a grant (issueGrant). |
| +func (s *sim) retryHeld() { |
| + var held [][2]string |
| + for _, h := range s.hosts { |
| + for _, b := range s.heldBuilds(h) { |
| + if s.published(b, h.name) != 0 { |
| + held = append(held, [2]string{h.name, b}) |
| + } |
| + } |
| + } |
| + if len(held) == 0 { |
| + return |
| + } |
| + p := held[s.gr.Intn(len(held))] |
| + s.issueGrant(s.byName[p[0]], p[1]) |
| +} |
| + |
| +// rewakeArm and rewakeRun, on a seed whose book is on disk, are an operator |
| +// pressing Retry on an instance its hold-ran marker holds (asleep, the |
| +// marker's build X still installed), and the instance then woken again and |
| +// stopped before that wake settles, before the grant is read. The wake |
| +// boots X under the marker as a hold first wake (decideHoldRan): a run of |
| +// X that ends with no verdict renews the marker as a new execution's, and |
| +// a run that fails holds it down. Either is a fact of (X, host) after the |
| +// grant was issued, which must leave the grant clearing nothing (I9; M14's |
| +// witness: a renewal that moved no generation). The random schedule meets |
| +// this only rarely, as an instance is seldom woken again under its marker |
| +// before another restriction of its app moves the generation; here the two |
| +// halves are one step apart. |
| +// |
| +// rewakeArm issues the grant and remembers the host. |
| +func (s *sim) rewakeArm() { |
| + var c []*simHost |
| + for _, h := range s.hosts { |
| + m, ok := s.book.HoldRanOn(h.name) |
| + if h.phase == simAsleep && ok && m.Build == s.installed[h.ch] && s.published(m.Build, h.name) != 0 { |
| + c = append(c, h) |
| + } |
| + } |
| + if len(c) == 0 { |
| + return |
| + } |
| + h := c[s.gr.Intn(len(c))] |
| + s.issueGrant(h, s.installed[h.ch]) |
| + s.rewake = h.name |
| + s.logf("rewake %s: retry pressed under its hold-ran marker", h.name) |
| +} |
| + |
| +// rewakeRun wakes the host rewakeArm armed, if it is still asleep: its plan |
| +// and exec, then its stop while the first wake is under observation. Then |
| +// the agent reads its grants. |
| +func (s *sim) rewakeRun() { |
| + h := s.byName[s.rewake] |
| + s.rewake = "" |
| + if h.phase != simAsleep { |
| + return |
| + } |
| + s.wakeBeginOn(h) |
| + if h.phase == simWaking { |
| + s.wakeExecOn(h) |
| + } |
| + if _, inSwap := s.swaps[h.name]; h.phase == simObserving && !inSwap { |
| + s.sleepOn(h) |
| + } |
| + s.deliverGrants() |
| +} |
| + |
| +// issueGrant is the console signing `carlos instances retry` for h onto |
| +// build b ("" picks one of heldBuilds), on a seed whose book is on disk: |
| +// it reads, as status does, the (book, generation) the box publishes for |
| +// b's restriction of h, and queues a grant for it, stamped with the step. |
| +// A build the box publishes no generation for gets no grant (the console |
| +// refuses the retry). |
| +func (s *sim) issueGrant(h *simHost, b string) { |
| + if b == "" { |
| + builds := s.heldBuilds(h) |
| + b = builds[s.gr.Intn(len(builds))] |
| + } |
| epoch := s.published(b, h.name) |
| if epoch == 0 { |
| return |
| } |
| g := simGrant{build: b, host: h.name, book: s.book.ID(), epoch: epoch, step: s.step, at: s.now} |
| + // Pending for the rest of the run: a grant expires after 24 h, and a |
| + // run is minutes long. |
| s.grants = append(s.grants, g) |
| - if len(s.grants) > simGrantsKept { |
| - s.grants = s.grants[1:] |
| - } |
| k := [2]string{b, h.name} |
| s.issued[k] = append(s.issued[k], s.now) |
| s.logf("grant: instances retry %s onto %s issued (book %.8s, generation %d)", h.name, b, g.book, epoch) |
| @@ -1988,6 +2082,7 @@ func (s *sim) reread() { |
| case !ok: |
| delete(s.holdRan, h.name) |
| case s.holdRan[h.name] != m.Build: |
| + // A marker the crash put back (its clear was never saved). |
| s.holdRan[h.name], s.holdRanAt[h.name], s.holdRanSet[h.name] = m.Build, m.At, s.now |
| } |
| } |
| @@ -2005,6 +2100,15 @@ func (s *sim) reread() { |
| s.holdRanOn(h, in.Build, start) |
| } |
| } |
| + // Every restriction standing now was set before the open, and so before |
| + // anything the new agent starts (its snapshots are newer than every |
| + // marker's Seq): a good write by an execution starting in this very |
| + // step lifts it, as the code's ClearHoldRanBefore does. |
| + for _, host := range sortedKeys(s.holdRan) { |
| + if !s.holdRanSet[host].Before(s.now) { |
| + s.holdRanSet[host] = s.now.Add(-time.Nanosecond) |
| + } |
| + } |
| } |
| |
| // forgetBook is the model's side of a box rebuild (agentRestart): the book |
amadan