| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "maps" |
| 9 | "math/rand" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "runtime" |
| 13 | "runtime/debug" |
| 14 | "slices" |
| 15 | "sort" |
| 16 | "strconv" |
| 17 | "strings" |
| 18 | "testing" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/carlosframework/platform/internal/activator" |
| 22 | "github.com/carlosframework/platform/internal/appconfig" |
| 23 | "github.com/carlosframework/platform/internal/registry" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | ) |
| 26 | |
| 27 | // TestSimInvariants is plan 5's invariant simulation (spec §12 |
| 28 | // "Concurrency"). It drives the real Manager, scheduler, appQueue, |
| 29 | // FailureBook, failCohort, decideWake and planCheck through random schedules |
| 30 | // of promotions, agent passes, admissions, attempt outcomes, planned wakes and |
| 31 | // their first-wake observations, hibernation, restarts, config deliveries, |
| 32 | // `carlos promote --retry`, `carlos instances retry` and restaging. Only the |
| 33 | // activator is modelled (simAct, and the wake steps below), along with root's |
| 34 | // install (a promotion writes the rows' installed build) and the scheduler's |
| 35 | // goroutine, whose loop body (tick) runs as a step of its own. Each attempt |
| 36 | // runs the real scheduler.execute and scheduler.done on a goroutine of its |
| 37 | // own, handed off so that exactly one goroutine runs at any moment: a seed is |
| 38 | // one deterministic interleaving, and every invariant check sees a state no |
| 39 | // transition is halfway through. An ordinary wake's healthy event may reach |
| 40 | // the agent some steps after its generation serves (sim.late), and one seed |
| 41 | // in four keeps its failure book on disk, where windows of failing saves |
| 42 | // open and close (sim.fault). |
| 43 | // |
| 44 | // The invariants, checked against the model's own ground truth (what it |
| 45 | // promoted, what failed, what it retried), never against the book: |
| 46 | // |
| 47 | // - I1: a release-cause failure never leaves an instance that has a good |
| 48 | // build unable to boot one: every wake of such an instance boots |
| 49 | // something, a first wake in overlap mode carries the good build as its |
| 50 | // fallback, and no wake boots, for an instance that has a good build and |
| 51 | // has not served it, a build whose failure on this box stands. Spec |
| 52 | // §6.4's hold-down (a hold build that ran and failed on the instance) is |
| 53 | // the one refusal it allows, and spec §8.5's bootstrap rule exempts an |
| 54 | // instance with no good build. An instance where a hold build ran |
| 55 | // without passing (the hold-ran marker, final review wave 2, N2) is |
| 56 | // restricted, not failed: its wakes carry no fallback, and may be |
| 57 | // refused for a build a record holds for it (the marked build included: |
| 58 | // the marker lifts no record, Astra R3 N7) or for its installed good |
| 59 | // build until a signed rollback lifts the marker (ruling 205). I5 is the |
| 60 | // restriction's own rule. |
| 61 | // - I2: swap_concurrency is never exceeded. |
| 62 | // - I3: a failure record never blocks an instance already serving that |
| 63 | // build. With capacity free, its restart and config attempts onto that |
| 64 | // build run unless their own cohort's canary is still unsettled on |
| 65 | // another host, and an attempt by an instance that serves (or has as its |
| 66 | // good build) the build it failed on never writes or changes a record. |
| 67 | // - I4: a swap never switches to a superseded build: at the switch, an |
| 68 | // adoption's build is the instance's installed build, and no failure of |
| 69 | // it stands against the instance. |
| 70 | // - I5: once a hold build has run on an instance without passing there |
| 71 | // (an adoption abandoned or failed after its exec, not held down; a hold |
| 72 | // first wake ended with no verdict after its exec), no wake of that |
| 73 | // instance boots its good build in place of another installed build, |
| 74 | // as a recovery or a fallback, until the instance records a good build |
| 75 | // from an execution that began after the hold build ran (Astra R3 N6) |
| 76 | // or a host retry names it (spec §6.4). A Pass may also lift the |
| 77 | // restriction, but only on rollback evidence (ruling 205): the |
| 78 | // instance's installed build is its good build, and its channel's |
| 79 | // pointer names that build signed after the execution that ran the hold |
| 80 | // build started (its dispatch, or its wake's plan). A marker the Pass |
| 81 | // cleared without it is an I5 violation. (The book is read here only to |
| 82 | // see what the Pass cleared; the evidence is the model's.) When the |
| 83 | // marked build is itself the installed good build, it boots under the |
| 84 | // marked-build rule, and I1 requires its wake to boot. A good write |
| 85 | // that clears a marker the model's restriction still stands for (one |
| 86 | // set after the write's execution began) is an I5 violation at the |
| 87 | // write (goodWrite). |
| 88 | // - I6: every execution's intent (Task 5) ends with its outcome: once the |
| 89 | // run has drained, an intent stands only for a first wake still in |
| 90 | // flight, and it is that wake's. |
| 91 | // - I7: a retry is never undone by an outcome that began before it |
| 92 | // (ruling 191, plan review finding 3): a host's `carlos instances |
| 93 | // retry` stands in the record until that host fails the build again in |
| 94 | // a later execution, and a record a re-signed pointer cleared is |
| 95 | // neither written again nor its adoption cohort halted again but by a |
| 96 | // failure of an execution that began after the clear. |
| 97 | // - I8: a failure outlives an agent restart even when no save of the book |
| 98 | // lands (Task 5, F5): at every failure, before the code records it, an |
| 99 | // agent opening the book from disk holds the build for the instance, |
| 100 | // and holds it down when the failure does. |
| 101 | // - I9: a stale grant never clears a later failure (the crash-epoch hard |
| 102 | // limit, readiness round 2; fix wave 1, item 3). The model numbers every |
| 103 | // restriction transition it causes or observes, in the order they land, |
| 104 | // apart from the book (sim.tseq): each one that adds a restriction is a |
| 105 | // fact (sim.facts: a record created, or merged with a change; a |
| 106 | // hold-down; a hold-ran marker set, renewed or replacing another; an |
| 107 | // intent a restart reconciles), and each one that removes one is a |
| 108 | // clear (sim.clears: a grant's applied retry or marker clear, a record |
| 109 | // a pointer clears or a prune drops, a marker lifted or replaced). A |
| 110 | // grant captures the number when it is issued from published status. |
| 111 | // When a grant is accepted (RetryHostAt clears anything), no fact of its |
| 112 | // (build, host), nor a new record of its build (which holds every host, |
| 113 | // spec §8.4), may have landed after that capture, unless a clear of that |
| 114 | // restriction was applied after the fact. Facts are ordered by when they |
| 115 | // land, not by when their execution began: a late outcome (an execution |
| 116 | // begun before the grant whose failure lands after it, not PreRetry) and |
| 117 | // a change in the grant's own step are inside the check. Issuing a grant |
| 118 | // answers nothing; only an applied clear does. The facts are the |
| 119 | // model's, never the book's verdicts. |
| 120 | // - I10: durable before use: every non-zero generation PublishedEpoch |
| 121 | // returns is the app's generation in the book's last landed file, |
| 122 | // re-read from disk at that moment, under the identity the file holds. |
| 123 | // |
| 124 | // I9 and I10 run on the seeds whose book is on disk (seed%4 == 3, not as |
| 125 | // root, where these seeds keep their book in memory and check neither), |
| 126 | // with I8's read-only windows, and what only those seeds draw (sim.gr): |
| 127 | // grants, queued and delivered later, again and again, as an agent |
| 128 | // re-reads its pending grants; agent restarts, which drop every execution |
| 129 | // in flight and reopen the book from disk under a new Manager; box |
| 130 | // rebuilds, which remove the book's file and open a new book; and a Retry |
| 131 | // on an instance its hold-ran marker holds, followed by a wake, of that |
| 132 | // instance or of another that never served the marked build, stopped |
| 133 | // before it settles (rewakeArm). The grant queue and the facts outlive |
| 134 | // restarts and rebuilds. |
| 135 | // |
| 136 | // Seeds: ROLLOUT_SIM_SEEDS (default 200; the per-revision sweep is 3000). |
| 137 | // ROLLOUT_SIM_SEED=<n> replays one seed. The production code iterates some |
| 138 | // maps, so a replay is best effort; the action log printed with a failure is |
| 139 | // the evidence. Every violation names its invariant ("I2 violated at seed"), |
| 140 | // which is what testdata/rolloutsim/mutate.sh reads. |
| 141 | func TestSimInvariants(t *testing.T) { |
| 142 | first, last := int64(0), int64(200) |
| 143 | if v := os.Getenv("ROLLOUT_SIM_SEEDS"); v != "" { |
| 144 | n, err := strconv.ParseInt(v, 10, 64) |
| 145 | if err != nil || n < 1 { |
| 146 | t.Fatalf("ROLLOUT_SIM_SEEDS=%q", v) |
| 147 | } |
| 148 | last = n |
| 149 | } |
| 150 | if v := os.Getenv("ROLLOUT_SIM_SEED"); v != "" { |
| 151 | n, err := strconv.ParseInt(v, 10, 64) |
| 152 | if err != nil { |
| 153 | t.Fatalf("ROLLOUT_SIM_SEED=%q", v) |
| 154 | } |
| 155 | first, last = n, n+1 |
| 156 | } |
| 157 | opt, books := t.TempDir(), t.TempDir() |
| 158 | staged := map[string]bool{} |
| 159 | failed := 0 |
| 160 | for seed := first; seed < last; seed++ { |
| 161 | v := runSimSeed(seed, opt, books, staged, simSteps) |
| 162 | if v == nil { |
| 163 | continue |
| 164 | } |
| 165 | failed++ |
| 166 | t.Errorf("rolloutsim: %s\n--- last actions ---\n%s", v.line, strings.Join(v.trace, "\n")) |
| 167 | if failed >= 3 { |
| 168 | t.FailNow() |
| 169 | } |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | const simSteps = 400 |
| 174 | |
| 175 | // TestSimSeedsEndTheirGoroutines: an agent restart drops the attempts parked |
| 176 | // in Swap (agentRestart), and the seed's teardown (runSimSeed) must end their |
| 177 | // goroutines. A goroutine left parked holds its sim, so a 3000-seed sweep |
| 178 | // would keep every seed with a restart for the life of the process. |
| 179 | func TestSimSeedsEndTheirGoroutines(t *testing.T) { |
| 180 | if os.Geteuid() == 0 { |
| 181 | t.Skip("as root no seed keeps its book on disk, so no seed restarts its agent") |
| 182 | } |
| 183 | opt, books := t.TempDir(), t.TempDir() |
| 184 | staged := map[string]bool{} |
| 185 | before := runtime.NumGoroutine() |
| 186 | // The seeds whose book is on disk (seed%4 == 3) are the ones that restart. |
| 187 | for seed := int64(3); seed < 16; seed += 4 { |
| 188 | if v := runSimSeed(seed, opt, books, staged, simSteps); v != nil { |
| 189 | t.Fatalf("rolloutsim: %s", v.line) |
| 190 | } |
| 191 | } |
| 192 | // An ended Manager's own goroutines leave on their cancelled context, a |
| 193 | // moment after it. |
| 194 | deadline := time.Now().Add(5 * time.Second) |
| 195 | for runtime.NumGoroutine() > before && time.Now().Before(deadline) { |
| 196 | time.Sleep(10 * time.Millisecond) |
| 197 | } |
| 198 | if after := runtime.NumGoroutine(); after > before { |
| 199 | t.Fatalf("%d goroutines before the seeds, %d after them", before, after) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | // The activator's first-wake window, as spawnPlanned reads it: B's health |
| 204 | // window ends firstWakeBudget less firstWakeKillReserve after the wake's |
| 205 | // start, and B is exec'd only with firstWakeMinVerdict of it left. |
| 206 | const ( |
| 207 | simHealthWindow = 55*time.Second - 25*time.Second |
| 208 | simMinVerdict = 20 * time.Second |
| 209 | ) |
| 210 | |
| 211 | // simViolation is an invariant violation, panicked by violate and recovered |
| 212 | // per seed. Any other panic is a crash of the simulation or the code under |
| 213 | // test, reported apart so the mutation script never counts it as a kill. |
| 214 | type simViolation struct { |
| 215 | inv, msg string |
| 216 | } |
| 217 | |
| 218 | type simCrash struct { |
| 219 | msg string |
| 220 | stack []byte |
| 221 | } |
| 222 | |
| 223 | type simFailure struct { |
| 224 | line string |
| 225 | trace []string |
| 226 | } |
| 227 | |
| 228 | func runSimSeed(seed int64, opt, books string, staged map[string]bool, steps int) (out *simFailure) { |
| 229 | var s *sim |
| 230 | defer func() { |
| 231 | if s != nil && s.cancel != nil { |
| 232 | s.cancel() |
| 233 | } |
| 234 | if s != nil { |
| 235 | // Ends every attempt still parked in Swap (one an agent restart |
| 236 | // dropped, or one a violation left), without returning from Swap, |
| 237 | // so none of them touches the book or the model, and nothing holds |
| 238 | // the sim after the seed. |
| 239 | close(s.dead) |
| 240 | } |
| 241 | if s != nil && s.dir != "" { |
| 242 | // Writable again, so the seed's book can go (and t.TempDir's |
| 243 | // own cleanup never meets a read-only directory). |
| 244 | os.Chmod(s.dir, 0o700) |
| 245 | os.RemoveAll(s.dir) |
| 246 | } |
| 247 | r := recover() |
| 248 | if r == nil { |
| 249 | return |
| 250 | } |
| 251 | step, trace := -1, []string(nil) |
| 252 | if s != nil { |
| 253 | step, trace = s.step, s.trace |
| 254 | } |
| 255 | switch v := r.(type) { |
| 256 | case simViolation: |
| 257 | out = &simFailure{line: fmt.Sprintf("%s violated at seed %d step %d: %s", v.inv, seed, step, v.msg), trace: trace} |
| 258 | case simCrash: |
| 259 | out = &simFailure{line: fmt.Sprintf("crash at seed %d step %d: %s\n%s", seed, step, v.msg, v.stack), trace: trace} |
| 260 | default: |
| 261 | out = &simFailure{line: fmt.Sprintf("crash at seed %d step %d: %v\n%s", seed, step, r, debug.Stack()), trace: trace} |
| 262 | } |
| 263 | }() |
| 264 | s = newSim(seed, opt, books, staged) |
| 265 | s.run(steps) |
| 266 | return nil |
| 267 | } |
| 268 | |
| 269 | func violate(inv, format string, args ...any) { |
| 270 | panic(simViolation{inv: inv, msg: fmt.Sprintf(format, args...)}) |
| 271 | } |
| 272 | |
| 273 | // simPhase is an instance's state as the modelled activator holds it. |
| 274 | type simPhase int |
| 275 | |
| 276 | const ( |
| 277 | simAsleep simPhase = iota |
| 278 | simWaking // planned, not yet exec'd: owned, nothing current |
| 279 | simObserving // a first wake inside its 10 s observation: owned, Current hides it |
| 280 | simAwake // serving gen |
| 281 | simStopping // hibernating: owned, nothing current |
| 282 | ) |
| 283 | |
| 284 | type simHost struct { |
| 285 | name string |
| 286 | ch int |
| 287 | phase simPhase |
| 288 | gen activator.Gen // the generation serving (awake) or observed (observing) |
| 289 | // good is the model's good build: the last build that settled here. It |
| 290 | // is ground truth, kept apart from the row the code writes. |
| 291 | good string |
| 292 | served map[string]bool |
| 293 | // A wake in flight: its plan, the route the activator holds for it, and |
| 294 | // the FailedUnder the model expects for its build. |
| 295 | plan activator.WakePlan |
| 296 | planRoute registry.Route |
| 297 | wakeStart time.Time |
| 298 | // planAt is when the plan the wake runs was made (the first plan, or |
| 299 | // its replan at the exec): the start of its execution, which a hold-ran |
| 300 | // marker the wake leaves is stamped with (ruling 205). |
| 301 | planAt time.Time |
| 302 | stamp time.Time |
| 303 | failLater bool // observing: the build exits inside its observation |
| 304 | // underMark: the wake's plan was made while a hold-ran restriction |
| 305 | // stood (N2), so it carries no fallback even if a retry lifts the |
| 306 | // restriction before its exec. |
| 307 | underMark bool |
| 308 | } |
| 309 | |
| 310 | type simBuild struct { |
| 311 | bad bool // fails on every instance that has not served it |
| 312 | flaky bool // fails half the time on an instance that has not served it |
| 313 | holdDemand bool // the release demands hold |
| 314 | misstaged bool // the staged file is not the canonical artifact until root restages |
| 315 | } |
| 316 | |
| 317 | // simSwap is one attempt inside the modelled Swap, parked until a complete |
| 318 | // step lets it finish. |
| 319 | type simSwap struct { |
| 320 | a activator.Attempt |
| 321 | newest func(string) uint64 |
| 322 | release chan struct{} |
| 323 | stamp time.Time // the FailedUnder the model expects |
| 324 | // started is when the attempt was dispatched (scheduler.execute's |
| 325 | // d.Started): the At of a hold-ran marker it leaves (ruling 205). |
| 326 | started time.Time |
| 327 | // Set by finishSwap for the I3 record check: whether the instance |
| 328 | // serves the attempt's build, and the book's record for it beforehand. |
| 329 | serves bool |
| 330 | before string |
| 331 | } |
| 332 | |
| 333 | // simLate is a healthy event on its way (sim.late): the event, and the |
| 334 | // start of the wake's execution (its plan), which the model's good write |
| 335 | // is ordered by. |
| 336 | type simLate struct { |
| 337 | ev activator.GenEvent |
| 338 | start time.Time |
| 339 | } |
| 340 | |
| 341 | // simGrant is a signed `carlos instances retry` grant (I9): the build and |
| 342 | // host it names, the book and generation status published for them, when |
| 343 | // it was issued (the step, and its time, for messages), and the model's |
| 344 | // transition number at that moment (cap, sim.tseq), which every fact is |
| 345 | // ordered against. |
| 346 | type simGrant struct { |
| 347 | build, host, book string |
| 348 | epoch uint64 |
| 349 | step int |
| 350 | at time.Time |
| 351 | cap uint64 |
| 352 | } |
| 353 | |
| 354 | // simFact is one of I9's facts: a transition (seq, sim.tseq) that added a |
| 355 | // restriction of build on host (kind says what), from an execution that |
| 356 | // began at start (for messages only: facts are ordered by seq, when they |
| 357 | // landed). app is a new record, made by a failure on host: it holds every |
| 358 | // host of the app (spec §8.4: "the build's next failure writes a new record, |
| 359 | // which holds every host"), so the fact is every host's, not host's alone. |
| 360 | type simFact struct { |
| 361 | build, host, kind string |
| 362 | start time.Time |
| 363 | seq uint64 |
| 364 | marker bool // a hold run: its host's hold-ran restriction |
| 365 | app bool // a new record: every host of the app |
| 366 | } |
| 367 | |
| 368 | // simClear is a transition (seq) that removed a restriction, as applied: |
| 369 | // never a grant's issue, only its clear. marker is the hold-ran |
| 370 | // restriction of host (build "" is whichever build it named: a lift); |
| 371 | // otherwise build's record restriction of host, or of every host when host |
| 372 | // is "" (the record left the book: a pointer's clear or a prune). |
| 373 | type simClear struct { |
| 374 | build, host, kind string |
| 375 | seq uint64 |
| 376 | marker bool |
| 377 | } |
| 378 | |
| 379 | // simRec is the model's own failure record of a build (sim.grec): the hosts |
| 380 | // a host retry cleared, and those held down. |
| 381 | type simRec struct { |
| 382 | retried, down map[string]bool |
| 383 | } |
| 384 | |
| 385 | type simEvent struct { |
| 386 | host string |
| 387 | swap *simSwap // the attempt reached Swap and is parked |
| 388 | crash any |
| 389 | stack []byte |
| 390 | } |
| 391 | |
| 392 | type sim struct { |
| 393 | r *rand.Rand |
| 394 | seed int64 |
| 395 | step int |
| 396 | now time.Time |
| 397 | trace []string |
| 398 | |
| 399 | opt string |
| 400 | staged map[string]bool |
| 401 | m *Manager |
| 402 | sched *scheduler |
| 403 | book *FailureBook |
| 404 | rows *fakeRoutes |
| 405 | |
| 406 | hosts []*simHost |
| 407 | byName map[string]*simHost |
| 408 | channels [2]string |
| 409 | installed [2]string |
| 410 | pointers map[string]release.ChannelPointer // channel -> its signed pointer |
| 411 | builds map[string]*simBuild |
| 412 | nb, nreq int |
| 413 | hold bool // the app's swap policy is hold |
| 414 | concurrency int |
| 415 | genN uint64 |
| 416 | |
| 417 | // Ground truth. A failure of build b stands while no pointer naming b |
| 418 | // has been signed with a PromotedAt newer than b's newest failure stamp |
| 419 | // (the FailedUnder its attempt or wake read, else the failure's time): |
| 420 | // spec rev 14 §8.4. exempt is `carlos instances retry` per host until |
| 421 | // that host fails the build again; heldDown is a hold build that ran and |
| 422 | // failed on the host, until a host retry. |
| 423 | lastStamp map[string]time.Time |
| 424 | lastSigned map[string]time.Time |
| 425 | failedEver map[string]bool |
| 426 | exempt map[string]map[string]bool |
| 427 | heldDown map[string]map[string]bool |
| 428 | // holdRan is the hold-ran restriction per host: the hold build that ran |
| 429 | // there without passing, until the host records a good build (goodAt), |
| 430 | // a host retry names it, or a Pass lifts it on rollback evidence. |
| 431 | // holdRanAt is the start of the execution that set it (an attempt's |
| 432 | // dispatch, a planned wake's plan; Task 3 fix round 1), which a rollback |
| 433 | // pointer must be newer than. |
| 434 | holdRan map[string]string |
| 435 | holdRanAt map[string]time.Time |
| 436 | // holdRanSet is when the restriction was last set (the step), which a |
| 437 | // good write must come after the start of: a good build whose execution |
| 438 | // began before a hold build ran on the host never lifts the restriction |
| 439 | // (Astra R3 N6). |
| 440 | holdRanSet map[string]time.Time |
| 441 | // Retry ordering (ruling 191). retryAt is each host's latest `carlos |
| 442 | // instances retry` of a build, clearedAt the latest Pass that cleared a |
| 443 | // build's record on a `carlos promote --retry` pointer. A failure of an |
| 444 | // execution that started before either is stale: it changes nothing a |
| 445 | // retry cleared and holds nothing down. |
| 446 | retryAt map[string]map[string]time.Time |
| 447 | clearedAt map[string]time.Time |
| 448 | // failedAt is when each build last failed in an execution that began |
| 449 | // after every retry of it (a failure the retry ordering lets stand): I7 |
| 450 | // reads it against clearedAt. |
| 451 | failedAt map[string]time.Time |
| 452 | |
| 453 | // The book on disk (I8): one seed in four keeps the failure book in a |
| 454 | // directory of its own, and fault makes that directory read-only for |
| 455 | // a while, so every save fails as a full or broken disk would. fr draws |
| 456 | // the fault windows, apart from r, so the seeds that keep their book in |
| 457 | // memory run exactly the schedules they ran before I8 existed. |
| 458 | dir string |
| 459 | fault bool |
| 460 | fr *rand.Rand |
| 461 | |
| 462 | // The crash-epoch hard limit (I9, I10), on the seeds whose book is on |
| 463 | // disk. gr draws the grants, their deliveries, the agent restarts, the |
| 464 | // box rebuilds and the rewakes (rewakeArm), apart from r and fr. cancel |
| 465 | // ends the running agent's Manager (its ack writer). |
| 466 | gr *rand.Rand |
| 467 | cancel context.CancelFunc |
| 468 | // grants are the signed host retries the console issued, oldest first, |
| 469 | // each delivered again at every delivery step (an agent re-reads its |
| 470 | // pending grants), as the agent does until a grant expires. They |
| 471 | // survive restarts and rebuilds. |
| 472 | grants []simGrant |
| 473 | // tseq numbers every restriction transition the model causes or |
| 474 | // observes, in the order they land (I9), kept apart from the book: facts |
| 475 | // are the ones that added a restriction, clears the applied ones that |
| 476 | // removed one. Facts survive restarts and rebuilds. Clears do not: a |
| 477 | // crash rolls back whatever clear the book had not saved, and the model |
| 478 | // cannot tell which had landed, so after a restart no earlier clear |
| 479 | // discharges anything (the strict direction). |
| 480 | tseq uint64 |
| 481 | facts []simFact |
| 482 | clears []simClear |
| 483 | // grec is the model's own failure record per build (I9): which hosts a |
| 484 | // retry clears and which are held down, so a failure is a fact only when |
| 485 | // it changes what the spec says restricts its host (plan finding 7: an |
| 486 | // unchanged record or a pre-retry outcome changes nothing). Kept from the |
| 487 | // model's events. It takes three things from the book: a record leaving |
| 488 | // it (a pass), a restart's reread of the disk, and a grant's clear, where |
| 489 | // deliver marks the host retried once the book's Lookup shows the record |
| 490 | // now lists it (a clear I9 has vetted first). |
| 491 | grec map[string]*simRec |
| 492 | // rewake is the host rewakeRun wakes and stops at the next step: the one |
| 493 | // rewakeArm granted a retry to, or another on its channel. |
| 494 | rewake string |
| 495 | // rewakeFail is the build whose first wake on rewake fails its health |
| 496 | // check (another instance's wake, M17's witness), or "". |
| 497 | rewakeFail string |
| 498 | |
| 499 | // late holds an ordinary wake's healthy event (GenHealthy, not a first |
| 500 | // wake) that the activator has not yet delivered, per host: its |
| 501 | // generation serves, and its good write waits for the event (Astra R3 |
| 502 | // N6). Such a wake has no intent, so an adoption may run on the host |
| 503 | // meanwhile; a hold one that fails leaves a marker the late write must |
| 504 | // not clear (I5). |
| 505 | late map[string]simLate |
| 506 | |
| 507 | swaps map[string]*simSwap |
| 508 | events chan simEvent |
| 509 | draining bool |
| 510 | // dead is closed when the seed ends (runSimSeed): an attempt goroutine |
| 511 | // still parked in Swap, or about to report on events, exits instead. |
| 512 | dead chan struct{} |
| 513 | } |
| 514 | |
| 515 | func newSim(seed int64, opt, books string, staged map[string]bool) *sim { |
| 516 | s := &sim{r: rand.New(rand.NewSource(seed)), seed: seed, now: t0, opt: opt, staged: staged, |
| 517 | book: &FailureBook{}, rows: &fakeRoutes{rows: map[string]registry.Route{}}, |
| 518 | byName: map[string]*simHost{}, channels: [2]string{"", "beta"}, |
| 519 | pointers: map[string]release.ChannelPointer{}, builds: map[string]*simBuild{}, |
| 520 | lastStamp: map[string]time.Time{}, lastSigned: map[string]time.Time{}, failedEver: map[string]bool{}, |
| 521 | exempt: map[string]map[string]bool{}, heldDown: map[string]map[string]bool{}, holdRan: map[string]string{}, |
| 522 | holdRanAt: map[string]time.Time{}, holdRanSet: map[string]time.Time{}, retryAt: map[string]map[string]time.Time{}, |
| 523 | clearedAt: map[string]time.Time{}, failedAt: map[string]time.Time{}, late: map[string]simLate{}, swaps: map[string]*simSwap{}, |
| 524 | grec: map[string]*simRec{}, |
| 525 | events: make(chan simEvent), dead: make(chan struct{})} |
| 526 | if seed%4 == 3 && os.Geteuid() != 0 { |
| 527 | // Root writes into a read-only directory, so it gets no faults, and |
| 528 | // none of I8, I9 and I10: its book stays in memory. |
| 529 | dir, err := os.MkdirTemp(books, "book") |
| 530 | if err != nil { |
| 531 | panic(simCrash{msg: "make the book's directory: " + err.Error()}) |
| 532 | } |
| 533 | s.dir = dir |
| 534 | if s.book, err = OpenFailureBook(dir); err != nil { |
| 535 | panic(simCrash{msg: "open the book: " + err.Error()}) |
| 536 | } |
| 537 | // The agent saves a new book before anything runs (internal/host). |
| 538 | if err := s.book.Flush(); err != nil { |
| 539 | panic(simCrash{msg: "save the new book: " + err.Error()}) |
| 540 | } |
| 541 | s.fr = rand.New(rand.NewSource(seed*7919 + 1)) |
| 542 | s.gr = rand.New(rand.NewSource(seed*7919 + 2)) |
| 543 | } |
| 544 | s.book.now = func() time.Time { return s.now } |
| 545 | s.hold = s.r.Intn(4) == 0 |
| 546 | s.concurrency = 1 + s.r.Intn(3) |
| 547 | s.addBuild("b0", &simBuild{}) |
| 548 | s.installed = [2]string{"b0", "b0"} |
| 549 | for _, ch := range s.channels { |
| 550 | s.sign(ch, "b0") |
| 551 | } |
| 552 | n := 3 + s.r.Intn(4) |
| 553 | for i := range n { |
| 554 | h := &simHost{name: "h" + strconv.Itoa(i), ch: i % 2, served: map[string]bool{}} |
| 555 | row := registry.Route{Host: h.name, Kind: "instance", App: "blog", Account: "bab", Status: "ready", |
| 556 | Hibernate: true, Version: "b0", Channel: s.channels[h.ch]} |
| 557 | switch { |
| 558 | case s.r.Intn(2) == 0: |
| 559 | // Awake: Bootstrap records the build it serves as good (spec §8.5). |
| 560 | h.phase, h.gen = simAwake, s.newGen(s.identity("b0")) |
| 561 | h.good, h.served["b0"] = "b0", true |
| 562 | case s.r.Intn(2) == 0: |
| 563 | id := s.identity("b0") |
| 564 | row.GoodVersion, row.GoodDigest, row.GoodDemand = id.Version, id.Digest, id.Demand |
| 565 | h.good, h.served["b0"] = "b0", true |
| 566 | default: |
| 567 | // Asleep with no good build: the upgrade's bootstrap case. |
| 568 | } |
| 569 | s.rows.put(row) |
| 570 | s.hosts = append(s.hosts, h) |
| 571 | s.byName[h.name] = h |
| 572 | } |
| 573 | s.startAgent() |
| 574 | return s |
| 575 | } |
| 576 | |
| 577 | // startAgent is the agent's start with s.book (internal/host): a new |
| 578 | // Manager, Bootstrap, and its first pass. |
| 579 | func (s *sim) startAgent() { |
| 580 | ctx, cancel := context.WithCancel(context.Background()) |
| 581 | s.cancel = cancel |
| 582 | s.m = NewManager(ctx, Deps{Act: simAct{s}, Routes: s.rows, Book: s.book, OptDir: s.opt, Box: "box-1", |
| 583 | Demand: s.demand, Policy: s.policy, Pointer: s.pointer, Now: func() time.Time { return s.now }, |
| 584 | Logf: func(string, ...any) {}}) |
| 585 | // The app's scheduler, registered by hand so its goroutine never starts: |
| 586 | // tick is its loop body, run as a step. The queue is built as |
| 587 | // Manager.scheduler builds it. |
| 588 | key := appKey{"bab", "blog"} |
| 589 | book := s.book |
| 590 | s.sched = &scheduler{m: s.m, key: key, kick: make(chan struct{}, 1), |
| 591 | q: newAppQueue(key.acct, key.app, func(build, host string, served bool) bool { |
| 592 | return book.Blocks(key.acct, key.app, build, host, served) |
| 593 | })} |
| 594 | s.m.apps[key] = s.sched |
| 595 | rs, _ := s.rows.List() |
| 596 | s.m.Bootstrap(rs) |
| 597 | s.pass() |
| 598 | } |
| 599 | |
| 600 | func (s *sim) logf(format string, args ...any) { |
| 601 | if len(s.trace) >= 60 { |
| 602 | s.trace = s.trace[1:] |
| 603 | } |
| 604 | s.trace = append(s.trace, fmt.Sprintf("%4d %s ", s.step, s.now.Sub(t0))+fmt.Sprintf(format, args...)) |
| 605 | } |
| 606 | |
| 607 | func (s *sim) addBuild(b string, sb *simBuild) { |
| 608 | s.builds[b] = sb |
| 609 | if !s.staged[b] { |
| 610 | if err := release.StageArtifact(s.opt, "bab", b, "blog", []byte(b)); err != nil { |
| 611 | panic(simCrash{msg: "stage " + b + ": " + err.Error()}) |
| 612 | } |
| 613 | s.staged[b] = true |
| 614 | } |
| 615 | } |
| 616 | |
| 617 | func (s *sim) identity(b string) release.StagedIdentity { |
| 618 | id := release.StagedIdentity{Version: b, StagedPath: release.VersionPath(s.opt, "bab", b, "blog"), Digest: "d-" + b} |
| 619 | if s.builds[b].holdDemand { |
| 620 | id.Demand = release.DemandHold |
| 621 | } |
| 622 | return id |
| 623 | } |
| 624 | |
| 625 | func (s *sim) newGen(id release.StagedIdentity) activator.Gen { |
| 626 | s.genN++ |
| 627 | return activator.Gen{N: s.genN, Build: id.Version, StagedPath: id.StagedPath, Digest: id.Digest, Demand: id.Demand} |
| 628 | } |
| 629 | |
| 630 | func (s *sim) sign(channel, b string) { |
| 631 | s.pointers[channel] = release.ChannelPointer{Schema: 1, Account: "bab", App: "blog", Channel: channel, Version: b, PromotedAt: s.now} |
| 632 | s.lastSigned[b] = s.now |
| 633 | } |
| 634 | |
| 635 | // demand is release.CanonicalDemand's contract, as the harness has it. |
| 636 | func (s *sim) demand(_ context.Context, _, _, ver, _ string) (string, string, error) { |
| 637 | b := s.builds[ver] |
| 638 | if b == nil || b.misstaged { |
| 639 | return "", "", fmt.Errorf("%w: the staged executable's digest differs from the canonical manifest", release.ErrStagedDigestMismatch) |
| 640 | } |
| 641 | if b.holdDemand { |
| 642 | return release.DemandHold, "d-" + ver, nil |
| 643 | } |
| 644 | return "", "d-" + ver, nil |
| 645 | } |
| 646 | |
| 647 | func (s *sim) policy(context.Context, string, string) (*appconfig.Manifest, error) { |
| 648 | swap := appconfig.SwapOverlap |
| 649 | if s.hold { |
| 650 | swap = appconfig.SwapHold |
| 651 | } |
| 652 | return &appconfig.Manifest{Schema: 1, App: "blog", Swap: swap, SwapConcurrency: s.concurrency}, nil |
| 653 | } |
| 654 | |
| 655 | func (s *sim) pointer(_ context.Context, _, _, channel string) (*release.ChannelPointer, error) { |
| 656 | p, ok := s.pointers[channel] |
| 657 | if !ok { |
| 658 | return nil, nil |
| 659 | } |
| 660 | return &p, nil |
| 661 | } |
| 662 | |
| 663 | // stampFor is the FailedUnder the model expects for build on host, read when |
| 664 | // its attempt or wake starts: the PromotedAt of the host's channel pointer |
| 665 | // when it names build, else zero (the failure's own time stands in). |
| 666 | func (s *sim) stampFor(h *simHost, build string) time.Time { |
| 667 | if p, ok := s.pointers[s.channels[h.ch]]; ok && p.Version == build { |
| 668 | return p.PromotedAt |
| 669 | } |
| 670 | return time.Time{} |
| 671 | } |
| 672 | |
| 673 | func (s *sim) expectHold(build string) bool { |
| 674 | return s.hold || s.builds[build].holdDemand |
| 675 | } |
| 676 | |
| 677 | // failure is the ground truth of a build that ran and failed on host, in the |
| 678 | // execution that started at start. One that started before a retry of the |
| 679 | // build since (the host's, or a Pass's clear on a re-signed pointer) is |
| 680 | // stale (ruling 191): it stands for nothing the retry cleared, and a |
| 681 | // hold-down it would have made leaves the hold-ran restriction instead (the |
| 682 | // hold build ran against the database; Task 4 fix round 1). A restriction a |
| 683 | // run leaves without a hold-down is the caller's, as for any run. |
| 684 | func (s *sim) failure(build, host string, stamp time.Time, heldDown bool, start time.Time) { |
| 685 | s.failureOf(build, host, stamp, heldDown, start, false) |
| 686 | } |
| 687 | |
| 688 | // failureOf is failure, and with reconciled an intent a restart turned into |
| 689 | // a failure (OpenFailureBook): never stale, since nothing of the retry |
| 690 | // ordering survives a restart, and always a fact, since the record it makes |
| 691 | // carries the open time, newer than any. |
| 692 | func (s *sim) failureOf(build, host string, stamp time.Time, heldDown bool, start time.Time, reconciled bool) { |
| 693 | if !reconciled && s.stale(build, host, start) { |
| 694 | s.logf("ground truth: %s failed on %s in an execution started at %s, before a retry of it: stale", build, host, start.Sub(t0)) |
| 695 | if heldDown { |
| 696 | s.holdRanOn(s.byName[host], build, start) |
| 697 | } |
| 698 | return |
| 699 | } |
| 700 | kind := "a failure" |
| 701 | if heldDown { |
| 702 | kind = "a hold-down" |
| 703 | } |
| 704 | if reconciled { |
| 705 | kind = "an intent the restart reconciled" |
| 706 | } |
| 707 | switch changed, created := s.recFailed(build, host, heldDown); { |
| 708 | case created: |
| 709 | s.addFact(simFact{build: build, host: host, kind: kind, start: start, app: true}) |
| 710 | case changed || reconciled: |
| 711 | s.addFact(simFact{build: build, host: host, kind: kind, start: start}) |
| 712 | } |
| 713 | s.checkI8(build, host, heldDown) |
| 714 | if stamp.IsZero() { |
| 715 | stamp = s.now |
| 716 | } |
| 717 | s.failedEver[build] = true |
| 718 | s.failedAt[build] = s.now |
| 719 | if stamp.After(s.lastStamp[build]) { |
| 720 | s.lastStamp[build] = stamp |
| 721 | } |
| 722 | delete(s.exempt[host], build) |
| 723 | if heldDown { |
| 724 | if s.heldDown[host] == nil { |
| 725 | s.heldDown[host] = map[string]bool{} |
| 726 | } |
| 727 | s.heldDown[host][build] = true |
| 728 | } |
| 729 | s.logf("ground truth: %s failed on %s (stamp %s, held down %v)", build, host, stamp.Sub(t0), heldDown) |
| 730 | } |
| 731 | |
| 732 | // stale reports whether an execution of build on host that started at start |
| 733 | // began before a retry of it since (ruling 191): the host's `carlos |
| 734 | // instances retry`, or a Pass's clear on a re-signed pointer. |
| 735 | func (s *sim) stale(build, host string, start time.Time) bool { |
| 736 | return !start.IsZero() && (start.Before(s.retryAt[host][build]) || start.Before(s.clearedAt[build])) |
| 737 | } |
| 738 | |
| 739 | // recFailed is a failure of build on host, not stale, in the model's own |
| 740 | // record of build (grec). It reports whether the failure made a new record |
| 741 | // (created), which restricts every host of the app, and otherwise whether it |
| 742 | // changed what restricts host (changed): host's retry spent, or host newly |
| 743 | // held down (I9). A failure that changes none of these moves no generation |
| 744 | // (rule 1 in FailureBook's doc), so it is no fact; a newer failure stamp |
| 745 | // alone is not counted either, the permissive direction. |
| 746 | func (s *sim) recFailed(build, host string, heldDown bool) (changed, created bool) { |
| 747 | r := s.grec[build] |
| 748 | if r == nil { |
| 749 | r = &simRec{retried: map[string]bool{}, down: map[string]bool{}} |
| 750 | s.grec[build] = r |
| 751 | created = true |
| 752 | } else { |
| 753 | changed = r.retried[host] || (heldDown && !r.down[host]) |
| 754 | } |
| 755 | delete(r.retried, host) |
| 756 | if heldDown { |
| 757 | r.down[host] = true |
| 758 | } |
| 759 | return changed, created |
| 760 | } |
| 761 | |
| 762 | // addFact files f, a transition that added a restriction (I9), at the |
| 763 | // model's next transition number. |
| 764 | func (s *sim) addFact(f simFact) { |
| 765 | s.tseq++ |
| 766 | f.seq = s.tseq |
| 767 | s.facts = append(s.facts, f) |
| 768 | } |
| 769 | |
| 770 | // addClear files c, an applied transition that removed a restriction (I9), |
| 771 | // at the model's next transition number. |
| 772 | func (s *sim) addClear(c simClear) { |
| 773 | s.tseq++ |
| 774 | c.seq = s.tseq |
| 775 | s.clears = append(s.clears, c) |
| 776 | s.logf("transition %d: %s cleared (build %q, host %q, marker %v)", c.seq, c.kind, c.build, c.host, c.marker) |
| 777 | } |
| 778 | |
| 779 | // discharged reports whether a clear applied after f landed removed f's |
| 780 | // restriction of host (I9): for a hold run, a lift or replacement of its |
| 781 | // host's hold-ran marker; for a record fact, a grant's retry of its build |
| 782 | // on host, or its build's record leaving the book. host is f's own host |
| 783 | // but for a new record (f.app), which holds every host: there it is the |
| 784 | // host of the grant I9 checks. A grant's issue never discharges anything. |
| 785 | func (s *sim) discharged(f simFact, host string) bool { |
| 786 | return slices.ContainsFunc(s.clears, func(c simClear) bool { |
| 787 | return c.seq > f.seq && c.marker == f.marker && (c.build == "" || c.build == f.build) && (c.host == "" || c.host == host) |
| 788 | }) |
| 789 | } |
| 790 | |
| 791 | // diskFault opens or closes a window in which every save of the book fails |
| 792 | // (I8), for a seed whose book is on disk: its directory is made read-only, |
| 793 | // so WriteFileAtomic cannot create its temp file there. |
| 794 | func (s *sim) diskFault() { |
| 795 | if s.dir == "" { |
| 796 | return |
| 797 | } |
| 798 | switch { |
| 799 | case !s.fault && s.fr.Intn(10) == 0: |
| 800 | s.fault = true |
| 801 | case s.fault && s.fr.Intn(4) == 0: |
| 802 | s.fault = false |
| 803 | default: |
| 804 | return |
| 805 | } |
| 806 | mode := os.FileMode(0o700) |
| 807 | if s.fault { |
| 808 | mode = 0o500 |
| 809 | } |
| 810 | if err := os.Chmod(s.dir, mode); err != nil { |
| 811 | panic(simCrash{msg: "the book's directory: " + err.Error()}) |
| 812 | } |
| 813 | s.logf("disk: saves of the failure book now fail: %v", s.fault) |
| 814 | } |
| 815 | |
| 816 | // checkI8 is I8 at a failure of build on host, called before the code has |
| 817 | // written it: an agent restarting now must still find it. The restart is |
| 818 | // simulated by a second book opened from the disk as the agent opens it at |
| 819 | // startup (OpenFailureBook, which turns intents into records). It must hold |
| 820 | // build for host, and hold host down when the failure does. Only the |
| 821 | // execution's intent, saved before the build ran (Task 5, F5), can carry |
| 822 | // that across a window in which every save fails. |
| 823 | func (s *sim) checkI8(build, host string, heldDown bool) { |
| 824 | if s.dir == "" { |
| 825 | return |
| 826 | } |
| 827 | b, err := OpenFailureBook(s.dir) |
| 828 | if err != nil { |
| 829 | panic(simCrash{msg: "reopen the book: " + err.Error()}) |
| 830 | } |
| 831 | if !b.Blocks("bab", "blog", build, host, false) || (heldDown && !b.HeldDown("bab", "blog", build, host)) { |
| 832 | violate("I8", "%s failed on %s (held down %v), and an agent restarting now would not hold it there (saves failing: %v)", build, host, heldDown, s.fault) |
| 833 | } |
| 834 | } |
| 835 | |
| 836 | // checkI7 is I7 after every step, against the model's retries: a failure of |
| 837 | // an execution that began before a retry never takes the retry back. |
| 838 | // - A `carlos instances retry` of host for build b (exempt) stands until |
| 839 | // host fails b again in an execution that began after it (failure |
| 840 | // deletes exempt) or b's record goes: the record still lists host as |
| 841 | // retried. |
| 842 | // - A record a re-signed pointer cleared (clearedAt) is written again, and |
| 843 | // b's adoption cohort is halted again, only by a failure of an execution |
| 844 | // that began after the clear (failedAt). A stale one's halt is caught |
| 845 | // here before the next Pass's liftClearedHalts could hide it. |
| 846 | func (s *sim) checkI7() { |
| 847 | for _, b := range sortedKeys(s.clearedAt) { |
| 848 | if !s.failedAt[b].Before(s.clearedAt[b]) { |
| 849 | continue |
| 850 | } |
| 851 | if r, ok := s.book.Lookup("bab", "blog", b); ok { |
| 852 | violate("I7", "%s's record, cleared by a re-signed pointer at %s, stands again with no failure begun after the clear: %+v", |
| 853 | b, s.clearedAt[b].Sub(t0), r) |
| 854 | } |
| 855 | s.sched.mu.Lock() |
| 856 | c := s.sched.q.cohorts[cohortKey(b, classAdopt)] |
| 857 | halted := c != nil && c.halted |
| 858 | s.sched.mu.Unlock() |
| 859 | if halted { |
| 860 | violate("I7", "%s's cohort, reopened by a re-signed pointer at %s, is halted again with no failure begun after the clear", b, s.clearedAt[b].Sub(t0)) |
| 861 | } |
| 862 | } |
| 863 | for _, host := range sortedKeys(s.exempt) { |
| 864 | for _, b := range sortedKeys(s.exempt[host]) { |
| 865 | if r, ok := s.book.Lookup("bab", "blog", b); ok && !slices.Contains(r.Retried, host) { |
| 866 | violate("I7", "%s's retry of %s was taken back with no failure there begun after it: %+v", host, b, r) |
| 867 | } |
| 868 | } |
| 869 | } |
| 870 | } |
| 871 | |
| 872 | func (s *sim) stands(build string) bool { |
| 873 | return s.failedEver[build] && !s.lastSigned[build].After(s.lastStamp[build]) |
| 874 | } |
| 875 | |
| 876 | func (s *sim) blockedFor(host, build string) bool { |
| 877 | return s.stands(build) && !s.exempt[host][build] |
| 878 | } |
| 879 | |
| 880 | // snap is the book's record for build, as text, for the I3 record check. |
| 881 | func (s *sim) snap(build string) string { |
| 882 | r, ok := s.book.Lookup("bab", "blog", build) |
| 883 | return fmt.Sprintf("%v %+v", ok, r) |
| 884 | } |
| 885 | |
| 886 | // simAct is the modelled activator. Only one goroutine runs at a time (the |
| 887 | // handoffs in launch and complete), so it reads the model without a lock. |
| 888 | type simAct struct{ s *sim } |
| 889 | |
| 890 | func (f simAct) Current(host string) (activator.Gen, bool) { |
| 891 | if h := f.s.byName[host]; h != nil && h.phase == simAwake { |
| 892 | return h.gen, true |
| 893 | } |
| 894 | return activator.Gen{}, false |
| 895 | } |
| 896 | |
| 897 | func (f simAct) Owns(host string) bool { |
| 898 | h := f.s.byName[host] |
| 899 | return h != nil && h.phase != simAsleep |
| 900 | } |
| 901 | |
| 902 | // Swap parks the attempt until a complete step releases it. When the seed |
| 903 | // ends first (sim.dead), the goroutine exits from here: Swap never returns, |
| 904 | // so done() never runs and the attempt's Manager never touches the book. |
| 905 | func (f simAct) Swap(_ context.Context, a activator.Attempt, newest func(string) uint64) activator.SwapResult { |
| 906 | sw := &simSwap{a: a, newest: newest, release: make(chan struct{})} |
| 907 | select { |
| 908 | case f.s.events <- simEvent{host: a.Host, swap: sw}: |
| 909 | case <-f.s.dead: |
| 910 | runtime.Goexit() |
| 911 | } |
| 912 | select { |
| 913 | case <-sw.release: |
| 914 | case <-f.s.dead: |
| 915 | runtime.Goexit() |
| 916 | } |
| 917 | return f.s.finishSwap(sw) |
| 918 | } |
| 919 | |
| 920 | func (f simAct) ReplaceWorker(context.Context, activator.Attempt, *release.StagedIdentity, func(string) uint64) activator.SwapResult { |
| 921 | panic(simCrash{msg: "the simulation runs no workers"}) |
| 922 | } |
| 923 | |
| 924 | func (f simAct) PublishHeldServing(string) {} |
| 925 | |
| 926 | // EffectiveMode: the simulation has no foreign-layout rows. |
| 927 | func (f simAct) EffectiveMode(_ string, m activator.SwapMode) activator.SwapMode { return m } |
| 928 | |
| 929 | // finishSwap is the modelled Swap's end (plan 3), run on the attempt's |
| 930 | // goroutine while the simulation waits: the re-check against newest, then |
| 931 | // the build's health, then the switch. It is one transition: nothing else |
| 932 | // runs until done() has returned. |
| 933 | func (s *sim) finishSwap(sw *simSwap) activator.SwapResult { |
| 934 | a := sw.a |
| 935 | h := s.byName[a.Host] |
| 936 | sw.serves = h.good == a.Build || (h.phase == simAwake && h.gen.Build == a.Build) |
| 937 | sw.before = s.snap(a.Build) |
| 938 | res := activator.SwapResult{Mode: a.Mode} |
| 939 | if s.draining || sw.newest(a.Host) != a.SchedRev || s.r.Intn(30) == 0 { |
| 940 | // Superseded, or a re-check that refused (a lost lease, a stop). |
| 941 | res.Outcome, res.Reason = activator.SwapAbandoned, "superseded" |
| 942 | if !s.draining && s.r.Intn(3) == 0 { |
| 943 | // After the build's exec (SwapResult.Executed). A hold stopped the |
| 944 | // old generation first, and the abandoned build is stopped too: |
| 945 | // nothing serves. Not seen to fail, so nothing is recorded, but an |
| 946 | // adoption's hold build ran against the database (N2). |
| 947 | res.Executed = true |
| 948 | if a.Mode == activator.ModeHold { |
| 949 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 950 | if !sw.serves { |
| 951 | s.holdRanOn(h, a.Build, sw.started) |
| 952 | } |
| 953 | } |
| 954 | } |
| 955 | s.logf("swap %s %s %s: abandoned (executed %v)", a.Host, a.Cause, a.Build, res.Executed) |
| 956 | return res |
| 957 | } |
| 958 | res.Executed = true |
| 959 | fresh := !h.served[a.Build] |
| 960 | b := s.builds[a.Build] |
| 961 | fails := false |
| 962 | switch { |
| 963 | case fresh: |
| 964 | fails = b.bad || (b.flaky && s.r.Intn(2) == 0) |
| 965 | case h.phase == simAwake && a.Build == h.gen.Build: |
| 966 | fails = s.r.Intn(8) == 0 // a restart or config attempt that fails |
| 967 | } |
| 968 | if fails { |
| 969 | res.Reason = "health check failed" |
| 970 | if a.Mode == activator.ModeHold && s.r.Intn(4) != 0 { |
| 971 | // The old generation was stopped first: nothing serves. |
| 972 | res.Outcome = activator.SwapDown |
| 973 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 974 | } else { |
| 975 | res.Outcome = activator.SwapFailed |
| 976 | } |
| 977 | s.logf("swap %s %s %s (%s): %s", a.Host, a.Cause, a.Build, a.Mode, res.Outcome) |
| 978 | if fresh { |
| 979 | s.failure(a.Build, a.Host, sw.stamp, res.Outcome == activator.SwapDown, sw.started) |
| 980 | } else if !sw.serves && !s.stale(a.Build, a.Host, sw.started) { |
| 981 | // An adoption onto a build the instance served once, and serves no |
| 982 | // more: not a failure the invariants count, but the code records |
| 983 | // it (done's transition), so the model's own record follows it |
| 984 | // (I9). It is no fact for its own host, unless it made a new |
| 985 | // record: that holds every host of the app, so it is the app's |
| 986 | // fact (simFact.app). |
| 987 | if _, created := s.recFailed(a.Build, a.Host, res.Outcome == activator.SwapDown); created { |
| 988 | s.addFact(simFact{build: a.Build, host: a.Host, kind: "an adoption failure", start: sw.started, app: true}) |
| 989 | } |
| 990 | } |
| 991 | if res.Outcome == activator.SwapFailed && a.Mode == activator.ModeHold && !sw.serves { |
| 992 | // A hold adoption that ran, failed and was not held down. |
| 993 | s.holdRanOn(h, a.Build, sw.started) |
| 994 | } |
| 995 | return res |
| 996 | } |
| 997 | if a.Cause == activator.CauseRelease || a.Cause == activator.CauseRetry { |
| 998 | if want := s.installed[h.ch]; a.Build != want { |
| 999 | violate("I4", "%s switched to %s (cause %s) while its installed build is %s", a.Host, a.Build, a.Cause, want) |
| 1000 | } |
| 1001 | if a.Build != h.good && !h.served[a.Build] && s.blockedFor(a.Host, a.Build) { |
| 1002 | violate("I4", "%s switched to %s (cause %s) after its failure on this box was recorded", a.Host, a.Build, a.Cause) |
| 1003 | } |
| 1004 | } |
| 1005 | g := s.newGen(release.StagedIdentity{Version: a.Build, StagedPath: a.StagedPath, Digest: a.Digest, Demand: a.Demand}) |
| 1006 | h.phase, h.gen = simAwake, g |
| 1007 | s.goodAt(h, a.Build, sw.started) |
| 1008 | h.served[a.Build] = true |
| 1009 | res.Outcome, res.Gen = activator.SwapDone, g |
| 1010 | s.logf("swap %s %s %s (%s): done", a.Host, a.Cause, a.Build, a.Mode) |
| 1011 | return res |
| 1012 | } |
| 1013 | |
| 1014 | // await is the handoff: the one running attempt goroutine has parked in Swap |
| 1015 | // or returned. |
| 1016 | func (s *sim) await() simEvent { |
| 1017 | select { |
| 1018 | case ev := <-s.events: |
| 1019 | if ev.crash != nil { |
| 1020 | if v, ok := ev.crash.(simViolation); ok { |
| 1021 | panic(v) |
| 1022 | } |
| 1023 | if c, ok := ev.crash.(simCrash); ok { |
| 1024 | panic(c) |
| 1025 | } |
| 1026 | panic(simCrash{msg: fmt.Sprint(ev.crash), stack: ev.stack}) |
| 1027 | } |
| 1028 | return ev |
| 1029 | case <-time.After(30 * time.Second): |
| 1030 | panic(simCrash{msg: "an attempt neither reached Swap nor returned within 30s"}) |
| 1031 | } |
| 1032 | } |
| 1033 | |
| 1034 | // launch runs the real scheduler.execute for d up to its Swap (or to its |
| 1035 | // return, for a staging refusal). |
| 1036 | func (s *sim) launch(d dispatch) { |
| 1037 | stamp := s.stampFor(s.byName[d.ID.Host], d.ID.Build) |
| 1038 | go func() { |
| 1039 | defer func() { |
| 1040 | ev := simEvent{host: d.ID.Host} |
| 1041 | if r := recover(); r != nil { |
| 1042 | ev.crash, ev.stack = r, debug.Stack() |
| 1043 | } |
| 1044 | select { |
| 1045 | case s.events <- ev: |
| 1046 | case <-s.dead: |
| 1047 | // The seed ended (Swap's Goexit, or no one left to await). |
| 1048 | } |
| 1049 | }() |
| 1050 | s.sched.execute(context.Background(), d) |
| 1051 | }() |
| 1052 | ev := s.await() |
| 1053 | if ev.swap == nil { |
| 1054 | s.logf("launch %s %s %s: returned before Swap", d.ID.Host, d.ID.Cause, d.ID.Build) |
| 1055 | return |
| 1056 | } |
| 1057 | ev.swap.stamp = stamp |
| 1058 | // execute ran up to Swap within this step, so its dispatch time is now. |
| 1059 | ev.swap.started = s.now |
| 1060 | s.swaps[d.ID.Host] = ev.swap |
| 1061 | s.logf("launch %s %s %s rev %d canary %v class %d", d.ID.Host, d.ID.Cause, d.ID.Build, d.SchedRev, d.Canary, d.Class) |
| 1062 | } |
| 1063 | |
| 1064 | func (s *sim) run(steps int) { |
| 1065 | for s.step = 0; s.step < steps; s.step++ { |
| 1066 | s.now = s.now.Add(time.Duration(1+s.r.Intn(3)) * time.Second) |
| 1067 | s.diskFault() |
| 1068 | if s.gr != nil { |
| 1069 | // Before the step's own action, so an execution it starts begins |
| 1070 | // after a restart's pass and after every grant delivered here, as |
| 1071 | // the code's snapshot sees it. |
| 1072 | switch { |
| 1073 | case s.gr.Intn(150) == 0: |
| 1074 | s.agentRestart(true) |
| 1075 | case s.gr.Intn(40) == 0: |
| 1076 | s.agentRestart(false) |
| 1077 | } |
| 1078 | if s.rewake != "" { |
| 1079 | s.rewakeRun() |
| 1080 | } else if s.gr.Intn(6) == 0 { |
| 1081 | s.rewakeArm() |
| 1082 | } |
| 1083 | if s.gr.Intn(3) == 0 { |
| 1084 | s.deliverGrants() |
| 1085 | } |
| 1086 | if s.gr.Intn(4) == 0 { |
| 1087 | s.retryHeld() |
| 1088 | } |
| 1089 | } |
| 1090 | if hosts := sortedKeys(s.late); len(hosts) > 0 && s.r.Intn(12) == 0 { |
| 1091 | s.deliverLate(hosts[s.r.Intn(len(hosts))]) |
| 1092 | } |
| 1093 | switch k := s.r.Intn(26); { |
| 1094 | case k < 2: |
| 1095 | s.promote() |
| 1096 | case k < 4: |
| 1097 | s.pass() |
| 1098 | case k < 10: |
| 1099 | s.tick() |
| 1100 | case k < 14: |
| 1101 | s.complete() |
| 1102 | case k < 16: |
| 1103 | s.wakeBegin() |
| 1104 | case k < 18: |
| 1105 | s.wakeExec() |
| 1106 | case k < 19: |
| 1107 | s.observeEnd() |
| 1108 | case k < 20: |
| 1109 | s.sleep() |
| 1110 | case k < 21: |
| 1111 | s.stopEnd() |
| 1112 | case k < 22: |
| 1113 | s.restart() |
| 1114 | case k < 23: |
| 1115 | s.config() |
| 1116 | case k < 24: |
| 1117 | s.pointerRetry() |
| 1118 | case k < 25: |
| 1119 | s.hostRetry() |
| 1120 | default: |
| 1121 | s.restage() |
| 1122 | } |
| 1123 | s.checkI2() |
| 1124 | s.checkI7() |
| 1125 | s.checkI10() |
| 1126 | } |
| 1127 | s.draining = true |
| 1128 | for _, host := range sortedKeys(s.swaps) { |
| 1129 | sw := s.swaps[host] |
| 1130 | delete(s.swaps, host) |
| 1131 | close(sw.release) |
| 1132 | s.await() |
| 1133 | } |
| 1134 | for _, host := range sortedKeys(s.late) { |
| 1135 | s.deliverLate(host) |
| 1136 | } |
| 1137 | s.checkIntents() |
| 1138 | } |
| 1139 | |
| 1140 | func sortedKeys[V any](m map[string]V) []string { |
| 1141 | out := make([]string, 0, len(m)) |
| 1142 | for k := range m { |
| 1143 | out = append(out, k) |
| 1144 | } |
| 1145 | sort.Strings(out) |
| 1146 | return out |
| 1147 | } |
| 1148 | |
| 1149 | func (s *sim) pick(ok func(h *simHost) bool) *simHost { |
| 1150 | var c []*simHost |
| 1151 | for _, h := range s.hosts { |
| 1152 | if ok(h) { |
| 1153 | c = append(c, h) |
| 1154 | } |
| 1155 | } |
| 1156 | if len(c) == 0 { |
| 1157 | return nil |
| 1158 | } |
| 1159 | return c[s.r.Intn(len(c))] |
| 1160 | } |
| 1161 | |
| 1162 | // pass is the agent pass (Manager.Pass) over every row. The only marker a |
| 1163 | // Pass may clear is one a signed rollback lifts (applyPointers, ruling 205), |
| 1164 | // so each marker it cleared is checked against the model's evidence, then |
| 1165 | // lifted in the model too. |
| 1166 | // |
| 1167 | // A record the Pass cleared while a row still installs its build was cleared |
| 1168 | // by a re-signed pointer (a prune drops only builds no row installs): that |
| 1169 | // is the retry ordering's app-wide clear (ruling 191), noted as clearedAt. |
| 1170 | // (Read from the book like the markers; the model's stands rule already |
| 1171 | // says whether it may go.) |
| 1172 | func (s *sim) pass() { |
| 1173 | marked := map[string]bool{} |
| 1174 | for _, h := range s.hosts { |
| 1175 | if _, ok := s.book.HoldRanOn(h.name); ok { |
| 1176 | marked[h.name] = true |
| 1177 | } |
| 1178 | } |
| 1179 | recorded := map[string]bool{} |
| 1180 | for _, r := range s.book.List() { |
| 1181 | recorded[r.Build] = true |
| 1182 | } |
| 1183 | rs, _ := s.rows.List() |
| 1184 | s.m.Pass(rs) |
| 1185 | installed := map[string]bool{} |
| 1186 | for _, r := range rs { |
| 1187 | installed[r.Version] = true |
| 1188 | } |
| 1189 | still := map[string]bool{} |
| 1190 | for _, r := range s.book.List() { |
| 1191 | still[r.Build] = true |
| 1192 | } |
| 1193 | // The model's own records (I9) leave with the book's: a pointer's clear |
| 1194 | // or a prune, a clear of the record for every host. |
| 1195 | for _, b := range sortedKeys(s.grec) { |
| 1196 | if !still[b] { |
| 1197 | delete(s.grec, b) |
| 1198 | s.addClear(simClear{build: b, kind: "the record (a pointer's clear or a prune)"}) |
| 1199 | } |
| 1200 | } |
| 1201 | for _, b := range sortedKeys(recorded) { |
| 1202 | if still[b] { |
| 1203 | continue |
| 1204 | } |
| 1205 | // The record is gone, cleared or pruned, and with it every host |
| 1206 | // retry's exemption from it: the build's next failure writes a new |
| 1207 | // record, which holds every host (spec §8.4). |
| 1208 | for _, host := range sortedKeys(s.exempt) { |
| 1209 | delete(s.exempt[host], b) |
| 1210 | } |
| 1211 | if installed[b] { |
| 1212 | s.clearedAt[b] = s.now |
| 1213 | s.logf("ground truth: a pass cleared %s's record on a re-signed pointer", b) |
| 1214 | } |
| 1215 | } |
| 1216 | // The Pass's prune drops the retry clears of builds no row installs and |
| 1217 | // no planned first wake still carries (the hosts waking on a first-wake |
| 1218 | // plan: Manager.planned, once a pass has let go of the unowned ones). A |
| 1219 | // failure arriving after is fresh again (Task 4 fix round 1). |
| 1220 | for _, h := range s.hosts { |
| 1221 | if h.phase == simWaking && h.plan.FirstWake { |
| 1222 | installed[h.plan.Boot.Version] = true |
| 1223 | } |
| 1224 | } |
| 1225 | for _, b := range sortedKeys(s.clearedAt) { |
| 1226 | if !installed[b] { |
| 1227 | delete(s.clearedAt, b) |
| 1228 | } |
| 1229 | } |
| 1230 | for _, host := range sortedKeys(s.retryAt) { |
| 1231 | for _, b := range sortedKeys(s.retryAt[host]) { |
| 1232 | if !installed[b] { |
| 1233 | delete(s.retryAt[host], b) |
| 1234 | } |
| 1235 | } |
| 1236 | } |
| 1237 | for _, h := range s.hosts { |
| 1238 | if _, still := s.book.HoldRanOn(h.name); !marked[h.name] || still { |
| 1239 | continue |
| 1240 | } |
| 1241 | if mark := s.holdRan[h.name]; mark != "" { |
| 1242 | p, ok := s.pointers[s.channels[h.ch]] |
| 1243 | if h.good == "" || s.installed[h.ch] != h.good || !ok || p.Version != h.good || !p.PromotedAt.After(s.holdRanAt[h.name]) { |
| 1244 | violate("I5", "a pass cleared %s's hold-ran restriction (hold build %s ran at %s) with no signed rollback onto its good build %q (installed %s, pointer %+v)", |
| 1245 | h.name, mark, s.holdRanAt[h.name].Sub(t0), h.good, s.installed[h.ch], p) |
| 1246 | } |
| 1247 | delete(s.holdRan, h.name) |
| 1248 | s.addClear(simClear{host: h.name, marker: true, kind: "a signed rollback's lift"}) |
| 1249 | s.logf("ground truth: a signed rollback onto %s lifted %s's hold-ran restriction (%s)", h.good, h.name, mark) |
| 1250 | } |
| 1251 | } |
| 1252 | } |
| 1253 | |
| 1254 | // promote is a promotion and root's install onto one channel's hosts, then |
| 1255 | // the agent pass that sees it: a fresh build, or the other channel's build |
| 1256 | // promoted here (which signs a newer pointer for it, spec rev 14 §8.4). |
| 1257 | func (s *sim) promote() { |
| 1258 | ch := s.r.Intn(2) |
| 1259 | var b string |
| 1260 | if other := s.installed[1-ch]; s.r.Intn(4) == 0 && other != s.installed[ch] { |
| 1261 | b = other |
| 1262 | } else { |
| 1263 | s.nb++ |
| 1264 | b = "b" + strconv.Itoa(s.nb) |
| 1265 | s.addBuild(b, &simBuild{bad: s.r.Intn(4) == 0, flaky: s.r.Intn(5) == 0, holdDemand: s.r.Intn(6) == 0, misstaged: s.r.Intn(10) == 0}) |
| 1266 | } |
| 1267 | s.installed[ch] = b |
| 1268 | s.sign(s.channels[ch], b) |
| 1269 | for _, h := range s.hosts { |
| 1270 | if h.ch == ch { |
| 1271 | row := s.rows.get(h.name) |
| 1272 | row.Version = b |
| 1273 | s.rows.put(row) |
| 1274 | } |
| 1275 | } |
| 1276 | s.logf("promote %s onto channel %q %+v", b, s.channels[ch], *s.builds[b]) |
| 1277 | s.pass() |
| 1278 | } |
| 1279 | |
| 1280 | // tick is scheduler.loop's body: the view, configSlept, admit, and the |
| 1281 | // attempts it hands out run up to their Swap. |
| 1282 | func (s *sim) tick() { |
| 1283 | view, pendingConfig := s.sched.view() |
| 1284 | for _, host := range sortedKeys(pendingConfig) { |
| 1285 | if serving, seen := view[host]; seen && serving == "" { |
| 1286 | s.m.configSlept(host, pendingConfig[host]) |
| 1287 | } |
| 1288 | } |
| 1289 | s.sched.mu.Lock() |
| 1290 | ds := s.sched.q.admit(view, s.now) |
| 1291 | s.sched.mu.Unlock() |
| 1292 | // Liveness is judged on the admission itself, before any attempt runs: |
| 1293 | // one that returns at once (a staging refusal) frees its slot after |
| 1294 | // admit stopped at the limit, and the loop's next pass admits the rest. |
| 1295 | s.checkI2() |
| 1296 | s.checkI3Liveness(view) |
| 1297 | for _, d := range ds { |
| 1298 | s.launch(d) |
| 1299 | } |
| 1300 | s.checkI2() |
| 1301 | } |
| 1302 | |
| 1303 | func (s *sim) checkI2() { |
| 1304 | s.sched.mu.Lock() |
| 1305 | running := s.sched.q.running |
| 1306 | s.sched.mu.Unlock() |
| 1307 | if len(s.swaps) > s.concurrency || running > s.concurrency { |
| 1308 | violate("I2", "%d attempts in Swap (%d running in the queue) with swap_concurrency %d", len(s.swaps), running, s.concurrency) |
| 1309 | } |
| 1310 | } |
| 1311 | |
| 1312 | // checkI3Liveness runs after an admission that left capacity free: every |
| 1313 | // awake instance's restart or config attempt onto the build it serves was |
| 1314 | // admitted, unless it waits out its own deferral or its own cohort's |
| 1315 | // unsettled canary on another host. A failure record (of the build it serves |
| 1316 | // or of the one it wants) never holds it. |
| 1317 | func (s *sim) checkI3Liveness(view map[string]string) { |
| 1318 | q := s.sched.q |
| 1319 | s.sched.mu.Lock() |
| 1320 | defer s.sched.mu.Unlock() |
| 1321 | if q.running >= s.concurrency { |
| 1322 | return |
| 1323 | } |
| 1324 | for _, h := range s.hosts { |
| 1325 | if serving, seen := view[h.name]; !seen || serving == "" { |
| 1326 | continue |
| 1327 | } |
| 1328 | slot := q.hosts[h.name] |
| 1329 | if slot == nil || slot.running != nil { |
| 1330 | continue |
| 1331 | } |
| 1332 | d, ok := q.pending(slot) |
| 1333 | if !ok || d.ID.Build != slot.serving { |
| 1334 | continue |
| 1335 | } |
| 1336 | if s.now.Before(slot.notBefore) || (d.ID.Cause == activator.CauseConfig && s.now.Before(slot.configAfter)) { |
| 1337 | continue |
| 1338 | } |
| 1339 | if c := q.cohorts[cohortKey(slot.serving, classRecover)]; c != nil && c.canary != "" && c.canary != h.name && !c.settled { |
| 1340 | continue |
| 1341 | } |
| 1342 | violate("I3", "%s's %s attempt onto %s, the build it serves, was held back with capacity free (cohort %s)", h.name, d.ID.Cause, d.ID.Build, d.Cohort) |
| 1343 | } |
| 1344 | } |
| 1345 | |
| 1346 | // complete lets one parked attempt finish: the modelled Swap's end, then the |
| 1347 | // real done(). |
| 1348 | func (s *sim) complete() { |
| 1349 | hosts := sortedKeys(s.swaps) |
| 1350 | if len(hosts) == 0 { |
| 1351 | return |
| 1352 | } |
| 1353 | host := hosts[s.r.Intn(len(hosts))] |
| 1354 | sw := s.swaps[host] |
| 1355 | delete(s.swaps, host) |
| 1356 | close(sw.release) |
| 1357 | if ev := s.await(); ev.swap != nil { |
| 1358 | panic(simCrash{msg: "a second attempt reached Swap while " + host + "'s finished"}) |
| 1359 | } |
| 1360 | if sw.serves && s.snap(sw.a.Build) != sw.before { |
| 1361 | violate("I3", "%s's %s attempt onto %s, a build it serves, changed its failure record: %s -> %s", host, sw.a.Cause, sw.a.Build, sw.before, s.snap(sw.a.Build)) |
| 1362 | } |
| 1363 | } |
| 1364 | |
| 1365 | // heldDownOn reports whether the model let host be held down on build: a |
| 1366 | // hold build that ran and failed there, with no host retry since. |
| 1367 | func (s *sim) heldDownOn(host, build string) bool { |
| 1368 | return s.heldDown[host][build] |
| 1369 | } |
| 1370 | |
| 1371 | // holdRanOn is the ground truth of a hold build that ran on h without |
| 1372 | // passing or being held down there (N2), in the execution that started at |
| 1373 | // start. Each execution renews the restriction (plan review finding 4): its |
| 1374 | // stamp is the latest start of the runs since it was last lifted, whatever |
| 1375 | // their builds (Task 4 fix round 1), so a rollback signed between two runs |
| 1376 | // lifts nothing, and it is set again now, so a good write decided before now |
| 1377 | // lifts nothing either. |
| 1378 | func (s *sim) holdRanOn(h *simHost, build string, start time.Time) { |
| 1379 | if s.holdRan[h.name] == "" || start.After(s.holdRanAt[h.name]) { |
| 1380 | s.holdRanAt[h.name] = start |
| 1381 | } |
| 1382 | if old := s.holdRan[h.name]; old != "" && old != build { |
| 1383 | // The marker now names build: old's restriction of h is gone (I9). |
| 1384 | s.addClear(simClear{build: old, host: h.name, marker: true, kind: "a marker replaced"}) |
| 1385 | } |
| 1386 | s.holdRan[h.name] = build |
| 1387 | s.holdRanSet[h.name] = s.now |
| 1388 | // Every run is a new execution, which sets or renews the marker (I9). |
| 1389 | s.addFact(simFact{build: build, host: h.name, kind: "a hold run", start: start, marker: true}) |
| 1390 | s.logf("ground truth: hold build %s ran on %s without a verdict", build, h.name) |
| 1391 | } |
| 1392 | |
| 1393 | // goodAt is h recording build as its good build (the code's good-build |
| 1394 | // write) for the execution that started at start. It lifts a hold-ran |
| 1395 | // restriction set before that start, and only such a one (Astra R3 N6): a |
| 1396 | // hold build that ran on the host after the passing execution began is not |
| 1397 | // one the instance has passed over. |
| 1398 | func (s *sim) goodAt(h *simHost, build string, start time.Time) { |
| 1399 | h.good = build |
| 1400 | if s.holdRanSet[h.name].Before(start) { |
| 1401 | if s.holdRan[h.name] != "" { |
| 1402 | s.addClear(simClear{host: h.name, marker: true, kind: "a good write's lift"}) |
| 1403 | } |
| 1404 | delete(s.holdRan, h.name) |
| 1405 | } |
| 1406 | } |
| 1407 | |
| 1408 | // checkI5 is I5 at the exec of build on h, for a wake planned against rt. |
| 1409 | // The installed good build is barred too (ruling 205): root's sibling |
| 1410 | // repoint installs it without any rollback, and only a Pass that saw a |
| 1411 | // signed one lifts the restriction. The marked build itself is exempt even |
| 1412 | // when it is the good build (Task 3 fix round 1): booting it is the |
| 1413 | // marked-build rule, not a boot of the good build in its place. |
| 1414 | func (s *sim) checkI5(h *simHost, rt registry.Route, build string) { |
| 1415 | if mark := s.holdRan[h.name]; mark != "" && build == h.good && build != mark { |
| 1416 | violate("I5", "%s boots its good build %s (installed %s) after hold build %s ran there, with no signed rollback since", h.name, build, rt.Version, mark) |
| 1417 | } |
| 1418 | } |
| 1419 | |
| 1420 | // bootable is the plan's Check for build at its exec (planGate): nil when |
| 1421 | // the exec may go ahead. |
| 1422 | func bootable(plan activator.WakePlan, build string) error { |
| 1423 | if plan.Check == nil { |
| 1424 | return nil |
| 1425 | } |
| 1426 | return plan.Check(build) |
| 1427 | } |
| 1428 | |
| 1429 | // wakeBegin is a wake's planning (Manager.Plan, as the activator calls it |
| 1430 | // before the lease). |
| 1431 | func (s *sim) wakeBegin() { |
| 1432 | if h := s.pick(func(h *simHost) bool { return h.phase == simAsleep }); h != nil { |
| 1433 | s.wakeBeginOn(h) |
| 1434 | } |
| 1435 | } |
| 1436 | |
| 1437 | // wakeBeginOn is wakeBegin for h, which is asleep. |
| 1438 | func (s *sim) wakeBeginOn(h *simHost) { |
| 1439 | // The activator's outcome fence (awaitOutcomes): an earlier wake's |
| 1440 | // events reach the agent before the host plans again. |
| 1441 | if _, ok := s.late[h.name]; ok { |
| 1442 | s.deliverLate(h.name) |
| 1443 | } |
| 1444 | row := s.rows.get(h.name) |
| 1445 | h.phase = simWaking |
| 1446 | plan := s.m.Plan(context.Background(), row) |
| 1447 | if plan.Refuse != "" || plan.Boot.StagedPath == "" { |
| 1448 | h.phase = simAsleep |
| 1449 | s.logf("wake %s: refused at planning: %s", h.name, plan.Refuse) |
| 1450 | s.checkWakeRefused(h, row.Version, "its plan refused: "+plan.Refuse) |
| 1451 | return |
| 1452 | } |
| 1453 | h.plan, h.planRoute, h.wakeStart, h.planAt, h.stamp = plan, row, s.now, s.now, s.stampFor(h, plan.Boot.Version) |
| 1454 | h.underMark = s.holdRan[h.name] != "" |
| 1455 | s.logf("wake %s planned: boot %s first %v fallback %v mode %s", h.name, plan.Boot.Version, plan.FirstWake, plan.Fallback != nil, plan.Mode) |
| 1456 | s.checkPlanned(h, plan.Boot.Version) |
| 1457 | } |
| 1458 | |
| 1459 | func (s *sim) checkWakeRefused(h *simHost, installed, why string) { |
| 1460 | if h.good == "" || s.heldDownOn(h.name, installed) { |
| 1461 | return |
| 1462 | } |
| 1463 | if s.fault && strings.Contains(why, BookUnreadableRefusal) { |
| 1464 | // A first wake whose intent could not be saved boots nothing (Task 5): |
| 1465 | // its failure could not outlive a restart (I8). |
| 1466 | return |
| 1467 | } |
| 1468 | if mark := s.holdRan[h.name]; mark != "" { |
| 1469 | if installed == mark && installed == h.good { |
| 1470 | // The marked build is the installed good build (Task 3 fix round |
| 1471 | // 1): it boots under the marked-build rule. A record does not |
| 1472 | // hold a build the instance served, and the hold-down returned |
| 1473 | // above, so only a staging refusal may refuse it. |
| 1474 | if s.builds[installed].misstaged { |
| 1475 | return |
| 1476 | } |
| 1477 | violate("I1", "%s has its good build %s installed, the hold build that ran there, but its wake boots nothing: %s", h.name, installed, why) |
| 1478 | } |
| 1479 | // Restricted (N2): refused for an installed build held for the |
| 1480 | // instance, the marked build included (Astra R3 N7: the marker lifts |
| 1481 | // no record), for its installed good build until a signed rollback |
| 1482 | // lifts the restriction (ruling 205; pass checks the lifting), or for |
| 1483 | // a build not staged correctly. "Held" is read loosely here (a |
| 1484 | // failure of it was recorded at some point and no host retry exempts |
| 1485 | // the instance): the book clears a record only at its next due |
| 1486 | // pointer read, after the model's re-sign. |
| 1487 | held := s.failedEver[installed] && !s.exempt[h.name][installed] |
| 1488 | if held || installed == h.good || s.builds[installed].misstaged { |
| 1489 | return |
| 1490 | } |
| 1491 | violate("I1", "%s (good build %s, hold build %s ran there) has %s installed but its wake boots nothing: %s", h.name, h.good, mark, installed, why) |
| 1492 | } |
| 1493 | violate("I1", "%s has good build %s but its wake boots nothing: %s", h.name, h.good, why) |
| 1494 | } |
| 1495 | |
| 1496 | // checkBoot is I1 at the exec of build on h. A hold-ran restriction exempts |
| 1497 | // nothing (Astra R3 N7): spec §8.4 exempts an instance that served the |
| 1498 | // build, not one where it merely ran. |
| 1499 | func (s *sim) checkBoot(h *simHost, build string) { |
| 1500 | if h.good != "" && build != h.good && !h.served[build] && s.blockedFor(h.name, build) { |
| 1501 | violate("I1", "%s (good build %s) boots %s, whose failure on this box stands", h.name, h.good, build) |
| 1502 | } |
| 1503 | } |
| 1504 | |
| 1505 | // checkPlanned is I1 at a wake's plan, its first gate: the planner never |
| 1506 | // names a build whose failure on this box stands for an instance with a |
| 1507 | // good build (decideWake, and decideHoldRan for the marked build, Astra R3 |
| 1508 | // N7). The plan's Check at the exec is the second gate, for records written |
| 1509 | // since the plan, and is not what keeps a held build out of a plan. |
| 1510 | func (s *sim) checkPlanned(h *simHost, build string) { |
| 1511 | if h.good != "" && build != h.good && !h.served[build] && s.blockedFor(h.name, build) { |
| 1512 | violate("I1", "%s (good build %s) plans a wake of %s, whose failure on this box stands", h.name, h.good, build) |
| 1513 | } |
| 1514 | } |
| 1515 | |
| 1516 | // wakeExec is the activator's checkPlan and exec (spec §7.4, §8.2): the |
| 1517 | // plan's Check, one replan if it refuses, the verdict floor, and the build's |
| 1518 | // outcome reported through the real OnGen. |
| 1519 | func (s *sim) wakeExec() { |
| 1520 | if h := s.pick(func(h *simHost) bool { return h.phase == simWaking }); h != nil { |
| 1521 | s.wakeExecOn(h, "") |
| 1522 | } |
| 1523 | } |
| 1524 | |
| 1525 | // wakeExecOn is wakeExec for h, which is waking. A first wake of failBuild |
| 1526 | // that runs fails its health check at once, as a flaky build may (rewakeRun's |
| 1527 | // scenario); "" fails nothing more than the build itself does. |
| 1528 | func (s *sim) wakeExecOn(h *simHost, failBuild string) { |
| 1529 | rt, plan := h.planRoute, h.plan |
| 1530 | if s.r.Intn(20) == 0 { |
| 1531 | // The lease was refused (a previous incarnation still holds it): the |
| 1532 | // wake ends before any exec. The activator reports it as cancelled |
| 1533 | // with nothing run (Task 5), which ends the plan's registration and |
| 1534 | // its intent. Not a build failure, so no I1 check. |
| 1535 | h.phase = simAsleep |
| 1536 | s.logf("wake %s: refused at its lease, cancelled with nothing run", h.name) |
| 1537 | s.cancelUnrun(rt, plan) |
| 1538 | return |
| 1539 | } |
| 1540 | if plan.Check != nil && plan.Check(plan.Boot.Version) != nil { |
| 1541 | next := s.m.Plan(context.Background(), rt) |
| 1542 | if next.Refuse != "" || next.Boot.StagedPath == "" || (next.Check != nil && next.Check(next.Boot.Version) != nil) { |
| 1543 | h.phase = simAsleep |
| 1544 | s.logf("wake %s: abandoned at exec (plan stale, replan %q)", h.name, next.Refuse) |
| 1545 | // The activator's checkPlan cancels a replan its own check |
| 1546 | // refused, then afterWake the wake's plan (Task 5): neither ran. |
| 1547 | if next.Refuse == "" && next.Boot.StagedPath != "" { |
| 1548 | s.cancelUnrun(rt, next) |
| 1549 | } |
| 1550 | s.cancelUnrun(rt, plan) |
| 1551 | // The replan read the row afresh (Plan replans a moved row). |
| 1552 | s.checkWakeRefused(h, s.rows.get(h.name).Version, "the plan went stale and the replan refused: "+next.Refuse) |
| 1553 | return |
| 1554 | } |
| 1555 | // The replan replaces the plan, which checkPlan cancels: it never ran. |
| 1556 | s.cancelUnrun(rt, plan) |
| 1557 | plan, h.plan, h.planAt, h.stamp = next, next, s.now, s.stampFor(h, next.Boot.Version) |
| 1558 | h.underMark = s.holdRan[h.name] != "" |
| 1559 | s.logf("wake %s replanned: boot %s first %v", h.name, plan.Boot.Version, plan.FirstWake) |
| 1560 | s.checkPlanned(h, plan.Boot.Version) |
| 1561 | } |
| 1562 | canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == activator.ModeOverlap |
| 1563 | if left := h.wakeStart.Add(simHealthWindow).Sub(s.now); canFall && left < simMinVerdict { |
| 1564 | // The verdict floor (spawnPlanned): B's health window ends 30 s |
| 1565 | // after the wake's start (55 s budget less the 25 s kill reserve), |
| 1566 | // and with less than 20 s of it left at the exec, the good build |
| 1567 | // boots untried and nothing is recorded against B. |
| 1568 | good := *plan.Fallback |
| 1569 | if err := bootable(plan, good.Version); err != nil { |
| 1570 | // The good build's own gate refused it at the exec: nothing ran. |
| 1571 | h.phase = simAsleep |
| 1572 | s.logf("wake %s: %s untried, and the good build %s was refused at its exec: %v", h.name, plan.Boot.Version, good.Version, err) |
| 1573 | s.cancelUnrun(rt, plan) |
| 1574 | s.checkWakeRefused(h, rt.Version, "the untried first wake's good build was refused at its exec") |
| 1575 | return |
| 1576 | } |
| 1577 | s.checkBoot(h, good.Version) |
| 1578 | s.checkI5(h, rt, good.Version) |
| 1579 | h.phase, h.gen = simAwake, s.newGen(good) |
| 1580 | h.served[good.Version] = true |
| 1581 | s.logf("wake %s: %s untried, booted the good build %s", h.name, plan.Boot.Version, good.Version) |
| 1582 | // served: the good build is written again, when the event lands. |
| 1583 | s.healthy(h, activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: h.gen, Mode: plan.Mode, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1584 | return |
| 1585 | } |
| 1586 | boot := plan.Boot |
| 1587 | s.checkBoot(h, boot.Version) |
| 1588 | s.checkI5(h, rt, boot.Version) |
| 1589 | marked := h.underMark || s.holdRan[h.name] != "" |
| 1590 | if h.good != "" && plan.FirstWake && !s.expectHold(boot.Version) && (plan.Fallback == nil || plan.Fallback.Version != h.good) && !marked { |
| 1591 | violate("I1", "%s (good build %s) first-wakes %s in overlap mode without its good build behind it", h.name, h.good, boot.Version) |
| 1592 | } |
| 1593 | if !plan.FirstWake { |
| 1594 | h.phase, h.gen = simAwake, s.newGen(boot) |
| 1595 | h.served[boot.Version] = true |
| 1596 | s.logf("wake %s: booted %s", h.name, boot.Version) |
| 1597 | s.healthy(h, activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: h.gen, Mode: plan.Mode, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1598 | return |
| 1599 | } |
| 1600 | if s.r.Intn(12) == 0 { |
| 1601 | // The wake's lifecycle ended it after the build's exec (a stop, a |
| 1602 | // fence, a step after the spawn): no verdict. The activator reports |
| 1603 | // it as cancelled after a run (GenCancelled, Executed): a hold one |
| 1604 | // leaves the hold-ran restriction (N2), an overlap one only ends the |
| 1605 | // wake (Task 5). |
| 1606 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 1607 | s.logf("wake %s: first wake of %s ended after its exec, no verdict (mode %s)", h.name, boot.Version, plan.Mode) |
| 1608 | if plan.Mode == activator.ModeHold { |
| 1609 | s.holdRanOn(h, boot.Version, h.planAt) |
| 1610 | } |
| 1611 | s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Gen: activator.Gen{Build: boot.Version}, Mode: plan.Mode, |
| 1612 | FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev, Executed: true}) |
| 1613 | return |
| 1614 | } |
| 1615 | b := s.builds[boot.Version] |
| 1616 | forced := boot.Version == failBuild |
| 1617 | fails := forced || b.bad || (b.flaky && s.r.Intn(2) == 0) |
| 1618 | if fails && !forced && s.r.Intn(2) == 0 { |
| 1619 | // It passed its health check and exits inside its observation. |
| 1620 | fails, h.failLater = false, true |
| 1621 | } else { |
| 1622 | h.failLater = false |
| 1623 | } |
| 1624 | switch { |
| 1625 | case !fails: |
| 1626 | h.phase, h.gen = simObserving, s.newGen(boot) |
| 1627 | s.logf("wake %s: first wake of %s healthy, observing", h.name, boot.Version) |
| 1628 | s.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: rt, Gen: h.gen, Mode: plan.Mode, FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1629 | case canFall && bootable(plan, plan.Fallback.Version) != nil: |
| 1630 | // B failed, and its fallback's own gate refused the good build at |
| 1631 | // its exec: B's failure is recorded, nothing fell back. |
| 1632 | s.failure(boot.Version, h.name, h.stamp, false, h.planAt) |
| 1633 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 1634 | s.logf("wake %s: first wake of %s failed, its fallback %s refused at its exec", h.name, boot.Version, plan.Fallback.Version) |
| 1635 | s.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: rt, Failed: boot, Mode: plan.Mode, |
| 1636 | FirstWake: true, Reason: "health check failed", At: s.now, FailedUnder: plan.FailedUnder, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1637 | if h.good != "" && s.holdRan[h.name] == "" { |
| 1638 | violate("I1", "%s (good build %s) failed a first wake of %s and its fallback was refused with no hold-ran restriction", h.name, h.good, boot.Version) |
| 1639 | } |
| 1640 | case canFall: |
| 1641 | s.checkI5(h, rt, plan.Fallback.Version) |
| 1642 | s.failure(boot.Version, h.name, h.stamp, false, h.planAt) |
| 1643 | h.phase, h.gen = simAwake, s.newGen(*plan.Fallback) |
| 1644 | s.logf("wake %s: first wake of %s failed, fell back to %s", h.name, boot.Version, plan.Fallback.Version) |
| 1645 | s.m.OnGen(activator.GenEvent{Kind: activator.GenFellBack, Route: rt, Gen: h.gen, Failed: boot, Mode: plan.Mode, |
| 1646 | FirstWake: true, Reason: "health check failed", At: s.now, FailedUnder: plan.FailedUnder, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1647 | default: |
| 1648 | hold := plan.Mode == activator.ModeHold |
| 1649 | s.failure(boot.Version, h.name, h.stamp, hold, h.planAt) |
| 1650 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 1651 | s.logf("wake %s: first wake of %s failed, nothing fell back (mode %s)", h.name, boot.Version, plan.Mode) |
| 1652 | s.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: rt, Failed: boot, Mode: plan.Mode, |
| 1653 | FirstWake: true, Reason: "health check failed", At: s.now, FailedUnder: plan.FailedUnder, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1654 | if h.good != "" && !hold && !marked { |
| 1655 | violate("I1", "%s (good build %s) failed a first wake of %s in overlap mode and nothing fell back", h.name, h.good, boot.Version) |
| 1656 | } |
| 1657 | } |
| 1658 | } |
| 1659 | |
| 1660 | // cancelUnrun is the activator's report of a planned wake that ended before |
| 1661 | // its build ran (Task 5): refused at its lease or its exec, or replaced by a |
| 1662 | // replan. It records nothing, and ends the plan's registration and intent. |
| 1663 | func (s *sim) cancelUnrun(rt registry.Route, plan activator.WakePlan) { |
| 1664 | s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: rt, Mode: plan.Mode, FirstWake: plan.FirstWake, At: s.now, |
| 1665 | Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1666 | } |
| 1667 | |
| 1668 | // healthy reports an ordinary wake's healthy event (not a first wake's), |
| 1669 | // which writes the good build (served). Half of them are delivered late |
| 1670 | // (sim.late): the activator serves the generation at once, but its event, |
| 1671 | // and with it the good write, reaches the agent some steps later (OnGen |
| 1672 | // takes a late event as a matter of course). The good write is then |
| 1673 | // decided by a wake whose execution began before whatever ran on the host |
| 1674 | // meanwhile: an adoption, admitted because the generation serves, whose |
| 1675 | // hold build ran and failed leaves a marker the write must not clear (Astra |
| 1676 | // R3 N6, ClearHoldRanBefore). |
| 1677 | func (s *sim) healthy(h *simHost, ev activator.GenEvent) { |
| 1678 | s.late[h.name] = simLate{ev: ev, start: h.planAt} |
| 1679 | if s.r.Intn(2) == 0 { |
| 1680 | s.logf("wake %s: its healthy event is late", h.name) |
| 1681 | return |
| 1682 | } |
| 1683 | s.deliverLate(h.name) |
| 1684 | } |
| 1685 | |
| 1686 | // deliverLate delivers host's healthy event. The good write follows only |
| 1687 | // while the event's generation is the one serving (Manager.acknowledged), |
| 1688 | // and it lifts only a restriction set before the wake's execution began |
| 1689 | // (goodAt). |
| 1690 | func (s *sim) deliverLate(host string) { |
| 1691 | l, ok := s.late[host] |
| 1692 | if !ok { |
| 1693 | return |
| 1694 | } |
| 1695 | delete(s.late, host) |
| 1696 | h := s.byName[host] |
| 1697 | s.logf("wake %s: healthy event of %s delivered", host, l.ev.Gen.Build) |
| 1698 | s.goodWrite(h, l.ev.Gen.Build, l.start, h.phase == simAwake && h.gen.N == l.ev.Gen.N, func() { s.m.OnGen(l.ev) }) |
| 1699 | } |
| 1700 | |
| 1701 | // goodWrite is a good-build write of build on h by an execution that began |
| 1702 | // at start (when acked: the code writes only while the generation serves), |
| 1703 | // made by deliver, the real event. It is I5 at the write: a write may lift |
| 1704 | // only a restriction set before its execution began (goodAt), so a marker |
| 1705 | // that stood in the book before the event and is gone after it, while the |
| 1706 | // model's restriction stands, was cleared by a write decided before its |
| 1707 | // hold build ran (Astra R3 N6). The book is read only to see what the write |
| 1708 | // cleared, as pass reads it for a Pass. |
| 1709 | func (s *sim) goodWrite(h *simHost, build string, start time.Time, acked bool, deliver func()) { |
| 1710 | _, had := s.book.HoldRanOn(h.name) |
| 1711 | if acked { |
| 1712 | s.goodAt(h, build, start) |
| 1713 | } |
| 1714 | deliver() |
| 1715 | if _, still := s.book.HoldRanOn(h.name); had && !still && s.holdRan[h.name] != "" { |
| 1716 | violate("I5", "%s's good write of %s, by an execution begun at %s, cleared the hold-ran restriction of %s, set at %s after it began", |
| 1717 | h.name, build, start.Sub(t0), s.holdRan[h.name], s.holdRanSet[h.name].Sub(t0)) |
| 1718 | } |
| 1719 | } |
| 1720 | |
| 1721 | // checkIntents is I6 once the run has drained (Task 5): every execution's |
| 1722 | // intent ends with its outcome, so the only intents left are those of first |
| 1723 | // wakes still in flight (planned or under observation), each the host's |
| 1724 | // current plan's. A leaked intent would refuse the host's adoptions, and an |
| 1725 | // agent restart would turn it into a failure nothing failed. |
| 1726 | func (s *sim) checkIntents() { |
| 1727 | s.book.mu.Lock() |
| 1728 | intents := maps.Clone(s.book.intents) |
| 1729 | s.book.mu.Unlock() |
| 1730 | for _, host := range sortedKeys(intents) { |
| 1731 | in, h := intents[host], s.byName[host] |
| 1732 | inFlight := h != nil && (h.phase == simWaking || h.phase == simObserving) && h.plan.FirstWake && in.ID == s.m.wakeExecID(h.plan.Token) |
| 1733 | if !inFlight { |
| 1734 | violate("I6", "%s's intent %+v outlived its execution", host, in) |
| 1735 | } |
| 1736 | } |
| 1737 | } |
| 1738 | |
| 1739 | // observeEnd ends a first wake's observation: it settles, or the build exits. |
| 1740 | func (s *sim) observeEnd() { |
| 1741 | h := s.pick(func(h *simHost) bool { return h.phase == simObserving }) |
| 1742 | if h == nil { |
| 1743 | return |
| 1744 | } |
| 1745 | rt, plan := h.planRoute, h.plan |
| 1746 | if h.failLater { |
| 1747 | s.failure(plan.Boot.Version, h.name, h.stamp, plan.Mode == activator.ModeHold, h.planAt) |
| 1748 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 1749 | s.logf("wake %s: %s exited inside its observation", h.name, plan.Boot.Version) |
| 1750 | s.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: rt, Failed: plan.Boot, Mode: plan.Mode, FirstWake: true, |
| 1751 | Reason: "exited within 10s of becoming healthy", At: s.now, FailedUnder: plan.FailedUnder, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1752 | return |
| 1753 | } |
| 1754 | h.phase = simAwake |
| 1755 | h.served[h.gen.Build] = true |
| 1756 | s.logf("wake %s: %s settled", h.name, h.gen.Build) |
| 1757 | s.goodWrite(h, h.gen.Build, h.planAt, true, func() { |
| 1758 | s.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: rt, Gen: h.gen, Mode: plan.Mode, FirstWake: true, At: s.now, Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 1759 | }) |
| 1760 | } |
| 1761 | |
| 1762 | // sleep hibernates an instance with no attempt in Swap: at once, or through a |
| 1763 | // stopping phase the view leaves out. An observation it ends is cancelled. |
| 1764 | func (s *sim) sleep() { |
| 1765 | if h := s.pick(func(h *simHost) bool { |
| 1766 | _, inSwap := s.swaps[h.name] |
| 1767 | return !inSwap && (h.phase == simAwake || h.phase == simObserving) |
| 1768 | }); h != nil { |
| 1769 | s.sleepOn(h) |
| 1770 | } |
| 1771 | } |
| 1772 | |
| 1773 | // sleepOn is sleep for h, which is awake or observing with no attempt in |
| 1774 | // Swap. |
| 1775 | func (s *sim) sleepOn(h *simHost) { |
| 1776 | observing := h.phase == simObserving |
| 1777 | g := h.gen |
| 1778 | h.phase, h.gen = simAsleep, activator.Gen{} |
| 1779 | if s.r.Intn(2) == 0 { |
| 1780 | h.phase = simStopping |
| 1781 | } |
| 1782 | s.logf("sleep %s (stopping phase %v, observation cancelled %v)", h.name, h.phase == simStopping, observing) |
| 1783 | if observing { |
| 1784 | // The observed build was healthy, so it ran: a hold one leaves the |
| 1785 | // hold-ran restriction (N2). |
| 1786 | if h.plan.Mode == activator.ModeHold { |
| 1787 | s.holdRanOn(h, g.Build, h.planAt) |
| 1788 | } |
| 1789 | s.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.planRoute, Gen: g, Mode: h.plan.Mode, FirstWake: true, At: s.now, |
| 1790 | Token: h.plan.Token, ConfigRev: h.plan.ConfigRev, Executed: true}) |
| 1791 | } |
| 1792 | } |
| 1793 | |
| 1794 | func (s *sim) stopEnd() { |
| 1795 | if h := s.pick(func(h *simHost) bool { return h.phase == simStopping }); h != nil { |
| 1796 | h.phase = simAsleep |
| 1797 | s.logf("stopped %s", h.name) |
| 1798 | } |
| 1799 | } |
| 1800 | |
| 1801 | func (s *sim) restart() { |
| 1802 | h := s.hosts[s.r.Intn(len(s.hosts))] |
| 1803 | s.nreq++ |
| 1804 | id := "r" + strconv.Itoa(s.nreq) |
| 1805 | ok := s.m.Restart(h.name, id) |
| 1806 | s.logf("restart %s %s: %v", h.name, id, ok) |
| 1807 | } |
| 1808 | |
| 1809 | func (s *sim) config() { |
| 1810 | h := s.hosts[s.r.Intn(len(s.hosts))] |
| 1811 | s.nreq++ |
| 1812 | rev := "c" + strconv.Itoa(s.nreq) |
| 1813 | s.m.ConfigDelivered(h.name, rev) |
| 1814 | s.logf("config %s %s", h.name, rev) |
| 1815 | } |
| 1816 | |
| 1817 | // pointerRetry is `carlos promote --retry`: the channel's head is signed again |
| 1818 | // with a newer PromotedAt. The next due agent pass clears the record. |
| 1819 | func (s *sim) pointerRetry() { |
| 1820 | ch := s.r.Intn(2) |
| 1821 | b := s.installed[ch] |
| 1822 | if !s.stands(b) { |
| 1823 | return |
| 1824 | } |
| 1825 | s.sign(s.channels[ch], b) |
| 1826 | s.logf("promote --retry %s on channel %q", b, s.channels[ch]) |
| 1827 | } |
| 1828 | |
| 1829 | // hostRetry is `carlos instances retry <host>` for a host whose installed |
| 1830 | // build's failure stands. |
| 1831 | func (s *sim) hostRetry() { |
| 1832 | h := s.hosts[s.r.Intn(len(s.hosts))] |
| 1833 | if s.gr != nil { |
| 1834 | // A seed whose book is on disk queues the grant (I9): delivered at |
| 1835 | // later steps, again and again (deliverGrants). |
| 1836 | s.issueGrant(h, "") |
| 1837 | return |
| 1838 | } |
| 1839 | b := s.installed[h.ch] |
| 1840 | if !s.stands(b) { |
| 1841 | return |
| 1842 | } |
| 1843 | // The retry orders later failures only when it un-retries the host (Task |
| 1844 | // 4 fix round 1): a record of b stands and does not already exempt h. |
| 1845 | // Whether the code's record still lists h as retried is read from the |
| 1846 | // book, like the clears pass() sees: the model's exempt outlives a record |
| 1847 | // that was cleared and written again. |
| 1848 | rec, recorded := s.book.Lookup("bab", "blog", b) |
| 1849 | clears := recorded && !slices.Contains(rec.Retried, h.name) |
| 1850 | // The console's grant names the build and the generation the box |
| 1851 | // publishes for it (Task 8; readiness round 2 Task 1): the sim reads it |
| 1852 | // just as the console would, and delivers it at once. This book is in |
| 1853 | // memory, so it never owes a save; the seeds whose book is on disk queue |
| 1854 | // their grants instead (issueGrant, deliverGrants). |
| 1855 | epoch := s.book.PublishedEpoch("bab", "blog", b, h.name) |
| 1856 | ok := s.m.RetryHost(h.name, b, s.book.ID(), epoch) |
| 1857 | // The retry exempts h from the record that stands now, and from no |
| 1858 | // record written after that one goes (pass): with no record there is |
| 1859 | // nothing to exempt it from. |
| 1860 | if recorded { |
| 1861 | if s.exempt[h.name] == nil { |
| 1862 | s.exempt[h.name] = map[string]bool{} |
| 1863 | } |
| 1864 | s.exempt[h.name][b] = true |
| 1865 | } |
| 1866 | if clears { |
| 1867 | if s.retryAt[h.name] == nil { |
| 1868 | s.retryAt[h.name] = map[string]time.Time{} |
| 1869 | } |
| 1870 | s.retryAt[h.name][b] = s.now |
| 1871 | } |
| 1872 | delete(s.heldDown[h.name], b) |
| 1873 | // A grant lifts the marker only when the marker names its build (plan |
| 1874 | // review finding 5). |
| 1875 | if s.holdRan[h.name] == b { |
| 1876 | delete(s.holdRan, h.name) |
| 1877 | } |
| 1878 | s.logf("instances retry %s onto %s: %v", h.name, b, ok) |
| 1879 | } |
| 1880 | |
| 1881 | // heldBuilds are the builds that may hold h as status shows it: the |
| 1882 | // installed build of h's channel, and its hold-ran marker's build. |
| 1883 | func (s *sim) heldBuilds(h *simHost) []string { |
| 1884 | builds := []string{s.installed[h.ch]} |
| 1885 | if m, ok := s.book.HoldRanOn(h.name); ok && m.Build != builds[0] { |
| 1886 | builds = append(builds, m.Build) |
| 1887 | } |
| 1888 | return builds |
| 1889 | } |
| 1890 | |
| 1891 | // retryHeld is an operator pressing Retry on a held instance, on a seed |
| 1892 | // whose book is on disk: one (host, build) the box publishes a generation |
| 1893 | // for, at random, gets a grant (issueGrant). |
| 1894 | func (s *sim) retryHeld() { |
| 1895 | var held [][2]string |
| 1896 | for _, h := range s.hosts { |
| 1897 | for _, b := range s.heldBuilds(h) { |
| 1898 | if s.published(b, h.name) != 0 { |
| 1899 | held = append(held, [2]string{h.name, b}) |
| 1900 | } |
| 1901 | } |
| 1902 | } |
| 1903 | if len(held) == 0 { |
| 1904 | return |
| 1905 | } |
| 1906 | p := held[s.gr.Intn(len(held))] |
| 1907 | s.issueGrant(s.byName[p[0]], p[1]) |
| 1908 | } |
| 1909 | |
| 1910 | // rewakeArm and rewakeRun, on a seed whose book is on disk, are an operator |
| 1911 | // pressing Retry on an instance its hold-ran marker holds (asleep, the |
| 1912 | // marker's build X still installed), and the instance then woken again and |
| 1913 | // stopped before that wake settles, before the grant is read. The wake |
| 1914 | // boots X under the marker as a hold first wake (decideHoldRan): a run of |
| 1915 | // X that ends with no verdict renews the marker as a new execution's, and |
| 1916 | // a run that fails holds it down. Either is a fact of (X, host) after the |
| 1917 | // grant was issued, which must leave the grant clearing nothing (I9; M14's |
| 1918 | // witness: a renewal that moved no generation). The random schedule meets |
| 1919 | // this only rarely, as an instance is seldom woken again under its marker |
| 1920 | // before another restriction of its app moves the generation; here the two |
| 1921 | // halves are one step apart. |
| 1922 | // |
| 1923 | // When X has no failure record yet, the wake is instead another asleep |
| 1924 | // instance's on the same channel, one that never served X, when there is one, |
| 1925 | // and its first wake of X, if it runs, fails its health check (rewakeFail, |
| 1926 | // as a flaky build may: a build with a marker and no record has seldom |
| 1927 | // failed anywhere). That failure writes X's first record, which holds every |
| 1928 | // host of the app, the granted one too (spec §8.4): a fact of every host |
| 1929 | // after the grant was issued (M17's witness: a new record that moved no |
| 1930 | // generation). The book is read here only to pick the scenario, never for |
| 1931 | // a verdict. |
| 1932 | // |
| 1933 | // rewakeArm issues the grant and remembers the host to wake. |
| 1934 | func (s *sim) rewakeArm() { |
| 1935 | var c []*simHost |
| 1936 | for _, h := range s.hosts { |
| 1937 | m, ok := s.book.HoldRanOn(h.name) |
| 1938 | if h.phase == simAsleep && ok && m.Build == s.installed[h.ch] && s.published(m.Build, h.name) != 0 { |
| 1939 | c = append(c, h) |
| 1940 | } |
| 1941 | } |
| 1942 | if len(c) == 0 { |
| 1943 | return |
| 1944 | } |
| 1945 | h := c[s.gr.Intn(len(c))] |
| 1946 | x := s.installed[h.ch] |
| 1947 | s.issueGrant(h, x) |
| 1948 | s.rewake = h.name |
| 1949 | if _, recorded := s.book.Lookup("bab", "blog", x); !recorded { |
| 1950 | var o []*simHost |
| 1951 | for _, h2 := range s.hosts { |
| 1952 | if h2 != h && h2.ch == h.ch && h2.phase == simAsleep && !h2.served[x] { |
| 1953 | o = append(o, h2) |
| 1954 | } |
| 1955 | } |
| 1956 | if len(o) > 0 { |
| 1957 | s.rewake, s.rewakeFail = o[s.gr.Intn(len(o))].name, x |
| 1958 | } |
| 1959 | } |
| 1960 | s.logf("rewake %s: retry pressed under its hold-ran marker; %s wakes next", h.name, s.rewake) |
| 1961 | } |
| 1962 | |
| 1963 | // rewakeRun wakes the host rewakeArm chose, if it is still asleep: its plan |
| 1964 | // and exec, then its stop while the first wake is under observation. Then |
| 1965 | // the agent reads its grants. |
| 1966 | func (s *sim) rewakeRun() { |
| 1967 | h, fail := s.byName[s.rewake], s.rewakeFail |
| 1968 | s.rewake, s.rewakeFail = "", "" |
| 1969 | if h.phase != simAsleep { |
| 1970 | return |
| 1971 | } |
| 1972 | s.wakeBeginOn(h) |
| 1973 | if h.phase == simWaking { |
| 1974 | s.wakeExecOn(h, fail) |
| 1975 | } |
| 1976 | if _, inSwap := s.swaps[h.name]; h.phase == simObserving && !inSwap { |
| 1977 | s.sleepOn(h) |
| 1978 | } |
| 1979 | s.deliverGrants() |
| 1980 | } |
| 1981 | |
| 1982 | // issueGrant is the console signing `carlos instances retry` for h onto |
| 1983 | // build b ("" picks one of heldBuilds), on a seed whose book is on disk: |
| 1984 | // it reads, as status does, the (book, generation) the box publishes for |
| 1985 | // b's restriction of h, and queues a grant for it, stamped with the step. |
| 1986 | // A build the box publishes no generation for gets no grant (the console |
| 1987 | // refuses the retry). |
| 1988 | func (s *sim) issueGrant(h *simHost, b string) { |
| 1989 | if b == "" { |
| 1990 | builds := s.heldBuilds(h) |
| 1991 | b = builds[s.gr.Intn(len(builds))] |
| 1992 | } |
| 1993 | epoch := s.published(b, h.name) |
| 1994 | if epoch == 0 { |
| 1995 | return |
| 1996 | } |
| 1997 | g := simGrant{build: b, host: h.name, book: s.book.ID(), epoch: epoch, step: s.step, at: s.now, cap: s.tseq} |
| 1998 | // Pending for the rest of the run: a grant expires after 24 h, and a |
| 1999 | // run is minutes long. |
| 2000 | s.grants = append(s.grants, g) |
| 2001 | s.logf("grant: instances retry %s onto %s issued (book %.8s, generation %d, transition %d)", h.name, b, g.book, epoch, g.cap) |
| 2002 | } |
| 2003 | |
| 2004 | // deliverGrants is the agent reading its pending grants (internal/host, |
| 2005 | // host-retry.json): each one, oldest first, through RetryHost, which applies |
| 2006 | // it with RetryHostAt. A grant spent or stale clears nothing (the plan's |
| 2007 | // ruling: grants are never skipped, only refused), so every pending one is |
| 2008 | // delivered again at every read, before and after restarts and rebuilds. |
| 2009 | func (s *sim) deliverGrants() { |
| 2010 | for _, g := range slices.Clone(s.grants) { |
| 2011 | s.deliver(g) |
| 2012 | } |
| 2013 | } |
| 2014 | |
| 2015 | // deliver applies g. When it clears anything, I9 is checked first, then the |
| 2016 | // model follows what it cleared: a record that now lists the host as |
| 2017 | // retried (exempt, and the retry's place in the ordering), a marker of g's |
| 2018 | // build gone (the hold-ran restriction lifted), each an applied clear |
| 2019 | // (sim.clears). |
| 2020 | func (s *sim) deliver(g simGrant) { |
| 2021 | h := s.byName[g.host] |
| 2022 | rec, recorded := s.book.Lookup("bab", "blog", g.build) |
| 2023 | wasRetried := recorded && slices.Contains(rec.Retried, g.host) |
| 2024 | mk, marked := s.book.HoldRanOn(g.host) |
| 2025 | marked = marked && mk.Build == g.build |
| 2026 | if !s.m.RetryHost(g.host, g.build, g.book, g.epoch) { |
| 2027 | return |
| 2028 | } |
| 2029 | s.checkI9(g) |
| 2030 | after, still := s.book.Lookup("bab", "blog", g.build) |
| 2031 | recCleared := recorded && still && !wasRetried && slices.Contains(after.Retried, g.host) |
| 2032 | _, markStill := s.book.HoldRanOn(g.host) |
| 2033 | if s.retryAt[g.host] == nil { |
| 2034 | s.retryAt[g.host] = map[string]time.Time{} |
| 2035 | } |
| 2036 | s.retryAt[g.host][g.build] = s.now |
| 2037 | if recCleared { |
| 2038 | if s.exempt[g.host] == nil { |
| 2039 | s.exempt[g.host] = map[string]bool{} |
| 2040 | } |
| 2041 | s.exempt[g.host][g.build] = true |
| 2042 | delete(s.heldDown[g.host], g.build) |
| 2043 | if r := s.grec[g.build]; r != nil { |
| 2044 | r.retried[g.host] = true |
| 2045 | delete(r.down, g.host) |
| 2046 | } |
| 2047 | s.addClear(simClear{build: g.build, host: g.host, kind: "a grant's retry"}) |
| 2048 | } |
| 2049 | if marked && !markStill { |
| 2050 | s.addClear(simClear{build: g.build, host: g.host, marker: true, kind: "a grant's marker clear"}) |
| 2051 | if s.holdRan[h.name] == g.build { |
| 2052 | delete(s.holdRan, h.name) |
| 2053 | } |
| 2054 | } |
| 2055 | s.logf("grant: instances retry %s onto %s from step %d (generation %d) cleared it (record %v, marker %v)", |
| 2056 | g.host, g.build, g.step, g.epoch, recCleared, marked && !markStill) |
| 2057 | } |
| 2058 | |
| 2059 | // checkI9 is I9 at a grant the code has just accepted (it cleared |
| 2060 | // something): no fact of its (build, host) landed after the grant's capture |
| 2061 | // (a seq above g.cap), unless a clear of that restriction was applied after |
| 2062 | // the fact (discharged). A new record is a fact of every host of the app |
| 2063 | // (simFact.app), so one made by a failure on another host counts too, |
| 2064 | // discharged only by a retry of g's build on g's host or the record leaving |
| 2065 | // the book. Facts are ordered by when they landed, so a late outcome and a |
| 2066 | // change in the grant's own step after its issue are inside the check, and |
| 2067 | // one that landed before the issue, in the same step or earlier, is not: |
| 2068 | // the published generation the grant names already held it. |
| 2069 | func (s *sim) checkI9(g simGrant) { |
| 2070 | for _, f := range s.facts { |
| 2071 | if f.build != g.build || (!f.app && f.host != g.host) || f.seq <= g.cap || s.discharged(f, g.host) { |
| 2072 | continue |
| 2073 | } |
| 2074 | what := "" |
| 2075 | if f.app { |
| 2076 | what = " (a new record, which holds every host)" |
| 2077 | } |
| 2078 | violate("I9", "a grant for %s on %s issued at step %d (%s, book %.8s, generation %d, transition %d) cleared it after %s of %s on %s%s (transition %d, an execution begun at %s), which no applied clear has discharged", |
| 2079 | g.build, g.host, g.step, g.at.Sub(t0), g.book, g.epoch, g.cap, f.kind, f.build, f.host, what, f.seq, f.start.Sub(t0)) |
| 2080 | } |
| 2081 | } |
| 2082 | |
| 2083 | // published is PublishedEpoch for build on host, as status reads it, with |
| 2084 | // I10 checked on every non-zero value. |
| 2085 | func (s *sim) published(build, host string) uint64 { |
| 2086 | e := s.book.PublishedEpoch("bab", "blog", build, host) |
| 2087 | if e != 0 && s.dir != "" { |
| 2088 | s.checkI10At(build, host, e) |
| 2089 | } |
| 2090 | return e |
| 2091 | } |
| 2092 | |
| 2093 | // checkI10 is I10 after every step, on a seed whose book is on disk: what |
| 2094 | // status would publish now for each host (its installed build, and its |
| 2095 | // hold-ran marker's) is on disk. |
| 2096 | func (s *sim) checkI10() { |
| 2097 | if s.dir == "" { |
| 2098 | return |
| 2099 | } |
| 2100 | for _, h := range s.hosts { |
| 2101 | s.published(s.installed[h.ch], h.name) |
| 2102 | if m, ok := s.book.HoldRanOn(h.name); ok && m.Build != s.installed[h.ch] { |
| 2103 | s.published(m.Build, h.name) |
| 2104 | } |
| 2105 | } |
| 2106 | } |
| 2107 | |
| 2108 | // checkI10At is I10 at one published value: the book's file, read now, is |
| 2109 | // the last save that landed, and it must hold epoch as the app's generation, |
| 2110 | // under the identity a grant for it would name. |
| 2111 | func (s *sim) checkI10At(build, host string, epoch uint64) { |
| 2112 | f, err := s.diskBook() |
| 2113 | if err != nil { |
| 2114 | violate("I10", "%s on %s published generation %d, and the book's file cannot be read: %v", build, host, epoch, err) |
| 2115 | } |
| 2116 | id := s.book.ID() |
| 2117 | if got := f.Gens[genKey("bab", "blog")]; got != epoch || f.Book != id { |
| 2118 | violate("I10", "%s on %s published generation %d under book %.8s, and the last landed file holds generation %d under book %.8s (saves failing: %v)", |
| 2119 | build, host, epoch, id, got, f.Book, s.fault) |
| 2120 | } |
| 2121 | } |
| 2122 | |
| 2123 | // diskBook reads the book's file as the agent would at its start. A missing |
| 2124 | // file is an empty book. |
| 2125 | func (s *sim) diskBook() (failureFile, error) { |
| 2126 | var f failureFile |
| 2127 | raw, err := os.ReadFile(filepath.Join(s.dir, FailureFile)) |
| 2128 | if errors.Is(err, os.ErrNotExist) { |
| 2129 | return f, nil |
| 2130 | } |
| 2131 | if err != nil { |
| 2132 | return f, err |
| 2133 | } |
| 2134 | err = json.Unmarshal(raw, &f) |
| 2135 | return f, err |
| 2136 | } |
| 2137 | |
| 2138 | // agentRestart is the agent process ending and starting again, on a seed |
| 2139 | // whose book is on disk (I9). Everything in flight goes with it. Each |
| 2140 | // attempt parked in Swap stays parked until the seed ends, when its |
| 2141 | // goroutine exits from Swap (sim.dead) without returning, so nothing of the |
| 2142 | // old Manager touches the book again, as a killed process cannot. A healthy |
| 2143 | // event on its way is lost, and every instance stops, the agent's children |
| 2144 | // with it. A window of failing saves closes first: an agent that cannot save |
| 2145 | // its book at startup refuses managed starts (internal/host), so the box |
| 2146 | // runs again only once the disk takes a save. The new agent opens the book |
| 2147 | // from disk (OpenFailureBook, which reconciles the intents left there), |
| 2148 | // saves it, and starts a Manager with it. |
| 2149 | // |
| 2150 | // rebuild is a box rebuilt: the book's file is removed first, so the new |
| 2151 | // book is empty under a new identity, and the model forgets what the old |
| 2152 | // book held (forgetBook). Grants and failure facts outlive both. |
| 2153 | func (s *sim) agentRestart(rebuild bool) { |
| 2154 | s.cancel() |
| 2155 | dropped := len(s.swaps) |
| 2156 | s.swaps = map[string]*simSwap{} |
| 2157 | s.late = map[string]simLate{} |
| 2158 | for _, h := range s.hosts { |
| 2159 | h.phase, h.gen, h.plan, h.planRoute, h.failLater, h.underMark = simAsleep, activator.Gen{}, activator.WakePlan{}, registry.Route{}, false, false |
| 2160 | } |
| 2161 | if s.fault { |
| 2162 | if err := os.Chmod(s.dir, 0o700); err != nil { |
| 2163 | panic(simCrash{msg: "the book's directory: " + err.Error()}) |
| 2164 | } |
| 2165 | s.fault = false |
| 2166 | } |
| 2167 | if rebuild { |
| 2168 | if err := os.Remove(filepath.Join(s.dir, FailureFile)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 2169 | panic(simCrash{msg: "remove the book: " + err.Error()}) |
| 2170 | } |
| 2171 | s.forgetBook() |
| 2172 | } else { |
| 2173 | s.reread() |
| 2174 | } |
| 2175 | // The retry ordering is in memory only (FailureBook.cleared): no |
| 2176 | // execution from before the restart is left for it to order. |
| 2177 | s.retryAt, s.clearedAt = map[string]map[string]time.Time{}, map[string]time.Time{} |
| 2178 | // A clear the book had not saved is rolled back, and the model keeps |
| 2179 | // none across a restart (sim.tseq): no earlier clear discharges a fact. |
| 2180 | s.clears = nil |
| 2181 | b, err := OpenFailureBook(s.dir) |
| 2182 | if err != nil { |
| 2183 | panic(simCrash{msg: "reopen the book: " + err.Error()}) |
| 2184 | } |
| 2185 | b.now = func() time.Time { return s.now } |
| 2186 | if err := b.Flush(); err != nil { |
| 2187 | panic(simCrash{msg: "save the reopened book: " + err.Error()}) |
| 2188 | } |
| 2189 | s.book = b |
| 2190 | s.logf("agent restart (box rebuilt %v): %d attempt(s) dropped, book %.8s", rebuild, dropped, b.ID()) |
| 2191 | s.startAgent() |
| 2192 | } |
| 2193 | |
| 2194 | // reread is the model's side of a restart's open (agentRestart). A crash |
| 2195 | // rolls back what the book held in memory and never saved: a clear made |
| 2196 | // while saves failed (a host retry's, a good write's marker clear) is undone |
| 2197 | // on disk, and the restriction is back. So the model rereads, as the agent |
| 2198 | // does, the restrictions on disk: each record's retried and held-down hosts |
| 2199 | // (exempt, heldDown, its own record grec) and the hold-ran markers. Nothing |
| 2200 | // is lost the other way: a failure or a marker set while saves failed has |
| 2201 | // its execution's intent on disk still (its End failed too), and the open |
| 2202 | // turns each intent into the failure it may have been, which the model |
| 2203 | // takes as a failure here (failureOf, reconciled), and as a hold run when |
| 2204 | // the intent was a hold's. |
| 2205 | func (s *sim) reread() { |
| 2206 | f, err := s.diskBook() |
| 2207 | if err != nil { |
| 2208 | panic(simCrash{msg: "read the book's file: " + err.Error()}) |
| 2209 | } |
| 2210 | s.grec = map[string]*simRec{} |
| 2211 | for _, r := range f.Records { |
| 2212 | g := &simRec{retried: map[string]bool{}, down: map[string]bool{}} |
| 2213 | s.grec[r.Build] = g |
| 2214 | for _, h := range s.hosts { |
| 2215 | delete(s.exempt[h.name], r.Build) |
| 2216 | } |
| 2217 | for _, host := range r.Retried { |
| 2218 | g.retried[host] = true |
| 2219 | if s.exempt[host] == nil { |
| 2220 | s.exempt[host] = map[string]bool{} |
| 2221 | } |
| 2222 | s.exempt[host][r.Build] = true |
| 2223 | } |
| 2224 | for _, host := range r.HeldDown { |
| 2225 | g.down[host] = true |
| 2226 | if g.retried[host] { |
| 2227 | continue |
| 2228 | } |
| 2229 | if s.heldDown[host] == nil { |
| 2230 | s.heldDown[host] = map[string]bool{} |
| 2231 | } |
| 2232 | s.heldDown[host][r.Build] = true |
| 2233 | } |
| 2234 | } |
| 2235 | for _, h := range s.hosts { |
| 2236 | m, ok := f.HoldRan[h.name] |
| 2237 | switch { |
| 2238 | case !ok: |
| 2239 | delete(s.holdRan, h.name) |
| 2240 | case s.holdRan[h.name] != m.Build: |
| 2241 | // A marker the crash put back (its clear was never saved). |
| 2242 | s.holdRan[h.name], s.holdRanAt[h.name], s.holdRanSet[h.name] = m.Build, m.At, s.now |
| 2243 | } |
| 2244 | } |
| 2245 | for _, host := range sortedKeys(f.Intents) { |
| 2246 | in := f.Intents[host] |
| 2247 | h := s.byName[host] |
| 2248 | hold := in.Mode == string(activator.ModeHold) |
| 2249 | s.logf("restart: %s's intent %s (%s, %s) is a failure now", host, in.ID, in.Build, in.Mode) |
| 2250 | s.failureOf(in.Build, host, s.now, hold, in.Start, true) |
| 2251 | if hold { |
| 2252 | start := in.Start |
| 2253 | if start.IsZero() { |
| 2254 | start = s.now |
| 2255 | } |
| 2256 | s.holdRanOn(h, in.Build, start) |
| 2257 | } |
| 2258 | } |
| 2259 | // Every restriction standing now was set before the open, and so before |
| 2260 | // anything the new agent starts (its snapshots are newer than every |
| 2261 | // marker's Seq): a good write by an execution starting in this very |
| 2262 | // step lifts it, as the code's ClearHoldRanBefore does. |
| 2263 | for _, host := range sortedKeys(s.holdRan) { |
| 2264 | if !s.holdRanSet[host].Before(s.now) { |
| 2265 | s.holdRanSet[host] = s.now.Add(-time.Nanosecond) |
| 2266 | } |
| 2267 | } |
| 2268 | } |
| 2269 | |
| 2270 | // forgetBook is the model's side of a box rebuild (agentRestart): the book |
| 2271 | // is gone, and with it every failure, retry, hold-down and hold-ran |
| 2272 | // restriction the model held for it. The facts and grants stay. |
| 2273 | func (s *sim) forgetBook() { |
| 2274 | s.failedEver, s.lastStamp, s.failedAt = map[string]bool{}, map[string]time.Time{}, map[string]time.Time{} |
| 2275 | s.exempt, s.heldDown = map[string]map[string]bool{}, map[string]map[string]bool{} |
| 2276 | s.holdRan, s.holdRanAt, s.holdRanSet = map[string]string{}, map[string]time.Time{}, map[string]time.Time{} |
| 2277 | s.grec = map[string]*simRec{} |
| 2278 | } |
| 2279 | |
| 2280 | // restage is root restaging a build whose staged file was wrong. |
| 2281 | func (s *sim) restage() { |
| 2282 | for _, b := range sortedKeys(s.builds) { |
| 2283 | if s.builds[b].misstaged { |
| 2284 | s.builds[b].misstaged = false |
| 2285 | s.logf("restaged %s", b) |
| 2286 | return |
| 2287 | } |
| 2288 | } |
| 2289 | } |
| 2290 | |
amadan