| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "crypto/rand" |
| 5 | "encoding/hex" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "maps" |
| 10 | "math" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "slices" |
| 14 | "strings" |
| 15 | "sync" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/activator" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | // FailureFile is the failure records' home in the agent's data dir (spec |
| 23 | // §8.4): on disk, so a record survives an agent restart. |
| 24 | const FailureFile = "swap-failures.json" |
| 25 | |
| 26 | // FailureRecord holds one build off this box for instances that have not |
| 27 | // served it (spec §8.4). Its scope is (Account, App, Build) on this box; |
| 28 | // Host is the instance whose attempt failed, kept for status. |
| 29 | type FailureRecord struct { |
| 30 | Account string `json:"account"` |
| 31 | App string `json:"app"` |
| 32 | Build string `json:"build"` |
| 33 | Host string `json:"host"` |
| 34 | Cause string `json:"cause"` |
| 35 | Mode string `json:"mode"` |
| 36 | Reason string `json:"reason"` |
| 37 | // PromotedAt is the PromotedAt of the channel pointer the build failed |
| 38 | // under (spec rev 14 §8.4). A pointer naming the same build with a newer |
| 39 | // one, which only `carlos promote --retry` or a fresh promotion can sign, |
| 40 | // clears the record: the refusal records' rule (internal/release/ |
| 41 | // refusal.go), so the two kinds of record behave alike. |
| 42 | PromotedAt time.Time `json:"promoted_at"` |
| 43 | At time.Time `json:"at"` |
| 44 | // HeldDown are instances where a hold build executed and failed (spec |
| 45 | // §6.4, §8.3): nothing falls back automatically on them, so their wakes |
| 46 | // are refused until a newer build or a retry. |
| 47 | HeldDown []string `json:"held_down,omitempty"` |
| 48 | // Retried are instances `carlos instances retry` cleared: the record no |
| 49 | // longer holds them, and their next attempt is a canary. |
| 50 | Retried []string `json:"retried,omitempty"` |
| 51 | // Epochs are the per-record epochs an agent before the crash-epoch hard |
| 52 | // limit (readiness round 2, Task 1) wrote: the book's sequence values at |
| 53 | // which the record last changed. They are read only so a reopen's |
| 54 | // sequence starts past them (OpenFailureBook), then dropped, and never |
| 55 | // written again: what a signed host retry names now is the app's |
| 56 | // generation (FailureBook.gens). |
| 57 | Epochs map[string]uint64 `json:"epochs,omitempty"` |
| 58 | // MarkHoldRan asks Record to set Host's hold-ran marker onto this |
| 59 | // record's build in the same save (final review wave 2, N2): an attempt |
| 60 | // that failed in hold mode, ran its build, and was not held down. It is |
| 61 | // an instruction to Record, never stored in the record. |
| 62 | MarkHoldRan bool `json:"-"` |
| 63 | // HoldRanAt is the marker's At for MarkHoldRan: when the attempt that |
| 64 | // ran the build started (its dispatch). An instruction too, never stored. |
| 65 | HoldRanAt time.Time `json:"-"` |
| 66 | // HoldRanExec is the marker's Exec: the identity of the execution that |
| 67 | // failed (HoldRan.Exec). An instruction, never stored. |
| 68 | HoldRanExec string `json:"-"` |
| 69 | // Seq is the book's snapshot (FailureBook.Snapshot) taken when the |
| 70 | // failing execution started; zero when it is not known. Record compares |
| 71 | // it with the retries since (Astra ruling 191, retry ordering): a failure |
| 72 | // of an execution that began before a retry never takes the retry back. |
| 73 | // An instruction, never stored. |
| 74 | Seq uint64 `json:"-"` |
| 75 | } |
| 76 | |
| 77 | // RecordResult is what Record did with a failure. failCohort runs the rest |
| 78 | // of the cohort-failure transition (the halt and the supersession) for every |
| 79 | // result but PreRetry (plan review finding 3). |
| 80 | type RecordResult int |
| 81 | |
| 82 | const ( |
| 83 | // Recorded: the build had no record; this failure is its first. |
| 84 | Recorded RecordResult = iota |
| 85 | // Merged: a record stood, and this failure changed it (its host's retry |
| 86 | // spent, a hold-down added, a newer PromotedAt). |
| 87 | Merged |
| 88 | // Unchanged: a record stood, and this failure added nothing to it. |
| 89 | Unchanged |
| 90 | // PreRetry: the failing execution began before a retry that has since |
| 91 | // cleared the build for its host or for the app (`carlos instances |
| 92 | // retry`, `carlos promote --retry`). It is stale: nothing of the record |
| 93 | // changes, and no halt or supersession may follow it. A hold-ran marker |
| 94 | // it carries, or one in place of its hold-down, is still set (Record). |
| 95 | PreRetry |
| 96 | ) |
| 97 | |
| 98 | func (r RecordResult) String() string { |
| 99 | switch r { |
| 100 | case Recorded: |
| 101 | return "recorded" |
| 102 | case Merged: |
| 103 | return "merged" |
| 104 | case Unchanged: |
| 105 | return "unchanged" |
| 106 | case PreRetry: |
| 107 | return "pre-retry" |
| 108 | } |
| 109 | return fmt.Sprintf("RecordResult(%d)", int(r)) |
| 110 | } |
| 111 | |
| 112 | // HoldRan is an instance's "hold ran" marker (final review wave 2, N2): a |
| 113 | // build that executed on it in hold mode and neither passed nor was held |
| 114 | // down there (an attempt superseded or stopped after its exec, a first wake |
| 115 | // whose observation was cancelled). It may have migrated the instance's |
| 116 | // database, so nothing falls back on that instance automatically (spec |
| 117 | // §6.4): while the marker stands the planner never boots the good build |
| 118 | // (decideWake, planCheck), and a worker's replacement carries no fallback |
| 119 | // onto it (executeWorker, Astra R3 N5). It is a restriction on the |
| 120 | // instance, not a failure of the build: it blocks no sibling and halts no |
| 121 | // cohort, and it lifts no failure record either (Astra R3 N7). A good-build |
| 122 | // write decided after the marker was set (a pass whose execution began after |
| 123 | // it, Astra R3 N6), a signed `carlos instances retry` naming the marker's |
| 124 | // build (RetryHostAt, plan review finding 5), or a signed rollback pointer |
| 125 | // newer than the marker (applyPointers, ruling 205) clears it. |
| 126 | type HoldRan struct { |
| 127 | Account string `json:"account"` |
| 128 | App string `json:"app"` |
| 129 | Build string `json:"build"` |
| 130 | // Exec is the identity of the execution that set the marker: |
| 131 | // "a<incarnation>-<n>" for a scheduled attempt (dispatch.ExecID), |
| 132 | // "w<incarnation>-<token>" for a planned wake (Manager.wakeExecID). The |
| 133 | // incarnation is the agent process's, so a run after a restart is never |
| 134 | // taken for one before it (final review round 1, finding 1); a file an |
| 135 | // earlier agent wrote holds "a<n>" or "w<token>", which no identity |
| 136 | // handed out now equals. A marker belongs to one execution, not to its build (plan review |
| 137 | // finding 4): another run of the same build renews the marker's Seq, so |
| 138 | // a good write decided between the two runs clears nothing. Empty when |
| 139 | // the caller could not tell, which counts as a new execution every time. |
| 140 | Exec string `json:"exec,omitempty"` |
| 141 | // Seq is the book's sequence when the marker was set (markLocked), so a |
| 142 | // good write clears only markers set before its execution began |
| 143 | // (ClearHoldRanBefore, Astra R3 N6). Zero in a file an earlier agent |
| 144 | // wrote, which any snapshot is newer than. |
| 145 | Seq uint64 `json:"seq,omitempty"` |
| 146 | // At is when the execution that set the marker started: an attempt's |
| 147 | // dispatch, a planned wake's plan (Task 3 fix round 1). Only a rollback |
| 148 | // pointer signed after it lifts the marker (ruling 205, Astra R3): a |
| 149 | // pointer from before the hold build started says nothing about |
| 150 | // recovering from it, while `carlos rollback` signed during the |
| 151 | // execution (the rollback that superseded it) does. It is not the time |
| 152 | // the marker was recorded: that comes after the supersession, so the |
| 153 | // rollback that caused it would always be older. Zero in a file an |
| 154 | // earlier agent wrote, which any signed rollback pointer is newer than. |
| 155 | At time.Time `json:"at,omitempty"` |
| 156 | } |
| 157 | |
| 158 | // same reports whether h and o mark the same build of the same app. |
| 159 | func (h HoldRan) same(o HoldRan) bool { |
| 160 | return h.Account == o.Account && h.App == o.App && h.Build == o.Build |
| 161 | } |
| 162 | |
| 163 | // Intent is an execution that may leave a failure behind, written to the |
| 164 | // book before the execution runs anything (Astra R1 #5, R3 §5 item 2): an |
| 165 | // adoption attempt, or a planned first wake. Its outcome's own book write |
| 166 | // (Record, MarkHoldRan) comes later, and End removes the intent only after |
| 167 | // it. A failure held only in memory because its save failed, followed by an |
| 168 | // agent restart, therefore still finds the intent on disk, and |
| 169 | // OpenFailureBook turns it into the failure record it may have been. |
| 170 | type Intent struct { |
| 171 | Account string `json:"account"` |
| 172 | App string `json:"app"` |
| 173 | Build string `json:"build"` |
| 174 | // Mode is the mode the execution runs in: the effective one, after the |
| 175 | // activator's own overrides (an overlap attempt on a foreign-layout row |
| 176 | // runs as a hold, plan review finding 2), so a reconciled hold intent |
| 177 | // holds the instance down. |
| 178 | Mode string `json:"mode"` |
| 179 | // ID is the execution's identity (HoldRan.Exec): "a<incarnation>-<n>" |
| 180 | // for an attempt (dispatch.ExecID), "w<incarnation>-<token>" for a |
| 181 | // planned wake (Manager.wakeExecID). A reconciled hold intent's marker carries it. |
| 182 | ID string `json:"id"` |
| 183 | // Start is when the execution started (dispatch.Started, a wake's plan |
| 184 | // time): a reconciled hold intent's marker carries it as its At (ruling |
| 185 | // 205). Zero when not known; reconciliation then uses the open time. |
| 186 | Start time.Time `json:"start"` |
| 187 | } |
| 188 | |
| 189 | // wakeIntent reports whether id is a planned wake's intent: "w" leads a |
| 190 | // wake's identity, in this agent's form and an earlier one's alike. |
| 191 | func wakeIntent(id string) bool { return strings.HasPrefix(id, "w") } |
| 192 | |
| 193 | // ErrIntentStands is Begin's refusal while another execution's intent stands |
| 194 | // for the host: an attempt's, or a wake's that an attempt would replace. Its |
| 195 | // outcome is not yet written, so replacing its intent would lose it across a |
| 196 | // failed save and a restart. |
| 197 | var ErrIntentStands = errors.New("another execution on this instance has not recorded its outcome yet") |
| 198 | |
| 199 | // ErrSeqExhausted is every save's answer once the book's sequence has run |
| 200 | // out (seqLimit, fix wave 1, item 5): the book is never saved or published |
| 201 | // again, so no generation repeats under its identity. Rotating the book's |
| 202 | // identity alone does not reset an exhausted sequence: a reopen keeps the |
| 203 | // highest saved sequence (highestSaved), so the next allocation exhausts |
| 204 | // again. There is no supported reset yet; an exhausted book refuses managed |
| 205 | // starts on an armed box, as an unreadable book does. The file must not be |
| 206 | // deleted: it holds the failure records and hold-ran markers that keep |
| 207 | // instances off builds that failed or ran on them. Reaching the limit takes |
| 208 | // about 1.8e19 book events, so this is a guard, not an operating condition. |
| 209 | var ErrSeqExhausted = errors.New("rollout: the failure book's sequence is exhausted; it is never saved or published again") |
| 210 | |
| 211 | // reconciledReason is a reconciled intent's Reason. reasonWord reads it as |
| 212 | // a failed health check (status.ReasonHealth): the execution's verdict was |
| 213 | // never recorded, which is all the box knows. |
| 214 | const reconciledReason = "the agent restarted before this execution recorded its outcome" |
| 215 | |
| 216 | // failureSchema is the schema the book is written as, whatever the file was, |
| 217 | // so an older agent refuses the file rather than read it and break what the |
| 218 | // newer one keeps. Schema 2 added the intents (Task 5): an agent before it |
| 219 | // accepts only schema 1 and would drop markers and intents. Schema 3 is the |
| 220 | // crash-epoch hard limit (readiness round 2, Task 1; FailureBook's doc): an |
| 221 | // agent before it accepts schema 1 and 2, and would keep the book's |
| 222 | // identity, publish epochs it never saved and drop durable_epochs, so a |
| 223 | // grant it issued could later match a generation this rule hands out. Either |
| 224 | // way an agent downgrade with the flag on refuses managed starts until the |
| 225 | // file is upgraded or removed. This agent reads schema 1, 2 and 3, and gives |
| 226 | // a file of schema 1 or 2 a new identity (rule 4). |
| 227 | const failureSchema = 3 |
| 228 | |
| 229 | type failureFile struct { |
| 230 | Schema int `json:"schema"` |
| 231 | Records []FailureRecord `json:"records"` |
| 232 | // HoldRan is the hold-ran markers by host. It is additive to schema 1: an |
| 233 | // agent that predates it ignores the key (encoding/json skips unknown |
| 234 | // fields), and a file without it has no markers. |
| 235 | HoldRan map[string]HoldRan `json:"hold_ran,omitempty"` |
| 236 | // Seq is the book's sequence (FailureBook.seq), additive like HoldRan, |
| 237 | // so a snapshot after a restart is newer than every marker before it. |
| 238 | Seq uint64 `json:"seq,omitempty"` |
| 239 | // Intents are the executions in flight by host (Intent), schema 2. |
| 240 | Intents map[string]Intent `json:"intents,omitempty"` |
| 241 | // Book is the book's identity (FailureBook.id), additive like HoldRan: |
| 242 | // 16 random bytes, hex, made when the book is first created, and again |
| 243 | // when a file from before the crash-epoch rule is adopted (Durable). |
| 244 | Book string `json:"book"` |
| 245 | // Gens are the apps' restriction generations (FailureBook.gens), keyed |
| 246 | // account + "/" + app. Additive: an older agent ignores them. |
| 247 | Gens map[string]uint64 `json:"gens,omitempty"` |
| 248 | // Durable says the file was written under the crash-epoch rule (rule 4 |
| 249 | // in FailureBook's doc): every generation it ever published was on disk |
| 250 | // first. A file without it, or of schema 1 or 2, was written by an |
| 251 | // earlier agent, which may have published epochs it never saved, so |
| 252 | // OpenFailureBook gives such a book a new identity. |
| 253 | Durable bool `json:"durable_epochs,omitempty"` |
| 254 | } |
| 255 | |
| 256 | // highestSaved is the largest sequence value the file holds: its own seq, |
| 257 | // every app's generation, every marker's, and every epoch an earlier agent |
| 258 | // wrote into a record. A reopen's sequence starts there (rule 3). In a file |
| 259 | // this rule wrote it is the file's own seq; the max keeps a hand-edited or |
| 260 | // older file from handing out a value one of its entries already holds. |
| 261 | func (f failureFile) highestSaved() uint64 { |
| 262 | n := f.Seq |
| 263 | for _, g := range f.Gens { |
| 264 | n = max(n, g) |
| 265 | } |
| 266 | for _, h := range f.HoldRan { |
| 267 | n = max(n, h.Seq) |
| 268 | } |
| 269 | for _, r := range f.Records { |
| 270 | for _, e := range r.Epochs { |
| 271 | n = max(n, e) |
| 272 | } |
| 273 | } |
| 274 | return n |
| 275 | } |
| 276 | |
| 277 | // randRead is crypto/rand's Read, the one source of a book's identity |
| 278 | // (rule 5). A variable so a test can make it fail. |
| 279 | var randRead = rand.Read |
| 280 | |
| 281 | // newBookID is a book's identity (FailureBook.id): 16 bytes from |
| 282 | // crypto/rand, hex. It never falls back to anything weaker: an identity |
| 283 | // two books could share would let a grant for one clear the other, so a |
| 284 | // read that fails is an error for the caller (OpenFailureBook) to fail on. |
| 285 | // (Since Go 1.24 crypto/rand.Read crashes the process rather than return an |
| 286 | // error, so in production this path is a guard; the seam reaches it.) |
| 287 | func newBookID() (string, error) { |
| 288 | var b [16]byte |
| 289 | if _, err := randRead(b[:]); err != nil { |
| 290 | return "", fmt.Errorf("make the failure book's identity: %w", err) |
| 291 | } |
| 292 | return hex.EncodeToString(b[:]), nil |
| 293 | } |
| 294 | |
| 295 | // genKey is an app's key in FailureBook.gens and failureFile.Gens. |
| 296 | func genKey(acct, app string) string { return acct + "/" + app } |
| 297 | |
| 298 | // FailureBook is the box's failure records. An empty path keeps them in |
| 299 | // memory only, which is what tests and the simulation use. |
| 300 | // |
| 301 | // Records handed out (Lookup, ForApp, List) are copies, inner slices |
| 302 | // included, because a status reader holds them outside the lock while |
| 303 | // RetryHost and Record edit the book. |
| 304 | // |
| 305 | // # The crash-epoch hard limit |
| 306 | // |
| 307 | // A signed `carlos instances retry` grant (release.HostRetryGrant) names a |
| 308 | // book, an app and a generation, which the box's status published, and a |
| 309 | // stale grant must never clear a later failure (Paul's ruling 2026-10-12, |
| 310 | // readiness round 2 Task 1). The book keeps these rules: |
| 311 | // |
| 312 | // 1. One generation per app. gens[acct/app] takes bumpLocked's next value |
| 313 | // (touchLocked) on every change to any restriction of that app: a |
| 314 | // record created, a merge that changes a record, a host retried, a |
| 315 | // record removed (ClearOnPointer, Prune), a marker of the app set, |
| 316 | // renewed or replaced (markLocked; a replaced marker of another app |
| 317 | // moves that app's too) or removed (ClearHoldRan, ClearHoldRanBefore, |
| 318 | // ClearHoldRanOnPointer, RetryHostAt), an intent reconciled into a |
| 319 | // record, and an epoch given at open to an older marker or an app |
| 320 | // without a generation (epochOlderLocked). A call that changes nothing |
| 321 | // (PreRetry, Unchanged, a repeated marker of the same execution) moves |
| 322 | // nothing. The generation of build's restriction of host is the app's |
| 323 | // when build's record blocks host or host's marker names build, else 0 |
| 324 | // (restrictionEpochLocked). Prune drops an app's generation only when |
| 325 | // the app has no record and no marker left; a later restriction of it |
| 326 | // takes a fresh value, larger than every earlier one. |
| 327 | // 2. Durable before use. dirty is true whenever memory differs from the |
| 328 | // last save that landed: every call that changes recs, held, intents, |
| 329 | // gens or seq saves under the same hold of mu (a failed save sets |
| 330 | // dirty), or the open leaves dirty set. PublishedEpoch answers 0 while |
| 331 | // dirty, and RetryHostAt first retries an owed save and clears nothing |
| 332 | // while the book is still dirty. |
| 333 | // 3. Monotonic across restarts. Every landed save writes the in-memory |
| 334 | // seq, which only grows, and a reopen starts from the file's seq, maxed |
| 335 | // with every saved generation, marker and epoch (highestSaved). So the |
| 336 | // disk's seq only grows, every published generation is at most it, and |
| 337 | // every value a later run hands out is larger. |
| 338 | // 4. Adoption rotates the identity. A file of schema 1 or 2, or without |
| 339 | // durable_epochs, written by an earlier agent that may have published |
| 340 | // epochs it never saved, opens with a new identity and owes a save |
| 341 | // (dirty), so nothing is published until the rotation is on disk, and |
| 342 | // every grant issued before names the old identity. The book is written |
| 343 | // as schema 3 (failureSchema), which an earlier agent refuses to open. |
| 344 | // 5. The identity comes from crypto/rand only (newBookID). An open that |
| 345 | // needs one and cannot get one fails. Only a memory-only book, which |
| 346 | // tests and the simulation use, falls back to the clock. |
| 347 | // 6. An owed save is retried on its own: the armed manager's pass and a |
| 348 | // declined box's periodic pass call SaveIfDirty, and a declined box |
| 349 | // saves a dirty book at startup. |
| 350 | // |
| 351 | // Proof sketch: a grant G names book X, app A and generation E, which |
| 352 | // status published, so E was on X's disk (2, 4). RetryHostAt clears only |
| 353 | // when X is this book, memory equals disk, and A's generation now equals E |
| 354 | // with build still restricting host. Within a run every change takes a |
| 355 | // fresh larger value (1), so equality means no change of A's restrictions |
| 356 | // since publication. The values never wrap: past seqLimit the book is |
| 357 | // exhausted and never saved or published again (ErrSeqExhausted). Across a crash the reopened book starts at the disk's |
| 358 | // seq, which is at least E, and hands out larger values (3). A book written |
| 359 | // before the rule has another identity (4), and another book has another |
| 360 | // random identity (5). So G clears only the restriction state it was |
| 361 | // issued for. |
| 362 | // |
| 363 | // The guarantee rests on two assumptions about the file, which the book |
| 364 | // cannot check from inside: |
| 365 | // |
| 366 | // - Only an agent of this rule writes it. An earlier agent would keep the |
| 367 | // identity while publishing epochs it never saved, and write the file |
| 368 | // back without durable_epochs. Schema 3 enforces this against a |
| 369 | // downgrade (failureSchema): an earlier agent refuses the file. A hand |
| 370 | // edit or another tool writing it is outside the guarantee. |
| 371 | // - It is never restored from an older copy. A snapshot, a backup or a |
| 372 | // cloned data dir carries the identity with a lower seq than the disk |
| 373 | // last held, so a reopen from it can hand out again a generation |
| 374 | // already published, and a stale grant naming that value would match |
| 375 | // (rule 3 fails); a clone also gives two books one identity (rule 5 |
| 376 | // fails). Restoring or cloning a data dir must give the copy a new |
| 377 | // identity: drop the file's "book" key, and the open makes one (rule 4). |
| 378 | type FailureBook struct { |
| 379 | write func([]byte) error |
| 380 | path string |
| 381 | mu sync.Mutex |
| 382 | recs []FailureRecord |
| 383 | // held is the hold-ran markers by host (HoldRan), saved with the records. |
| 384 | held map[string]HoldRan |
| 385 | // intents are the executions in flight by host (Intent), saved with the |
| 386 | // records. reconciled is how many intents OpenFailureBook turned into |
| 387 | // records. |
| 388 | intents map[string]Intent |
| 389 | reconciled int |
| 390 | // dirty is set whenever memory differs from the last save that landed |
| 391 | // (rule 2): a save failed, or the open changed something (a rotated |
| 392 | // identity, reconciled intents, epochs given to older entries). Memory is |
| 393 | // the truth meanwhile, and every later mutating call, Prune and |
| 394 | // SaveIfDirty retry the save until it lands; nothing is published or |
| 395 | // retried until then. What keeps a failure across an agent restart before |
| 396 | // that is the execution's intent (Begin), saved before the execution ran |
| 397 | // anything and removed only after its outcome was written (End): a |
| 398 | // restart finds it on disk and records the failure from it (spec §8.4, |
| 399 | // Astra R1 #5). |
| 400 | dirty bool |
| 401 | // now stamps a marker whose caller gave no At (markLocked). Nil means |
| 402 | // time.Now; tests and the simulation set their own clock. |
| 403 | now func() time.Time |
| 404 | // seq orders the book's events that later outcomes are compared with |
| 405 | // (Astra R3 N6, ruling 191): every marker set, every retry clear and |
| 406 | // every app generation (gens) takes the next value, and an execution |
| 407 | // takes a Snapshot (seq+1) when it starts. An outcome is older than an |
| 408 | // event exactly when its snapshot is not greater than the event's value. |
| 409 | // It is saved with the markers and the generations, so snapshots and |
| 410 | // generations after a restart are newer than every one on disk (rule 3). |
| 411 | // It never reaches seqLimit (bumpLocked). The cleared map below is not |
| 412 | // saved; a host retry's clear moves the |
| 413 | // app's generation, which is, so a spent grant stays spent across a |
| 414 | // restart. |
| 415 | seq uint64 |
| 416 | // exhausted is set once a change needed a sequence value past the last |
| 417 | // usable one (bumpLocked): from then on no save lands and nothing is |
| 418 | // published (saveLocked), so memory never publishes a repeated value. |
| 419 | exhausted bool |
| 420 | // gens are the apps' restriction generations by genKey (rule 1), saved |
| 421 | // with the records. A signed host retry names the one it was issued for. |
| 422 | gens map[string]uint64 |
| 423 | // cleared holds, in memory only, the sequence of the latest retry clear |
| 424 | // per (account, app, build, host): `carlos instances retry` per host |
| 425 | // (RetryHost), `carlos promote --retry` app-wide with host "" (a |
| 426 | // ClearOnPointer that cleared a record). Executions in flight do not |
| 427 | // survive an agent restart, so neither need these. Prune drops the |
| 428 | // entries of builds no route installs, with their records. |
| 429 | cleared map[[4]string]uint64 |
| 430 | // id is the book's identity, saved with it (failureFile.Book). Status |
| 431 | // publishes it beside every epoch (HostRollout.HeldBook), the console |
| 432 | // signs it into its retry grant, and RetryHostAt clears only for a grant |
| 433 | // naming it (final review round 1, finding 3, Astra): epochs are a |
| 434 | // counter every book starts alike, so two books (a host relocated, a box |
| 435 | // rebuilt) can give one value to two restrictions, and without the |
| 436 | // identity a grant for one would clear the other. OpenFailureBook sets |
| 437 | // it (rules 4 and 5); a memory-only book makes one on first use |
| 438 | // (idLocked). |
| 439 | id string |
| 440 | } |
| 441 | |
| 442 | // ID is the book's identity (FailureBook.id). |
| 443 | func (b *FailureBook) ID() string { |
| 444 | b.mu.Lock() |
| 445 | defer b.mu.Unlock() |
| 446 | return b.idLocked() |
| 447 | } |
| 448 | |
| 449 | // idLocked is the book's identity. Only a memory-only book (tests, the |
| 450 | // simulation) reaches it without one, since OpenFailureBook always sets |
| 451 | // it; such a book falls back to the clock if crypto/rand fails (rule 5). |
| 452 | func (b *FailureBook) idLocked() string { |
| 453 | if b.id == "" { |
| 454 | id, err := newBookID() |
| 455 | if err != nil { |
| 456 | id = fmt.Sprintf("m%x", time.Now().UnixNano()) |
| 457 | } |
| 458 | b.id = id |
| 459 | } |
| 460 | return b.id |
| 461 | } |
| 462 | |
| 463 | // touchLocked moves (acct, app)'s restriction generation to the book's next |
| 464 | // sequence value (rule 1). The caller holds b.mu and saves. |
| 465 | func (b *FailureBook) touchLocked(acct, app string) { |
| 466 | if b.gens == nil { |
| 467 | b.gens = map[string]uint64{} |
| 468 | } |
| 469 | b.gens[genKey(acct, app)] = b.bumpLocked() |
| 470 | } |
| 471 | |
| 472 | func (b *FailureBook) clock() time.Time { |
| 473 | if b.now != nil { |
| 474 | return b.now() |
| 475 | } |
| 476 | return time.Now() |
| 477 | } |
| 478 | |
| 479 | // cloneRec copies r's host lists so the copy shares nothing with r. A |
| 480 | // record in memory has no Epochs (OpenFailureBook drops them). |
| 481 | func cloneRec(r FailureRecord) FailureRecord { |
| 482 | r.HeldDown = slices.Clone(r.HeldDown) |
| 483 | r.Retried = slices.Clone(r.Retried) |
| 484 | return r |
| 485 | } |
| 486 | |
| 487 | // OpenFailureBook loads the book from dataDir. A file that exists but cannot |
| 488 | // be read is an error, never an empty book: an empty book would move every |
| 489 | // held instance straight back onto the build that failed. Intents left in |
| 490 | // the file become failure records (reconcileLocked, Reconciled), which the |
| 491 | // caller saves with Flush. |
| 492 | // |
| 493 | // The sequence starts at the largest value the file holds (rule 3 in |
| 494 | // FailureBook's doc), so nothing handed out from here repeats a generation |
| 495 | // status published: that was on disk. A book with no file gets a new |
| 496 | // identity, and so does a file of schema 1 or 2 or without durable_epochs |
| 497 | // (rule 4): an earlier agent wrote it and may have published epochs it |
| 498 | // never saved. Either owes a save (dirty), as does an open that reconciled |
| 499 | // intents or gave older entries epochs; nothing is published until it |
| 500 | // lands. An identity crypto/rand cannot make is an error (rule 5): the |
| 501 | // caller treats the book as unreadable. A book this rule wrote opens with |
| 502 | // no save owed unless the open changed something. |
| 503 | func OpenFailureBook(dataDir string) (*FailureBook, error) { |
| 504 | b := &FailureBook{path: filepath.Join(dataDir, FailureFile)} |
| 505 | raw, err := os.ReadFile(b.path) |
| 506 | if errors.Is(err, os.ErrNotExist) { |
| 507 | id, err := newBookID() |
| 508 | if err != nil { |
| 509 | return nil, fmt.Errorf("%s: %w", b.path, err) |
| 510 | } |
| 511 | b.id, b.dirty = id, true |
| 512 | return b, nil |
| 513 | } |
| 514 | if err != nil { |
| 515 | return nil, err |
| 516 | } |
| 517 | var f failureFile |
| 518 | if err := json.Unmarshal(raw, &f); err != nil { |
| 519 | return nil, fmt.Errorf("%s: %w", b.path, err) |
| 520 | } |
| 521 | // Schema 1 is an agent before Task 5 (no intents), 2 an agent before the |
| 522 | // crash-epoch rule; 3 is written now. |
| 523 | if f.Schema != 1 && f.Schema != 2 && f.Schema != failureSchema { |
| 524 | return nil, fmt.Errorf("%s: unknown schema %d", b.path, f.Schema) |
| 525 | } |
| 526 | b.recs = f.Records |
| 527 | b.held = f.HoldRan |
| 528 | b.intents = f.Intents |
| 529 | b.gens = f.Gens |
| 530 | b.id = f.Book |
| 531 | b.seq = f.highestSaved() // the reopen's sequence (rule 3); rolloutsim:M15 |
| 532 | for i := range b.recs { |
| 533 | // An earlier agent's per-record epochs: read for the sequence above, |
| 534 | // never written again, and never what a grant is checked against. |
| 535 | if b.recs[i].Epochs != nil { |
| 536 | b.recs[i].Epochs = nil |
| 537 | b.dirty = true |
| 538 | } |
| 539 | } |
| 540 | if f.Schema < failureSchema || !f.Durable || b.id == "" { |
| 541 | // Adoption of the rule (rule 4): grants issued before name the old |
| 542 | // identity, or none. Only an agent of this rule writes schema 3, so a |
| 543 | // file of an older schema is adopted even if it says durable_epochs. |
| 544 | id, err := newBookID() |
| 545 | if err != nil { |
| 546 | return nil, fmt.Errorf("%s: %w", b.path, err) |
| 547 | } |
| 548 | b.id, b.dirty = id, true |
| 549 | } |
| 550 | b.mu.Lock() |
| 551 | b.epochOlderLocked() |
| 552 | b.reconcileLocked() |
| 553 | b.mu.Unlock() |
| 554 | return b, nil |
| 555 | } |
| 556 | |
| 557 | // epochOlderLocked gives what an earlier agent wrote without a sequence |
| 558 | // value one now: each marker with no Seq (by host, sorted), then each app |
| 559 | // that has a record or a marker and no generation (sorted by key), so two |
| 560 | // opens of the same file hand out the same values. A zero generation is |
| 561 | // never accepted by a signed retry, so without one such a hold could not be |
| 562 | // retried from the console. A marker given a Seq moves its app's generation |
| 563 | // (rule 1). The book then owes a save (dirty), which the caller or the next |
| 564 | // mutating call makes; nothing is written here. The caller holds b.mu. |
| 565 | func (b *FailureBook) epochOlderLocked() { |
| 566 | hosts := make([]string, 0, len(b.held)) |
| 567 | for host, h := range b.held { |
| 568 | if h.Seq == 0 { |
| 569 | hosts = append(hosts, host) |
| 570 | } |
| 571 | } |
| 572 | slices.Sort(hosts) |
| 573 | apps := map[string][2]string{} |
| 574 | for _, host := range hosts { |
| 575 | h := b.held[host] |
| 576 | h.Seq = b.bumpLocked() |
| 577 | b.held[host] = h |
| 578 | apps[genKey(h.Account, h.App)] = [2]string{h.Account, h.App} |
| 579 | } |
| 580 | for _, r := range b.recs { |
| 581 | if b.gens[genKey(r.Account, r.App)] == 0 { |
| 582 | apps[genKey(r.Account, r.App)] = [2]string{r.Account, r.App} |
| 583 | } |
| 584 | } |
| 585 | for _, h := range b.held { |
| 586 | if b.gens[genKey(h.Account, h.App)] == 0 { |
| 587 | apps[genKey(h.Account, h.App)] = [2]string{h.Account, h.App} |
| 588 | } |
| 589 | } |
| 590 | for _, k := range slices.Sorted(maps.Keys(apps)) { |
| 591 | b.touchLocked(apps[k][0], apps[k][1]) |
| 592 | b.dirty = true |
| 593 | } |
| 594 | } |
| 595 | |
| 596 | // reconcileLocked turns each intent read from the file into the failure |
| 597 | // record its execution may have left (rule 3; spec §8.4, Astra R1 #5): its |
| 598 | // build on its host, cause release, the intent's mode, stamped with the open |
| 599 | // time, so only a pointer signed after the restart clears it. A hold intent |
| 600 | // holds the instance down and sets its hold-ran marker too, with the |
| 601 | // intent's identity and start (Task 4), as a hold that went down would have: |
| 602 | // the build may have migrated the database, so nothing falls back there |
| 603 | // automatically. An agent crash mid-execution therefore halts that build's |
| 604 | // cohort until a retry, even when the execution would have passed (the |
| 605 | // plan's ruling). The intents then go from memory and the book is owed a |
| 606 | // save, which the caller makes (Flush); nothing is written here, so a |
| 607 | // read-only open of a book never changes the file. The caller holds b.mu. |
| 608 | func (b *FailureBook) reconcileLocked() { |
| 609 | if len(b.intents) == 0 { |
| 610 | b.intents = nil |
| 611 | return |
| 612 | } |
| 613 | now := b.clock() |
| 614 | hosts := make([]string, 0, len(b.intents)) |
| 615 | for host := range b.intents { |
| 616 | hosts = append(hosts, host) |
| 617 | } |
| 618 | slices.Sort(hosts) // one order for every open of the same file |
| 619 | for _, host := range hosts { |
| 620 | in := b.intents[host] |
| 621 | r := FailureRecord{Account: in.Account, App: in.App, Build: in.Build, Host: host, |
| 622 | Cause: string(activator.CauseRelease), Mode: in.Mode, Reason: reconciledReason, PromotedAt: now, At: now} |
| 623 | if in.Mode == string(activator.ModeHold) { |
| 624 | // A zero start leaves the marker's At to markLocked, which stamps |
| 625 | // it with the book's clock: the open time. |
| 626 | r.HeldDown, r.MarkHoldRan, r.HoldRanExec, r.HoldRanAt = []string{host}, true, in.ID, in.Start |
| 627 | } |
| 628 | b.recordLocked(r) |
| 629 | } |
| 630 | b.reconciled = len(hosts) |
| 631 | b.intents = nil |
| 632 | b.dirty = true |
| 633 | } |
| 634 | |
| 635 | // Reconciled is how many intents OpenFailureBook turned into failure |
| 636 | // records: executions an earlier agent began and never saw recorded. The |
| 637 | // caller logs them and flushes the book (Flush). |
| 638 | func (b *FailureBook) Reconciled() int { |
| 639 | b.mu.Lock() |
| 640 | defer b.mu.Unlock() |
| 641 | return b.reconciled |
| 642 | } |
| 643 | |
| 644 | // Flush saves the book now: after reconciliation, so the records it made |
| 645 | // and the intents it removed reach the disk before anything runs. |
| 646 | func (b *FailureBook) Flush() error { |
| 647 | b.mu.Lock() |
| 648 | defer b.mu.Unlock() |
| 649 | return b.saveLocked() |
| 650 | } |
| 651 | |
| 652 | // SaveIfDirty saves the book when a save is owed (dirty: a save failed, or |
| 653 | // the open changed something), else does nothing and returns nil (rule 6). |
| 654 | // The armed manager's pass and a declined box's periodic pass call it, so a |
| 655 | // book whose disk recovers publishes its generations again without waiting |
| 656 | // for another change, pointer or grant. |
| 657 | func (b *FailureBook) SaveIfDirty() error { |
| 658 | b.mu.Lock() |
| 659 | defer b.mu.Unlock() |
| 660 | return b.retryLocked() |
| 661 | } |
| 662 | |
| 663 | // SaveOwed reports whether the book owes a save (dirty): while it does, |
| 664 | // PublishedEpoch answers 0 and RetryHostAt clears nothing. |
| 665 | func (b *FailureBook) SaveOwed() bool { |
| 666 | b.mu.Lock() |
| 667 | defer b.mu.Unlock() |
| 668 | return b.dirty |
| 669 | } |
| 670 | |
| 671 | // Begin writes host's intent and saves it before the execution runs |
| 672 | // anything (rule 1). On a save error nothing of it stays in memory, the host's |
| 673 | // earlier intent (if this one replaced it) is put back, and the error is |
| 674 | // returned: the caller must not execute, since its failure could then be |
| 675 | // lost across a failed save and a restart. |
| 676 | // |
| 677 | // One intent stands per host. A planned wake's intent replaces an older |
| 678 | // wake's intent for the host: the activator delivers a planned wake's |
| 679 | // outcome before a newer wake of the host plans (its outcome fence, |
| 680 | // internal/activator awaitOutcomes, final review F3 and wave 2 N1), so the |
| 681 | // older wake's intent is one whose outcome is written, or one whose build |
| 682 | // never ran (GenCancelled, not executed), which an event on its way ends |
| 683 | // anyway. Two plans for one host never race: the activator plans a host |
| 684 | // only from its one waking entry. Any other standing intent refuses the |
| 685 | // Begin (ErrIntentStands): an attempt's, whose outcome done() has yet to |
| 686 | // write, or, for an attempt, a wake's, whose outcome may still be on its way |
| 687 | // (a fallen-back wake's entry serves before its GenFellBack is delivered). |
| 688 | // The refused execution is retried as any refused one is. |
| 689 | func (b *FailureBook) Begin(host string, in Intent) error { |
| 690 | if in.ID == "" { |
| 691 | return errors.New("an intent needs its execution's identity") |
| 692 | } |
| 693 | b.mu.Lock() |
| 694 | defer b.mu.Unlock() |
| 695 | prev, had := b.intents[host] |
| 696 | if had && prev.ID != in.ID && !(wakeIntent(prev.ID) && wakeIntent(in.ID)) { |
| 697 | return fmt.Errorf("%s: %w (%s)", host, ErrIntentStands, prev.ID) |
| 698 | } |
| 699 | if b.intents == nil { |
| 700 | b.intents = map[string]Intent{} |
| 701 | } |
| 702 | b.intents[host] = in |
| 703 | if err := b.saveLocked(); err != nil { |
| 704 | if had { |
| 705 | b.intents[host] = prev |
| 706 | } else { |
| 707 | delete(b.intents, host) |
| 708 | } |
| 709 | return err |
| 710 | } |
| 711 | return nil |
| 712 | } |
| 713 | |
| 714 | // End removes host's intent when it is id's, and saves (rule 2). It runs |
| 715 | // after the execution's outcome is written to the book (Record, |
| 716 | // MarkHoldRan), never before. Its save failing leaves the intent on disk, |
| 717 | // which is the safe direction: a restart then records the failure the |
| 718 | // outcome may also have written. An intent of another execution (a newer |
| 719 | // wake that replaced this one's) is left alone. |
| 720 | func (b *FailureBook) End(host, id string) error { |
| 721 | b.mu.Lock() |
| 722 | defer b.mu.Unlock() |
| 723 | if cur, ok := b.intents[host]; !ok || id == "" || cur.ID != id { |
| 724 | return b.retryLocked() |
| 725 | } |
| 726 | delete(b.intents, host) |
| 727 | return b.saveLocked() |
| 728 | } |
| 729 | |
| 730 | // saveLocked writes the book and records whether the file is now behind |
| 731 | // memory, so a failure is retried by retryLocked rather than forgotten. It |
| 732 | // writes the in-memory seq, which only grows (rule 3), the generations, and |
| 733 | // durable_epochs, since everything this agent publishes is on disk first |
| 734 | // (rule 2). |
| 735 | func (b *FailureBook) saveLocked() error { |
| 736 | if b.exhausted { |
| 737 | b.dirty = true |
| 738 | return ErrSeqExhausted |
| 739 | } |
| 740 | if b.path == "" && b.write == nil { |
| 741 | return nil |
| 742 | } |
| 743 | raw, err := json.MarshalIndent(failureFile{Schema: failureSchema, Records: b.recs, HoldRan: b.held, Seq: b.seq, Intents: b.intents, |
| 744 | Book: b.idLocked(), Gens: b.gens, Durable: true}, "", " ") |
| 745 | if err == nil { |
| 746 | if b.write != nil { |
| 747 | err = b.write(raw) |
| 748 | } else { |
| 749 | err = release.WriteFileAtomic(b.path, raw, 0o644) |
| 750 | } |
| 751 | } |
| 752 | b.dirty = err != nil |
| 753 | return err |
| 754 | } |
| 755 | |
| 756 | // retryLocked is the save a call that changed nothing still owes when an |
| 757 | // earlier save failed. |
| 758 | func (b *FailureBook) retryLocked() error { |
| 759 | if !b.dirty { |
| 760 | return nil |
| 761 | } |
| 762 | return b.saveLocked() |
| 763 | } |
| 764 | |
| 765 | func (b *FailureBook) find(acct, app, build string) int { |
| 766 | return slices.IndexFunc(b.recs, func(r FailureRecord) bool { |
| 767 | return r.Account == acct && r.App == app && r.Build == build |
| 768 | }) |
| 769 | } |
| 770 | |
| 771 | // Snapshot is the book's sequence plus one, taken when an execution starts |
| 772 | // (an attempt in scheduler.execute, a planned wake in Plan): every marker |
| 773 | // and retry clear already in the book is older than it, and every one set |
| 774 | // later is not (Astra R3 N6, ruling 191). Zero is never a snapshot: it |
| 775 | // means "not known", which clears no marker and makes no outcome stale. |
| 776 | func (b *FailureBook) Snapshot() uint64 { |
| 777 | b.mu.Lock() |
| 778 | defer b.mu.Unlock() |
| 779 | if b.exhausted || b.seq >= seqLimit-1 { |
| 780 | // At the boundary a snapshot would be seqLimit or more: "not known" |
| 781 | // instead, the safe answer (fix wave 1, item 5). |
| 782 | return 0 |
| 783 | } |
| 784 | return b.seq + 1 |
| 785 | } |
| 786 | |
| 787 | // seqLimit bounds the book's sequence (fix wave 1, item 5): every value it |
| 788 | // hands out, and every snapshot, is below it, so the sequence never wraps |
| 789 | // and no generation repeats under one identity. The last value handed out |
| 790 | // is seqLimit-1 (math.MaxUint64-2). |
| 791 | const seqLimit = math.MaxUint64 - 1 |
| 792 | |
| 793 | // bumpLocked takes the book's next sequence value. At the boundary it hands |
| 794 | // out nothing new: the book is exhausted and owes a save that never lands |
| 795 | // (saveLocked), so nothing is published or cleared again, and the value it |
| 796 | // returns (seqLimit, never a real one) is never seen outside memory. The |
| 797 | // caller holds b.mu. |
| 798 | func (b *FailureBook) bumpLocked() uint64 { |
| 799 | if b.exhausted || b.seq >= seqLimit-1 { |
| 800 | b.exhausted, b.dirty = true, true |
| 801 | return seqLimit |
| 802 | } |
| 803 | b.seq++ |
| 804 | return b.seq |
| 805 | } |
| 806 | |
| 807 | // clearKey is a retry clear's key in FailureBook.cleared; host "" is the |
| 808 | // app-wide clear of `carlos promote --retry`. |
| 809 | func clearKey(acct, app, build, host string) [4]string { |
| 810 | return [4]string{acct, app, build, host} |
| 811 | } |
| 812 | |
| 813 | // preRetryLocked reports whether r's execution began before a retry that |
| 814 | // cleared its build since, for its host or app-wide (ruling 191): its |
| 815 | // snapshot is not newer than the clear. The caller holds b.mu. |
| 816 | func (b *FailureBook) preRetryLocked(r FailureRecord) bool { |
| 817 | if r.Seq == 0 { // rolloutsim:M9 |
| 818 | return false |
| 819 | } |
| 820 | return r.Seq <= b.cleared[clearKey(r.Account, r.App, r.Build, r.Host)] || |
| 821 | r.Seq <= b.cleared[clearKey(r.Account, r.App, r.Build, "")] |
| 822 | } |
| 823 | |
| 824 | // clearedLocked notes a retry clear of build for host ("" app-wide) at the |
| 825 | // book's next sequence value. The caller holds b.mu. |
| 826 | func (b *FailureBook) clearedLocked(acct, app, build, host string) { |
| 827 | if b.cleared == nil { |
| 828 | b.cleared = map[[4]string]uint64{} |
| 829 | } |
| 830 | b.cleared[clearKey(acct, app, build, host)] = b.bumpLocked() |
| 831 | } |
| 832 | |
| 833 | // Record writes r unless a record for its build already stands, in which case |
| 834 | // the first failure stands and only r's host is merged in: it leaves Retried |
| 835 | // (its retry was spent), joins HeldDown if r holds it down, and the record |
| 836 | // keeps the newer of the two PromotedAt stamps, so only a pointer signed |
| 837 | // after the latest failure clears it. The in-memory record is kept even when |
| 838 | // the file write fails, so the caller's transition still blocks the build; |
| 839 | // the error is for the journal, and the next call on the book (this one |
| 840 | // again, if nothing new, or Prune) retries the save. Until a save lands, what |
| 841 | // keeps the failure across an agent restart is the execution's intent |
| 842 | // (Begin), still on disk because its End comes after this call: the restart |
| 843 | // records the failure from it. r.MarkHoldRan sets r's host's hold-ran marker |
| 844 | // in the same save. |
| 845 | // |
| 846 | // A failure of an execution that began before a retry of its build (r.Seq, |
| 847 | // ruling 191) is stale: the record is neither created nor changed, the |
| 848 | // host's retry is not spent and no hold-down is added, and Record reports |
| 849 | // PreRetry so the caller runs none of the transition (plan review finding |
| 850 | // 3). What the execution did to the instance's database still stands: a |
| 851 | // requested hold-ran marker is set, and so is one in place of the hold-down |
| 852 | // r asked for (a hold build that ran there and went down, Task 4 fix round |
| 853 | // 1), with the execution's identity and start (HoldRanExec, HoldRanAt), so |
| 854 | // nothing falls back onto the instance automatically (spec §6.4). A stale |
| 855 | // hold run that went down is restricted as one that was superseded is. |
| 856 | func (b *FailureBook) Record(r FailureRecord) (RecordResult, error) { |
| 857 | b.mu.Lock() |
| 858 | defer b.mu.Unlock() |
| 859 | res, changed := b.recordLocked(r) |
| 860 | if changed { |
| 861 | return res, b.saveLocked() |
| 862 | } |
| 863 | return res, b.retryLocked() |
| 864 | } |
| 865 | |
| 866 | // recordLocked is Record's change to memory, and reports whether it changed |
| 867 | // anything to save. Reconciliation (reconcileLocked) shares it, so a |
| 868 | // reconciled intent merges into a standing record as a live failure does. |
| 869 | // The caller holds b.mu and saves. |
| 870 | func (b *FailureBook) recordLocked(r FailureRecord) (RecordResult, bool) { |
| 871 | stale := b.preRetryLocked(r) |
| 872 | mark := r.MarkHoldRan || (stale && slices.Contains(r.HeldDown, r.Host)) |
| 873 | marked := mark && b.markLocked(r.Host, HoldRan{Account: r.Account, App: r.App, Build: r.Build, Exec: r.HoldRanExec, At: r.HoldRanAt}) |
| 874 | r.MarkHoldRan, r.HoldRanAt, r.HoldRanExec, r.Seq = false, time.Time{}, "", 0 |
| 875 | if stale { |
| 876 | return PreRetry, marked |
| 877 | } |
| 878 | i := b.find(r.Account, r.App, r.Build) |
| 879 | if i < 0 { |
| 880 | r.Retried, r.Epochs = nil, nil |
| 881 | // A new record is a new restriction of every host of the app (rule 1). |
| 882 | b.recs = append(b.recs, cloneRec(r)) |
| 883 | b.touchLocked(r.Account, r.App) // rolloutsim:M17 |
| 884 | return Recorded, true |
| 885 | } |
| 886 | cur := &b.recs[i] |
| 887 | changed := false |
| 888 | if j := slices.Index(cur.Retried, r.Host); j >= 0 { |
| 889 | cur.Retried = slices.Delete(slices.Clone(cur.Retried), j, j+1) |
| 890 | changed = true |
| 891 | } |
| 892 | if r.PromotedAt.After(cur.PromotedAt) { |
| 893 | cur.PromotedAt = r.PromotedAt |
| 894 | changed = true |
| 895 | } |
| 896 | for _, h := range r.HeldDown { |
| 897 | if !slices.Contains(cur.HeldDown, h) { |
| 898 | cur.HeldDown = append(slices.Clone(cur.HeldDown), h) |
| 899 | changed = true |
| 900 | } |
| 901 | } |
| 902 | if changed { |
| 903 | // A merge with a change moves the app's generation, so a grant |
| 904 | // signed before it clears nothing (plan review finding 13, rule 1). |
| 905 | b.touchLocked(r.Account, r.App) |
| 906 | return Merged, true |
| 907 | } |
| 908 | return Unchanged, marked |
| 909 | } |
| 910 | |
| 911 | // markLocked sets host's hold-ran marker to h and reports whether that |
| 912 | // changed anything. h.At is the execution's start, from the caller. A zero |
| 913 | // one (a caller that could not know it) is stamped with the book's clock |
| 914 | // now: later than any real start, so only a pointer signed later still |
| 915 | // lifts the marker, the safe direction. |
| 916 | // |
| 917 | // A marker is an execution's (plan review finding 4). A repeated report of |
| 918 | // the execution that set the standing marker (same build, same non-empty |
| 919 | // Exec) changes nothing and writes nothing. Any other execution gets the |
| 920 | // book's next sequence value, so a good write whose execution began before |
| 921 | // it clears nothing (ClearHoldRanBefore). It keeps the later of the two |
| 922 | // stamps, whatever the builds (Task 4 fix round 1): a late report of an |
| 923 | // older execution never moves At back, so a rollback signed between the two |
| 924 | // runs never looks newer than the marker. |
| 925 | // |
| 926 | // Every marker set, renewed or replaced moves its app's generation (rule |
| 927 | // 1); a marker of another app it replaces (a host now of another app) moves |
| 928 | // that app's too. The caller holds b.mu and saves. |
| 929 | func (b *FailureBook) markLocked(host string, h HoldRan) bool { |
| 930 | cur, ok := b.held[host] |
| 931 | if ok && cur.same(h) && h.Exec != "" && cur.Exec == h.Exec { |
| 932 | return false |
| 933 | } |
| 934 | if b.held == nil { |
| 935 | b.held = map[string]HoldRan{} |
| 936 | } |
| 937 | if h.At.IsZero() { |
| 938 | h.At = b.clock() |
| 939 | } |
| 940 | if ok && cur.At.After(h.At) { |
| 941 | h.At = cur.At |
| 942 | } |
| 943 | h.Seq = b.bumpLocked() // rolloutsim:M12 |
| 944 | if ok && (cur.Account != h.Account || cur.App != h.App) { |
| 945 | b.touchLocked(cur.Account, cur.App) // the replaced marker's app lost a restriction |
| 946 | } |
| 947 | // The marker set, renewed or replaced: its app's generation moves. |
| 948 | b.touchLocked(h.Account, h.App) // rolloutsim:M14 |
| 949 | b.held[host] = h |
| 950 | return true |
| 951 | } |
| 952 | |
| 953 | // MarkHoldRan sets host's hold-ran marker (HoldRan) and saves it: the |
| 954 | // outcome that set it changed nothing else in the book. A marker already |
| 955 | // standing for another build is replaced: the newest hold build to run on |
| 956 | // the instance is the one its database may now need. h.At is the |
| 957 | // execution's start (markLocked). |
| 958 | func (b *FailureBook) MarkHoldRan(host string, h HoldRan) error { |
| 959 | b.mu.Lock() |
| 960 | defer b.mu.Unlock() |
| 961 | if !b.markLocked(host, h) { |
| 962 | return b.retryLocked() |
| 963 | } |
| 964 | return b.saveLocked() |
| 965 | } |
| 966 | |
| 967 | // HoldRanOn returns host's hold-ran marker, if one stands. |
| 968 | func (b *FailureBook) HoldRanOn(host string) (HoldRan, bool) { |
| 969 | b.mu.Lock() |
| 970 | defer b.mu.Unlock() |
| 971 | h, ok := b.held[host] |
| 972 | return h, ok |
| 973 | } |
| 974 | |
| 975 | // HoldRanCount is how many hold-ran markers stand. A box that declined |
| 976 | // CARLOS_SWAP=overlap honours its book when it holds records or markers |
| 977 | // (internal/host, honourDeclinedFailures). |
| 978 | func (b *FailureBook) HoldRanCount() int { |
| 979 | b.mu.Lock() |
| 980 | defer b.mu.Unlock() |
| 981 | return len(b.held) |
| 982 | } |
| 983 | |
| 984 | // ClearHoldRan drops host's hold-ran marker, whatever it is, and reports |
| 985 | // whether a marker stood. A host retry no longer uses it: its grant clears |
| 986 | // only a marker naming the grant's build, at the grant's epoch |
| 987 | // (RetryHostAt, plan review findings 5 and 13). |
| 988 | func (b *FailureBook) ClearHoldRan(host string) (bool, error) { |
| 989 | b.mu.Lock() |
| 990 | defer b.mu.Unlock() |
| 991 | cur, ok := b.held[host] |
| 992 | if !ok { |
| 993 | return false, b.retryLocked() |
| 994 | } |
| 995 | delete(b.held, host) |
| 996 | b.touchLocked(cur.Account, cur.App) |
| 997 | return true, b.saveLocked() |
| 998 | } |
| 999 | |
| 1000 | // ClearHoldRanBefore drops host's hold-ran marker for a good-build write |
| 1001 | // whose execution took snapshot snap (Astra R3 N6): only a marker set before |
| 1002 | // that execution began, which the instance has since passed over. A marker |
| 1003 | // set later is a hold build that ran after the write was decided (the write |
| 1004 | // was delayed, say, between its acknowledgement check and here), and it |
| 1005 | // stands. A zero snapshot (Bootstrap, a wake whose start is not known) |
| 1006 | // clears nothing. The check and the clear are one hold of the lock. |
| 1007 | func (b *FailureBook) ClearHoldRanBefore(host string, snap uint64) (bool, error) { |
| 1008 | b.mu.Lock() |
| 1009 | defer b.mu.Unlock() |
| 1010 | cur, ok := b.held[host] |
| 1011 | if !ok || snap == 0 || cur.Seq >= snap { // rolloutsim:M8 |
| 1012 | return false, b.retryLocked() |
| 1013 | } |
| 1014 | delete(b.held, host) |
| 1015 | b.touchLocked(cur.Account, cur.App) |
| 1016 | return true, b.saveLocked() |
| 1017 | } |
| 1018 | |
| 1019 | // ClearHoldRanOnPointer is an explicit rollback as this box sees it (ruling |
| 1020 | // 205, Astra R3): host's route installs its good build, and the signed |
| 1021 | // pointer of its channel names that build with promotedAt. The caller has |
| 1022 | // checked both. It clears host's marker only when the marker is (acct, |
| 1023 | // app)'s and older than the pointer: `carlos rollback` signs a fresh |
| 1024 | // pointer, while root's automatic sibling repoint signs none, so the |
| 1025 | // pointer the good build was first promoted under, from before the hold |
| 1026 | // build ran, lifts nothing. The check and the clear are one hold of the |
| 1027 | // lock, so a marker set by a newer run between the caller's reads and this |
| 1028 | // call is never the one cleared. |
| 1029 | func (b *FailureBook) ClearHoldRanOnPointer(host, acct, app string, promotedAt time.Time) (bool, error) { |
| 1030 | b.mu.Lock() |
| 1031 | defer b.mu.Unlock() |
| 1032 | cur, ok := b.held[host] |
| 1033 | if !ok || cur.Account != acct || cur.App != app || !promotedAt.After(cur.At) { |
| 1034 | return false, b.retryLocked() |
| 1035 | } |
| 1036 | delete(b.held, host) |
| 1037 | b.touchLocked(acct, app) |
| 1038 | return true, b.saveLocked() |
| 1039 | } |
| 1040 | |
| 1041 | // Lookup returns the record for (acct, app, build), if one stands. |
| 1042 | func (b *FailureBook) Lookup(acct, app, build string) (FailureRecord, bool) { |
| 1043 | b.mu.Lock() |
| 1044 | defer b.mu.Unlock() |
| 1045 | if i := b.find(acct, app, build); i >= 0 { |
| 1046 | return cloneRec(b.recs[i]), true |
| 1047 | } |
| 1048 | return FailureRecord{}, false |
| 1049 | } |
| 1050 | |
| 1051 | // Blocks reports whether host may not be moved onto build (spec §8.4). An |
| 1052 | // instance that has served the build (served) is never blocked, and neither |
| 1053 | // is one a host retry cleared. |
| 1054 | func (b *FailureBook) Blocks(acct, app, build, host string, served bool) bool { |
| 1055 | if served { |
| 1056 | return false |
| 1057 | } |
| 1058 | b.mu.Lock() |
| 1059 | defer b.mu.Unlock() |
| 1060 | i := b.find(acct, app, build) |
| 1061 | return i >= 0 && !slices.Contains(b.recs[i].Retried, host) |
| 1062 | } |
| 1063 | |
| 1064 | // Retried reports whether build's record stands and lists host as retried: |
| 1065 | // a `carlos instances retry` of host cleared it and no failure of host has |
| 1066 | // spent it since. The scheduler keeps a retry's canary entitlement only |
| 1067 | // while this holds (appQueue.dropRetries, final review round 2, finding 5). |
| 1068 | func (b *FailureBook) Retried(acct, app, build, host string) bool { |
| 1069 | b.mu.Lock() |
| 1070 | defer b.mu.Unlock() |
| 1071 | i := b.find(acct, app, build) |
| 1072 | return i >= 0 && slices.Contains(b.recs[i].Retried, host) |
| 1073 | } |
| 1074 | |
| 1075 | // HeldDown reports whether host's wakes are refused after a failed hold |
| 1076 | // attempt onto build (spec §6.4). |
| 1077 | func (b *FailureBook) HeldDown(acct, app, build, host string) bool { |
| 1078 | b.mu.Lock() |
| 1079 | defer b.mu.Unlock() |
| 1080 | i := b.find(acct, app, build) |
| 1081 | return i >= 0 && slices.Contains(b.recs[i].HeldDown, host) && !slices.Contains(b.recs[i].Retried, host) |
| 1082 | } |
| 1083 | |
| 1084 | // RetryHost is `carlos instances retry <host>`: it clears the record for that |
| 1085 | // host only (spec §8.4). It checks no grant: the box reaches it through |
| 1086 | // RetryHostAt, once the console's signed grant has verified and its epoch |
| 1087 | // is still the restriction's. |
| 1088 | // |
| 1089 | // A retry that clears the host also notes a host clear at the book's next |
| 1090 | // sequence value (ruling 191): a failure of an execution of host that began |
| 1091 | // before it is then stale (Record), and takes nothing back. A retry of a |
| 1092 | // host already retried notes nothing (Task 4 fix round 1): the canary the |
| 1093 | // first retry let through may be running, and its failure must spend the |
| 1094 | // retry, or the host would be left neither held nor allowed through the |
| 1095 | // halt, with no later retry able to free it. |
| 1096 | func (b *FailureBook) RetryHost(acct, app, build, host string) (bool, error) { |
| 1097 | b.mu.Lock() |
| 1098 | defer b.mu.Unlock() |
| 1099 | if !b.retryHostLocked(acct, app, build, host) { |
| 1100 | return false, b.retryLocked() |
| 1101 | } |
| 1102 | return true, b.saveLocked() |
| 1103 | } |
| 1104 | |
| 1105 | // retryHostLocked is RetryHost's change to memory, and reports whether it |
| 1106 | // un-retried the host. It moves the app's generation (rule 1), saved with |
| 1107 | // the book: a grant that was spent here never matches again, even after a |
| 1108 | // reopen. The caller holds b.mu and saves. |
| 1109 | func (b *FailureBook) retryHostLocked(acct, app, build, host string) bool { |
| 1110 | i := b.find(acct, app, build) |
| 1111 | if i < 0 || slices.Contains(b.recs[i].Retried, host) { |
| 1112 | return false |
| 1113 | } |
| 1114 | b.clearedLocked(acct, app, build, host) |
| 1115 | cur := &b.recs[i] |
| 1116 | cur.Retried = append(slices.Clone(cur.Retried), host) |
| 1117 | if j := slices.Index(cur.HeldDown, host); j >= 0 { |
| 1118 | cur.HeldDown = slices.Delete(slices.Clone(cur.HeldDown), j, j+1) |
| 1119 | } |
| 1120 | b.touchLocked(acct, app) |
| 1121 | return true |
| 1122 | } |
| 1123 | |
| 1124 | // RestrictionEpoch is the generation of build's restriction of host as it |
| 1125 | // stands in memory (restrictionEpochLocked), saved or not. It is for logs |
| 1126 | // and tests: status publishes PublishedEpoch, never this. |
| 1127 | func (b *FailureBook) RestrictionEpoch(acct, app, build, host string) uint64 { |
| 1128 | b.mu.Lock() |
| 1129 | defer b.mu.Unlock() |
| 1130 | return b.restrictionEpochLocked(acct, app, build, host) |
| 1131 | } |
| 1132 | |
| 1133 | // PublishedEpoch is the generation of build's restriction of host that |
| 1134 | // status publishes beside HeldBy (HostRollout.HeldEpoch) and the console |
| 1135 | // signs into its retry grant: the app's generation when build's record |
| 1136 | // blocks host or host's marker names build, and the book's memory equals |
| 1137 | // its last landed save (rule 2). Zero otherwise: nothing of build restricts |
| 1138 | // host, or the book owes a save, so the value might not survive a crash. |
| 1139 | func (b *FailureBook) PublishedEpoch(acct, app, build, host string) uint64 { |
| 1140 | b.mu.Lock() |
| 1141 | defer b.mu.Unlock() |
| 1142 | if b.dirty { // durable before use (rule 2); rolloutsim:M13 |
| 1143 | return 0 |
| 1144 | } |
| 1145 | return b.restrictionEpochLocked(acct, app, build, host) |
| 1146 | } |
| 1147 | |
| 1148 | // restrictionEpochLocked is (acct, app)'s generation when build restricts |
| 1149 | // host, through its record (which blocks every host of the app it does not |
| 1150 | // list as retried) or host's hold-ran marker naming build; else 0 (rule 1). |
| 1151 | // The caller holds b.mu. |
| 1152 | func (b *FailureBook) restrictionEpochLocked(acct, app, build, host string) uint64 { |
| 1153 | i := b.find(acct, app, build) |
| 1154 | byRecord := i >= 0 && !slices.Contains(b.recs[i].Retried, host) |
| 1155 | m, ok := b.held[host] |
| 1156 | byMarker := ok && m.Account == acct && m.App == app && m.Build == build |
| 1157 | if !byRecord && !byMarker { |
| 1158 | return 0 |
| 1159 | } |
| 1160 | return b.gens[genKey(acct, app)] |
| 1161 | } |
| 1162 | |
| 1163 | // RetryHostAt is a signed `carlos instances retry <host>` grant (spec §8.4, |
| 1164 | // §11; HostRetryGrant) applied to the book: when the grant names this book |
| 1165 | // (book, the book's ID), the book owes no save, and epoch is build's |
| 1166 | // restriction of host now (the app's generation, PublishedEpoch), it clears |
| 1167 | // build's record for host, when that record holds host (RetryHost), and |
| 1168 | // host's hold-ran marker, when the marker names build (plan review finding |
| 1169 | // 5). A grant from before any restriction of the app last changed, a zero |
| 1170 | // epoch, or one for a build that restricts nothing clears nothing. Nor does |
| 1171 | // a grant for another book, or one that names no book (a console or a box |
| 1172 | // from before the identity): its epoch could be another book's (final |
| 1173 | // review round 1, finding 3). An owed save is retried first, and while it |
| 1174 | // fails nothing is cleared (rule 2): the generation in memory may be one no |
| 1175 | // disk holds. The check and the clears are one hold of the lock. now is the |
| 1176 | // generation at the check, for the caller's log; err is the save's. |
| 1177 | func (b *FailureBook) RetryHostAt(acct, app, build, host, book string, epoch uint64) (rec, mark bool, now uint64, err error) { |
| 1178 | b.mu.Lock() |
| 1179 | defer b.mu.Unlock() |
| 1180 | now = b.restrictionEpochLocked(acct, app, build, host) |
| 1181 | if err := b.retryLocked(); b.dirty { |
| 1182 | return false, false, now, err |
| 1183 | } |
| 1184 | if book == "" || book != b.idLocked() || epoch == 0 || now != epoch { // rolloutsim:M16 |
| 1185 | return false, false, now, nil |
| 1186 | } |
| 1187 | if m, ok := b.held[host]; ok && m.Account == acct && m.App == app && m.Build == build { |
| 1188 | delete(b.held, host) |
| 1189 | mark = true |
| 1190 | } |
| 1191 | rec = b.retryHostLocked(acct, app, build, host) |
| 1192 | if !rec && !mark { |
| 1193 | return false, false, now, nil |
| 1194 | } |
| 1195 | if !rec { |
| 1196 | // Only the marker was removed (no record held the host): still a |
| 1197 | // retry, so its place in the sequence is noted as one (final review |
| 1198 | // round 1, finding 4, Astra). A failure of an execution of build on |
| 1199 | // host that began before it is then stale (Record, PreRetry): no |
| 1200 | // record, no halt, though a late hold outcome still sets its marker. |
| 1201 | // retryHostLocked has noted it, and moved the generation, when it |
| 1202 | // cleared a record, and a retry that removed nothing notes nothing. |
| 1203 | b.clearedLocked(acct, app, build, host) |
| 1204 | b.touchLocked(acct, app) |
| 1205 | } |
| 1206 | return rec, mark, now, b.saveLocked() |
| 1207 | } |
| 1208 | |
| 1209 | // ClearOnPointer is `carlos promote --retry` as this box sees it: a signed |
| 1210 | // channel pointer naming build whose PromotedAt is newer than the record's |
| 1211 | // clears the record (spec rev 14 §8.4). The pointer's signature is verified |
| 1212 | // by the caller's read; nobody can forge a fresh one. A clear is noted |
| 1213 | // app-wide at the book's next sequence value (ruling 191), so a failure of |
| 1214 | // any instance's execution that began before it is stale (Record) and never |
| 1215 | // writes the record back. |
| 1216 | func (b *FailureBook) ClearOnPointer(acct, app, build string, promotedAt time.Time) (bool, error) { |
| 1217 | b.mu.Lock() |
| 1218 | defer b.mu.Unlock() |
| 1219 | i := b.find(acct, app, build) |
| 1220 | if i < 0 || !promotedAt.After(b.recs[i].PromotedAt) { |
| 1221 | return false, b.retryLocked() |
| 1222 | } |
| 1223 | b.recs = slices.Delete(b.recs, i, i+1) |
| 1224 | b.clearedLocked(acct, app, build, "") |
| 1225 | b.touchLocked(acct, app) |
| 1226 | return true, b.saveLocked() |
| 1227 | } |
| 1228 | |
| 1229 | // ForApp returns the app's records. |
| 1230 | func (b *FailureBook) ForApp(acct, app string) []FailureRecord { |
| 1231 | b.mu.Lock() |
| 1232 | defer b.mu.Unlock() |
| 1233 | var out []FailureRecord |
| 1234 | for _, r := range b.recs { |
| 1235 | if r.Account == acct && r.App == app { |
| 1236 | out = append(out, cloneRec(r)) |
| 1237 | } |
| 1238 | } |
| 1239 | return out |
| 1240 | } |
| 1241 | |
| 1242 | // Prune drops the records for builds no route on this box still has |
| 1243 | // installed. "The record stays for any instance still on the failed build" |
| 1244 | // (spec §8.4), and only for those. installed is called under the book's lock |
| 1245 | // and must take no lock of its own (nor call back into the book). The |
| 1246 | // scheduler runs Prune every sweep, so it also retries a save that failed. |
| 1247 | // The retry clears of such builds go too (they are in memory only): a stale |
| 1248 | // failure of one arriving after is a new record, which the next Prune drops |
| 1249 | // again. |
| 1250 | // |
| 1251 | // An app that lost a record and still has a record or a marker gets a new |
| 1252 | // generation; an app with neither left loses its generation, which is the |
| 1253 | // only way one is dropped (rule 1): nothing of it restricts any host, and a |
| 1254 | // later restriction takes a fresh value, larger than every earlier one. |
| 1255 | func (b *FailureBook) Prune(installed func(acct, app, build string) bool) error { |
| 1256 | b.mu.Lock() |
| 1257 | defer b.mu.Unlock() |
| 1258 | for k := range b.cleared { |
| 1259 | if !installed(k[0], k[1], k[2]) { |
| 1260 | delete(b.cleared, k) |
| 1261 | } |
| 1262 | } |
| 1263 | lost := map[string][2]string{} // apps that lost a record |
| 1264 | kept := b.recs[:0] |
| 1265 | for _, r := range b.recs { |
| 1266 | if installed(r.Account, r.App, r.Build) { |
| 1267 | kept = append(kept, r) |
| 1268 | } else { |
| 1269 | lost[genKey(r.Account, r.App)] = [2]string{r.Account, r.App} |
| 1270 | } |
| 1271 | } |
| 1272 | b.recs = kept |
| 1273 | standing := map[string]bool{} // apps with a record or a marker left |
| 1274 | for _, r := range b.recs { |
| 1275 | standing[genKey(r.Account, r.App)] = true |
| 1276 | } |
| 1277 | for _, h := range b.held { |
| 1278 | standing[genKey(h.Account, h.App)] = true |
| 1279 | } |
| 1280 | changed := false |
| 1281 | for k := range b.gens { |
| 1282 | if !standing[k] { |
| 1283 | delete(b.gens, k) |
| 1284 | changed = true |
| 1285 | } |
| 1286 | } |
| 1287 | for _, k := range slices.Sorted(maps.Keys(lost)) { |
| 1288 | if standing[k] { |
| 1289 | b.touchLocked(lost[k][0], lost[k][1]) |
| 1290 | } |
| 1291 | changed = true |
| 1292 | } |
| 1293 | if !changed { |
| 1294 | return b.retryLocked() |
| 1295 | } |
| 1296 | return b.saveLocked() |
| 1297 | } |
| 1298 | |
| 1299 | // List returns a copy of every record, for status. |
| 1300 | func (b *FailureBook) List() []FailureRecord { |
| 1301 | b.mu.Lock() |
| 1302 | defer b.mu.Unlock() |
| 1303 | out := make([]FailureRecord, len(b.recs)) |
| 1304 | for i, r := range b.recs { |
| 1305 | out[i] = cloneRec(r) |
| 1306 | } |
| 1307 | return out |
| 1308 | } |
| 1309 | |
amadan