| 1 | package hostlife |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "sort" |
| 6 | "strings" |
| 7 | ) |
| 8 | |
| 9 | // ---------- relocation (region HA spec 1 §5.8; claim rev 6.30 §4, L1; plan 1b-next rev 16) ---------- |
| 10 | // |
| 11 | // A same-region executor change. The model's steps are abstract time: |
| 12 | // probeStartWindow (30) is a probe attempt's readiness window, so the |
| 13 | // probe deadline holds three attempts; sigAgeH (45) is the 15-minute |
| 14 | // signature age, so the failover pin delay (30 s) is two steps. Each |
| 15 | // guard names the plan 1b-next rev 16 function it mirrors. |
| 16 | |
| 17 | const ( |
| 18 | failoverPinDelay = 2 // §5.8: pin no earlier than the fence's observed_at plus 30 s (hostclaim.FailoverPinDelay) |
| 19 | probeDeadline = 3*probeStartWindow + 10 // §5.8: at most three attempts within the probe deadline (hostrelocate.ProbeDeadline) |
| 20 | releaseDeadline = probeDeadline // 1b-next ruling 14: a release not journaled this long after releasing or probe-failed began is stuck (hostrelocate.ReleaseDeadline, 10 minutes like the probe deadline) |
| 21 | maxReplacements = 2 // §5.8: after two replacements of the target, stuck (hostclaim.MaxTargetReplacements) |
| 22 | enrollDelay = 20 // H16, H17: a launched box is confirmed and enrolled by a worker some time later |
| 23 | replaceDelay = 30 // H5: the group launches a replacement for a terminated box |
| 24 | ) |
| 25 | |
| 26 | // userRelocate is a scale-in drain (§5.8, reason drain; spec 4 owns the |
| 27 | // decision): the serving host's executor is asked to give the host to |
| 28 | // another eligible box of its region. The start CAS (StartRelocation) |
| 29 | // records the intent, op.exec stays on the source, and both participants |
| 30 | // acknowledge by writing execution leases (role()). The two-box step |
| 31 | // draws it. A host that never admitted a writer is not drained (1b-next |
| 32 | // ruling 11, Worker.Start; rule reloc-drain-needs-writer): its release has |
| 33 | // no writer session to stop, so root refuses its stop evidence and the |
| 34 | // drain could only wait for its release deadline; a failover relocates it. |
| 35 | func (w *world) userRelocate() { |
| 36 | c := w.claim |
| 37 | if c == nil || c.Holder == nil || c.Op.Kind != "serving" || w.haOff { |
| 38 | return |
| 39 | } |
| 40 | from := w.execBox(c) |
| 41 | if from == nil || from.Destroyed || from.Old || !from.Enrolled { |
| 42 | return |
| 43 | } |
| 44 | // Worker.Start's order (rev 7): the source unfenced, a writer to |
| 45 | // release, an eligible target, the admission registration; the CAS |
| 46 | // rechecks the admission and applies the start guard |
| 47 | if on("reloc-drain-needs-writer") && c.Holder.Session == 0 { |
| 48 | w.stats["drain refused: the host never had a writer"]++ |
| 49 | return |
| 50 | } |
| 51 | to := w.chooseBoxExcept(from.Region, c.Holder.App, from.ID) |
| 52 | if to == nil || to.Old { |
| 53 | return |
| 54 | } |
| 55 | if ad := w.admit[c.Holder.App]; ad != nil && ad.Closed { |
| 56 | return // a box-started relocation needs the app's admission registration (1b-next ruling 4): a purge closed it |
| 57 | } |
| 58 | id := w.newID("rl") |
| 59 | if w.cas(c, func(n *claim) bool { |
| 60 | if n.Holder == nil || n.Op.Kind != "serving" || n.Op.Exec != from.ID || contains(n.Lost, to.ID) { |
| 61 | return false |
| 62 | } |
| 63 | if ad := w.admit[n.Holder.App]; ad != nil && ad.Closed { |
| 64 | return false // Admission.Recheck |
| 65 | } |
| 66 | if !w.relocationStartAllowed(n, from, to) { |
| 67 | return false |
| 68 | } |
| 69 | n.Op = opState{Kind: "relocating", ID: id, Phase: "starting", Exec: from.ID, Parts: []int{from.ID, to.ID}, |
| 70 | Src: from.ID, Tgt: to.ID, SrcRegion: from.Region, RelocReason: "drain", ReqAt: w.now, App: n.Holder.App, |
| 71 | SrcGen: n.Holder.Gen, SrcStamp: n.Holder.Stamp} |
| 72 | return true |
| 73 | }) { |
| 74 | w.stats["relocation started: drain"]++ |
| 75 | w.logf("relocation %s: drain of inc %d from box %d to box %d in region %d", id, c.Holder.Inc, from.ID, to.ID, from.Region) |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | // relocationStartAllowed is the claim's start guard as L11 amends it for |
| 80 | // relocating (RelocationStartGuard): a park-replica or any other |
| 81 | // whole-location chore in the region blocks (CC11), a park-local on the |
| 82 | // target or on a live source blocks, and a park-generation in the region |
| 83 | // does not: relocation writes only its own new prepared generation, never |
| 84 | // one being retired. The mutation of reloc-start-guard starts over |
| 85 | // anything. |
| 86 | func (w *world) relocationStartAllowed(c *claim, from, to *box) bool { |
| 87 | return w.relocationStartGuard(c, from, to, false) |
| 88 | } |
| 89 | |
| 90 | // relocationStartGuard is RelocationStartGuard(c, region, source, target, |
| 91 | // sourceFenced): a failover's fenced source's park-local does not block. |
| 92 | func (w *world) relocationStartGuard(c *claim, from, to *box, sourceFenced bool) bool { |
| 93 | fenced := -1 |
| 94 | if sourceFenced { |
| 95 | fenced = from.ID |
| 96 | } |
| 97 | if w.completionBarrier(c, fenced) { |
| 98 | return false // ruling 77, before the chores (RelocationStartGuard) |
| 99 | } |
| 100 | if !on("reloc-start-guard") { |
| 101 | return true |
| 102 | } |
| 103 | for _, o := range c.Obl { |
| 104 | switch { |
| 105 | case o.Kind == "park-generation": |
| 106 | continue // L11 |
| 107 | case o.Kind == "park-local": |
| 108 | if o.Box == to.ID || o.Box == from.ID && !sourceFenced && !contains(c.Lost, from.ID) { |
| 109 | return false |
| 110 | } |
| 111 | case o.Region == from.Region: |
| 112 | return false |
| 113 | } |
| 114 | } |
| 115 | return true |
| 116 | } |
| 117 | |
| 118 | // relocExcept is every box the operation may not name as its target |
| 119 | // (replaceTargetSlot, Worker.next): the source, unless an operator's |
| 120 | // resolve made it eligible again (model ruling 13), every replaced target, |
| 121 | // and the extra boxes given. |
| 122 | func relocExcept(op opState, extra ...int) []int { |
| 123 | var out []int |
| 124 | if !op.Resolved { |
| 125 | out = append(out, op.Src) |
| 126 | } |
| 127 | out = append(out, op.Excluded...) |
| 128 | return append(out, extra...) |
| 129 | } |
| 130 | |
| 131 | // relocRelease is the release sequence (§5.8, A3-2; 1b-next Task 7's |
| 132 | // Release) a live box runs to give the host up: the source in releasing, |
| 133 | // a failed target in probe-failed. One step per call; it returns the |
| 134 | // journal entry once every step is done, nil while one is owed. |
| 135 | // |
| 136 | // 1. stop every writer: admission closed (the row held), the process |
| 137 | // tree stopped and empty; an escaped descendant fails the step (L10); |
| 138 | // 2. flush: the final checkpoint and SyncAndWait, passing only once the |
| 139 | // listing shows the stopped database's position N (moves §5.1 step |
| 140 | // 1). It needs the session's own replicator and the box's own files |
| 141 | // directory, or the positions this box recorded for this stop (rule |
| 142 | // reloc-release-needs-flush, rev 9). When the replicator died with |
| 143 | // its agent (claim rev 6.10) and nothing was recorded there is no |
| 144 | // release, and the release deadline makes the operation stuck; the |
| 145 | // mutation journals what the remote holds, Flushed false; |
| 146 | // 3. publish every pending file and wake-state mutation as one final |
| 147 | // manifest of the generation (ReleaseFlushFiles, Task 4); |
| 148 | // 4. record the positions, then the producers' drain and root's |
| 149 | // receipt; recorded positions finish a release only for the |
| 150 | // controller that recorded them or with that receipt (rule |
| 151 | // reloc-release-needs-receipt, rev 11). |
| 152 | func (w *world) relocRelease(b *box, c *claim) *releaseEntry { |
| 153 | h := c.Holder |
| 154 | // StopWriters (L18 joint round 2 finding 3): the current attempt's and |
| 155 | // session's launch journal closes first, so the stopped session is never |
| 156 | // relaunched; root's durable admission fence (relFence) comes only with |
| 157 | // Collect, after the flush and the recorded positions (ruling 28). In |
| 158 | // between, no wake admits a writer: AdmitWriter's relocatingAdmission |
| 159 | // admits nobody in starting or releasing (rule reloc-admits-probe-only). |
| 160 | // The row hold is reconcile's alone (reloc-row-cases, the catch-up hold) |
| 161 | w.closeLaunches(b, c.Op.ID, h.Session) |
| 162 | if !w.evidence(b, c.Op.ID) { |
| 163 | w.stats["release: a process survives the stop"]++ |
| 164 | return nil |
| 165 | } |
| 166 | if r := b.relRecord; r != nil && on("reloc-release-needs-flush") && r.Op == c.Op.ID && r.Phase == c.Op.Phase && r.E.Session == h.Session && r.E.Gen == h.Gen { |
| 167 | // a flush this box recorded for this stop (ReleasePositions.Recorded): |
| 168 | // never flushed or guessed again (rev 9). Rev 11 (Astra round 8): |
| 169 | // the positions are not proof of the stop. The controller that |
| 170 | // recorded them resumes at the producers' drain, and root persists |
| 171 | // the receipt (step 4); after a restart only root's durable receipt |
| 172 | // for the recorded identity finishes the release, and with none |
| 173 | // RecoveryReleaseDrain refuses: no release, and the deadline |
| 174 | // makes the operation stuck. The mutation of |
| 175 | // reloc-release-needs-receipt replays the positions alone, as rev |
| 176 | // 10 did |
| 177 | switch { |
| 178 | case r.Receipt || !on("reloc-release-needs-receipt"): |
| 179 | if r.Ctl != b.Epoch { |
| 180 | w.stats["release: replayed from the recorded positions"]++ |
| 181 | } |
| 182 | case r.Ctl == b.Epoch: |
| 183 | r.Receipt = true // the producers drain; root persists the receipt |
| 184 | w.stats["release: receipt persisted"]++ |
| 185 | b.relFence = c.Op.ID // Collect: root's durable admission fence, after the flush and the positions |
| 186 | default: |
| 187 | w.stats["release: recorded positions without a receipt, refused"]++ |
| 188 | return nil |
| 189 | } |
| 190 | b.relFence = c.Op.ID // root's fence stands with the receipt (Collect) |
| 191 | e := r.E |
| 192 | e.Files, e.Timers = r.E.Files.clone(), append([]int(nil), r.E.Timers...) |
| 193 | return &e |
| 194 | } |
| 195 | prefix := dataKey(h.Location, h.Gen, "") |
| 196 | remote := w.remoteMax(w.buckets[b.Region], prefix) |
| 197 | n, flushed := remote, false |
| 198 | if on("reloc-release-needs-flush") && !(b.DB != nil && b.DB.Inc == h.Inc && b.DB.Gen == h.Gen && b.DB.Epoch == b.Epoch && b.DB.Session == h.Session) { |
| 199 | // relocationFlushDB needs the session's own replicator, and |
| 200 | // nothing recorded a flush for this stop: no release (1b-next Task |
| 201 | // 7; rev 9, Astra round 6). The release deadline makes the |
| 202 | // operation stuck; the mutation journals the remote position with |
| 203 | // Flushed false, as rev 8 did |
| 204 | w.stats["release: no replicator and no recorded flush, refused"]++ |
| 205 | return nil |
| 206 | } |
| 207 | if db := b.DB; db != nil && db.Inc == h.Inc && db.Gen == h.Gen { |
| 208 | if db.Epoch == b.Epoch && db.Session == h.Session { |
| 209 | if len(db.Pending) > 0 { |
| 210 | bl := w.deltaOf(db, db.Inc) |
| 211 | db.FirstFull = false |
| 212 | db.Queue = append(db.Queue, bl) |
| 213 | db.Pos, db.Pending = bl.Max, nil |
| 214 | } |
| 215 | w.ship(b) |
| 216 | if len(db.Queue) > 0 || db.InFlight != nil || remote < db.Pos { |
| 217 | w.stats["release: waits for the flush"]++ |
| 218 | return nil |
| 219 | } |
| 220 | n, flushed = db.Pos, true |
| 221 | } else { |
| 222 | w.stats["release: the session's replicator is gone, unflushed"]++ |
| 223 | } |
| 224 | } |
| 225 | // the files the box holds, or, when it holds none of this incarnation's |
| 226 | // (a failed target whose unopened install a scoped cleanup retired), |
| 227 | // the set its own installation applied: a release never invents a |
| 228 | // files identity |
| 229 | files := b.Files.clone() |
| 230 | if files == nil || files.Inc != h.Inc { |
| 231 | if on("reloc-release-needs-flush") { |
| 232 | // ReleaseFlushFiles publishes this box's own files directory; |
| 233 | // there is none to publish (rev 9) |
| 234 | w.stats["release: no files directory to flush, refused"]++ |
| 235 | return nil |
| 236 | } |
| 237 | files = w.relocFiles(c).clone() |
| 238 | } |
| 239 | // 3. (Task 4) publish every pending file and wake-state mutation, |
| 240 | // whatever the app's files_durability, as one final manifest of the |
| 241 | // generation (ReleaseFlushFiles): create-only, acknowledged before the |
| 242 | // entry is journaled, its sequence journaled as manifest_seq |
| 243 | seq := 0 |
| 244 | if w.perRegion > 1 { |
| 245 | if b.pubGen != h.Gen { |
| 246 | b.pubGen, b.pubSeq, b.releaseSeq = h.Gen, 0, 0 |
| 247 | } |
| 248 | if !w.ensureObject(b, h, files) { |
| 249 | w.stats["release: waits for its files object"]++ |
| 250 | return nil |
| 251 | } |
| 252 | if b.releaseSeq == 0 { |
| 253 | b.pubSeq++ |
| 254 | b.releaseSeq = b.pubSeq |
| 255 | } |
| 256 | w.issued++ |
| 257 | bl := &blob{Kind: "files", Inc: h.Inc, Gen: h.Gen, Seq: b.releaseSeq, Files: files.clone(), Timers: append([]int(nil), b.Timers...), Base: w.issued} |
| 258 | if !w.ackedPut(b, manifestKey(h.Location, h.Gen, b.releaseSeq), bl, true) { |
| 259 | w.stats["release: waits for its manifest"]++ |
| 260 | return nil |
| 261 | } |
| 262 | seq, b.releaseSeq = b.releaseSeq, 0 |
| 263 | b.published = filesState(b.Timers, files) |
| 264 | } |
| 265 | e := &releaseEntry{Box: b.ID, Session: h.Session, Gen: h.Gen, FinalTX: n, ManifestSeq: seq, Flushed: flushed, |
| 266 | Files: files, Timers: append([]int(nil), b.Timers...), At: w.now} |
| 267 | if on("reloc-release-needs-flush") { |
| 268 | // ReleasePositions.Record: durable before the producers close |
| 269 | b.relRecord = &releaseRecord{Op: c.Op.ID, Phase: c.Op.Phase, E: *e, Ctl: b.Epoch} |
| 270 | b.relRecord.E.Files, b.relRecord.E.Timers = files.clone(), append([]int(nil), b.Timers...) |
| 271 | if on("reloc-release-needs-receipt") { |
| 272 | // the producers' drain and root's receipt are the next step: |
| 273 | // a crash here leaves positions and no receipt (rev 11) |
| 274 | return nil |
| 275 | } |
| 276 | } |
| 277 | if b.relFence != c.Op.ID { |
| 278 | b.relFence = c.Op.ID // Collect, on the mutations' paths that skip the record or the receipt |
| 279 | } |
| 280 | return e |
| 281 | } |
| 282 | |
| 283 | // pinSource is the generation the pin reads (StorePinner's genselect.Select |
| 284 | // over the admitted chain): holder.generation when it holds an object or |
| 285 | // the marker names it; else the baseline the recovery walk selects for an |
| 286 | // unfinished run of cold admissions (claim rev 6.22 §6.2a), as a cold |
| 287 | // admission would restore it. An unexplained empty generation refuses: |
| 288 | // absence is never evidence (§5.9). |
| 289 | func (w *world) pinSource(b *box, c *claim) (string, bool) { |
| 290 | h := c.Holder |
| 291 | loc := h.Location |
| 292 | if w.hasFiles(b, loc, h.Gen) { |
| 293 | return h.Gen, true |
| 294 | } |
| 295 | bg, _ := w.bucketGen(b, loc) |
| 296 | if bg == h.Gen { |
| 297 | return h.Gen, true |
| 298 | } |
| 299 | run, ok := w.unfinishedRun(c, bg) |
| 300 | if !ok { |
| 301 | return "", false |
| 302 | } |
| 303 | return w.runBaseline(b, c, run, bg) |
| 304 | } |
| 305 | |
| 306 | // selectPin is StorePinner.Pin against the observation c: the source |
| 307 | // generation and its listing's newest position N, after a release only |
| 308 | // once N reaches the release's final_txid (rule reloc-pin-reaches-release), |
| 309 | // and an empty listing only with evidence (ruling 15, rule |
| 310 | // reloc-pin-evidenced-empty): a journaled release that flushed at 0, or a |
| 311 | // holder that never admitted a writer; any other empty listing marks the |
| 312 | // operation stuck for an operator. The pin carries the claim version it |
| 313 | // was selected against (ruling 16, RecoveryPinKey). Task 4 adds the files |
| 314 | // pin. It returns nil while the pin must wait. |
| 315 | func (w *world) selectPin(b *box, c *claim) *pinEntry { |
| 316 | op, h := c.Op, c.Holder |
| 317 | src, ok := w.pinSource(b, c) |
| 318 | if !ok { |
| 319 | w.stats["pin: the source generation cannot be selected yet"]++ |
| 320 | return nil |
| 321 | } |
| 322 | n := w.remoteMax(w.buckets[b.Region], dataKey(h.Location, src, "")) |
| 323 | r := op.Release |
| 324 | if r != nil && on("reloc-pin-reaches-release") && n < r.FinalTX { |
| 325 | w.stats["pin: waits for the release's final txid"]++ |
| 326 | return nil |
| 327 | } |
| 328 | if n == 0 && on("reloc-pin-evidenced-empty") && !op.Resolved && h.Session != 0 && !(r != nil && r.Flushed && r.FinalTX == 0) { |
| 329 | w.markStuck(c, fmt.Sprintf("an empty listing of %q is no evidence of an empty database (session %d)", src, h.Session)) |
| 330 | return nil |
| 331 | } |
| 332 | pin := &pinEntry{Gen: src, TX: n, ClaimVer: c.Ver, At: w.now, Files: w.relocFiles(c).clone(), Timers: append([]int(nil), w.relocTimers(c)...)} |
| 333 | // (Task 4) the files pin: the newest landed manifest of the admitted |
| 334 | // chain, by exact version, independently of the database baseline |
| 335 | // (B2-4); after a release it must reach the release's manifest_seq |
| 336 | if mk, mv, ms, mg, _ := w.newestManifest(b.Region, h.Location, h.Gen); mk != "" { |
| 337 | if r != nil && r.ManifestSeq > 0 && on("reloc-pin-reaches-release") && (mg != r.Gen || ms < r.ManifestSeq) { |
| 338 | w.stats["pin: waits for the release's manifest"]++ |
| 339 | return nil |
| 340 | } |
| 341 | pin.ManKey, pin.ManVer, pin.ManSeq, pin.ManGen = mk, mv, ms, mg |
| 342 | } |
| 343 | // 1b-next ruling 25 as rev 18 amended it (finding F2; rule |
| 344 | // files-pin-needs-key): when a writer was admitted and the pinned |
| 345 | // database is nonempty, the pinned manifest must carry a live |
| 346 | // instance.key, or the operation is stuck; no operator replay accepts |
| 347 | // it, since a database is never installed without its key. The L21 |
| 348 | // window (the app mints the key after it starts, and its first |
| 349 | // manifest with the key has not landed) ends here. The mutation pins |
| 350 | // what it found, rev 16's behaviour. Only the two-box shape has a |
| 351 | // files publisher (ruling 11) |
| 352 | if w.perRegion > 1 && on("files-pin-needs-key") && h.Session != 0 && n > 0 && !manifestHasKey(w.pinnedManifestOf(b.Region, pin)) { |
| 353 | w.stats["pin: files unevidenced, no instance.key"]++ |
| 354 | w.markStuck(c, fmt.Sprintf("the pinned files carry no instance.key for a nonempty database (TXID %d)", n)) |
| 355 | return nil |
| 356 | } |
| 357 | if m := w.pinnedManifestOf(b.Region, pin); m != nil && m.Files != nil && m.Files.Obj != "" && !w.originAuthorized(b.Region, c, m.Files.Obj) { |
| 358 | // filesPin's referencedOrigins (1b-next rev 12 finding 1, R11-1): |
| 359 | // the object the pinned manifest names must be current in this |
| 360 | // region's bucket, under this holder's location, at an origin the |
| 361 | // pin may cite; otherwise the pin refuses (ErrOrigin) and the |
| 362 | // worker retries on a later pass |
| 363 | w.stats["pin: files object refused"]++ |
| 364 | w.logf("DEBUG pin refused obj %s cur=%v", m.Files.Obj, w.buckets[b.Region].current(m.Files.Obj) != nil) |
| 365 | return nil |
| 366 | } |
| 367 | return pin |
| 368 | } |
| 369 | |
| 370 | // ---------- files objects (rev 4, Astra R1-8; spec §5.6 "Objects are immutable") ---------- |
| 371 | |
| 372 | // objectKey is a files object: <holder.location>/files/<generation>/objects/<content>, |
| 373 | // written create-only by the generation that first held the content. |
| 374 | func objectKey(loc, gen string, f *filesDir) string { |
| 375 | return dataKey(loc, gen, fmt.Sprintf("files/objects/%d-%d", f.Key, f.Env)) |
| 376 | } |
| 377 | |
| 378 | // isFilesKey: a manifest or an object of the files tree. |
| 379 | func isFilesKey(k string) bool { |
| 380 | return strings.Contains(k, "files/m-") || strings.Contains(k, "files/objects/") |
| 381 | } |
| 382 | |
| 383 | // objectCurrent: f names an object holding its content, in b's region |
| 384 | // (the region binding; the mutation of files-objects-region-bound accepts |
| 385 | // another region's object, as a moved host's carried reference). |
| 386 | func (w *world) objectCurrent(b *box, f *filesDir) bool { |
| 387 | if f == nil || f.Obj == "" || !strings.HasSuffix(f.Obj, fmt.Sprintf("/%d-%d", f.Key, f.Env)) { |
| 388 | return false |
| 389 | } |
| 390 | return f.ObjRegion == b.Region || !on("files-objects-region-bound") |
| 391 | } |
| 392 | |
| 393 | // ensureObject uploads f's content as an object of the current |
| 394 | // generation when f names none for it in this region (a changed file, or |
| 395 | // a host that came from another region), and reports once one is |
| 396 | // acknowledged; an unchanged file keeps naming the object an older |
| 397 | // generation wrote, which is what keeps that generation's objects live. |
| 398 | func (w *world) ensureObject(b *box, h *holder, f *filesDir) bool { |
| 399 | if f == nil || w.objectCurrent(b, f) { |
| 400 | return true |
| 401 | } |
| 402 | k := objectKey(h.Location, h.Gen, f) |
| 403 | if !w.ackedPut(b, k, &blob{Kind: "files-object", Inc: h.Inc, Gen: h.Gen, Files: f.clone()}, true) { |
| 404 | return false |
| 405 | } |
| 406 | f.Obj, f.ObjRegion = k, b.Region |
| 407 | w.stats["files objects written"]++ |
| 408 | return true |
| 409 | } |
| 410 | |
| 411 | // originAuthorized is the pin's check of one referenced object: current |
| 412 | // in region r's bucket under the holder's location, and at an authorized |
| 413 | // origin. With the rule (files-origins-referenced) the origins are the |
| 414 | // generations the pinned manifest references, so any such object is |
| 415 | // authorized; the mutation derives them from holder.generation and the |
| 416 | // retained admissions, as rev 11 did, which the trim to eight shortens. |
| 417 | func (w *world) originAuthorized(r int, c *claim, obj string) bool { |
| 418 | h := c.Holder |
| 419 | if !strings.HasPrefix(obj, h.Location) || w.buckets[r].current(obj) == nil { |
| 420 | return false |
| 421 | } |
| 422 | if on("files-origins-referenced") { |
| 423 | return true |
| 424 | } |
| 425 | g := genOf(h.Location, obj) |
| 426 | if g == h.Gen || c.Op.Prepared != "" && g == c.Op.Prepared { |
| 427 | return true |
| 428 | } |
| 429 | for _, e := range h.Admissions { |
| 430 | if e.Gen == g || e.Predecessor.Gen == g { |
| 431 | return true |
| 432 | } |
| 433 | } |
| 434 | return false |
| 435 | } |
| 436 | |
| 437 | // pinnedManifestOf is the manifest pin names, read by version. |
| 438 | func (w *world) pinnedManifestOf(r int, pin *pinEntry) *blob { |
| 439 | if pin.ManKey == "" { |
| 440 | return nil |
| 441 | } |
| 442 | if v := w.buckets[r].byID(pin.ManKey, pin.ManVer); v != nil && !v.Marker { |
| 443 | return v.B |
| 444 | } |
| 445 | return nil |
| 446 | } |
| 447 | |
| 448 | // manifestHasKey: the manifest carries a live instance.key (InstanceKeyPath, |
| 449 | // liveEntry): its files directory names one. |
| 450 | func manifestHasKey(m *blob) bool { return m != nil && m.Files != nil && m.Files.Key != 0 } |
| 451 | |
| 452 | // restoreUpTo restores generation gen of region r from its current |
| 453 | // listing, capped at TXID upto: the newest current snapshot at or below |
| 454 | // upto as the root, then the planner's rule (ltx.IsContiguous, highest |
| 455 | // max, lowest min, higher level) over the current level-0 and level-1 |
| 456 | // files whose range ends at or below upto. It is §5.8's build from the |
| 457 | // pinned recovery set: nothing past the pin is restored, and a listing |
| 458 | // that does not reach upto refuses (nil), as a pin short of its release |
| 459 | // would. It reads the bucket and writes nothing. |
| 460 | func (w *world) restoreUpTo(r, inc int, loc, gen string, upto int) *localDB { |
| 461 | bk := w.buckets[r] |
| 462 | prefix := dataKey(loc, gen, "") |
| 463 | db := &localDB{Inc: inc, Loc: loc, Gen: gen, State: map[string]int{}} |
| 464 | var root *blob |
| 465 | for _, s := range w.currentLTX(bk, prefix, 9) { |
| 466 | if s.B.Max <= upto && (root == nil || s.B.Max > root.Max) { |
| 467 | root = s.B |
| 468 | } |
| 469 | } |
| 470 | cur := 0 |
| 471 | var applied []*blob |
| 472 | if root != nil { |
| 473 | apply(db.State, root) |
| 474 | cur = root.Max |
| 475 | applied = append(applied, root) |
| 476 | db.Key = root.SealKey |
| 477 | if root.Inc != 0 { |
| 478 | db.Inc = root.Inc |
| 479 | } |
| 480 | } |
| 481 | var cands []*blob |
| 482 | for _, level := range []int{0, 1} { |
| 483 | for _, f := range w.currentLTX(bk, prefix, level) { |
| 484 | if f.B.Max <= upto { |
| 485 | cands = append(cands, f.B) |
| 486 | } |
| 487 | } |
| 488 | } |
| 489 | for cur < upto { |
| 490 | var best *blob |
| 491 | for _, f := range cands { |
| 492 | if f.Min <= cur+1 && f.Max > cur && (best == nil || f.Max > best.Max || f.Max == best.Max && (f.Min < best.Min || f.Min == best.Min && f.Level > best.Level)) { |
| 493 | best = f |
| 494 | } |
| 495 | } |
| 496 | if best == nil { |
| 497 | return nil // the listing does not reach upto: a gap, or objects still landing |
| 498 | } |
| 499 | apply(db.State, best) |
| 500 | cur = best.Max |
| 501 | applied = append(applied, best) |
| 502 | if db.Key == 0 { |
| 503 | db.Key = best.SealKey |
| 504 | } |
| 505 | if best.Inc != 0 { |
| 506 | db.Inc = best.Inc |
| 507 | } |
| 508 | } |
| 509 | db.Log = txsOf(applied...) |
| 510 | db.TX = w.lineageTX(applied...) |
| 511 | db.Pos = cur |
| 512 | return db |
| 513 | } |
| 514 | |
| 515 | // relocFiles is the files directory the local install applies: the |
| 516 | // pinned set's (the pin carries the files and wake state it was taken |
| 517 | // with), else the release's (a drain, a nested relocation), else the |
| 518 | // converted move's or the failover's stand-in for the files publisher |
| 519 | // (moves §14.2.6). Task 4 puts the pinned manifest first. |
| 520 | func (w *world) relocFiles(c *claim) *filesDir { |
| 521 | if m := w.pinnedManifest(c); m != nil && m.Files != nil { |
| 522 | return m.Files |
| 523 | } |
| 524 | if p := c.Op.Pin; p != nil && p.Files != nil { |
| 525 | return p.Files |
| 526 | } |
| 527 | if r := c.Op.Release; r != nil && r.Files != nil { |
| 528 | return r.Files |
| 529 | } |
| 530 | return c.Op.StopFiles |
| 531 | } |
| 532 | |
| 533 | // relocTimers is the wake state the local install replaces timers with. |
| 534 | func (w *world) relocTimers(c *claim) []int { |
| 535 | if m := w.pinnedManifest(c); m != nil { |
| 536 | return m.Timers |
| 537 | } |
| 538 | if p := c.Op.Pin; p != nil && p.Files != nil { |
| 539 | return p.Timers |
| 540 | } |
| 541 | if r := c.Op.Release; r != nil { |
| 542 | return r.Timers |
| 543 | } |
| 544 | return nil |
| 545 | } |
| 546 | |
| 547 | // nameReplacement names next as the operation's target in n, counting a |
| 548 | // replacement when it replaces another box (replaceTargetSlot): the |
| 549 | // replaced box is excluded for this host, and past maxReplacements the |
| 550 | // operation is marked stuck at the current phase instead (rule |
| 551 | // reloc-replacement-cap). Inside the CAS it refuses a box that owes a |
| 552 | // park-local for the host (rule reloc-replace-not-owing, Task 3). It |
| 553 | // reports whether next was named. |
| 554 | func (w *world) nameReplacement(n *claim, next int, phase string) bool { |
| 555 | if on("reloc-replace-not-owing") { |
| 556 | if b := w.boxByID(next); b != nil && w.owesLocal(n, b) { |
| 557 | w.stats["replacement refused: the box owes a park-local"]++ |
| 558 | return false |
| 559 | } |
| 560 | } |
| 561 | old := n.Op.Tgt |
| 562 | if old >= 0 && old != next { |
| 563 | if on("reloc-replacement-cap") && n.Op.Replacements >= maxReplacements { |
| 564 | n.Op.Stuck = fmt.Sprintf("two replacements of the target used; a third was needed at %s", n.Op.Phase) |
| 565 | w.stats["relocation stuck: replacement cap"]++ |
| 566 | return false |
| 567 | } |
| 568 | n.Op.Replacements++ |
| 569 | n.Op.Excluded = addUniq(n.Op.Excluded, old) |
| 570 | var parts []int |
| 571 | for _, p := range n.Op.Parts { |
| 572 | if p != old { |
| 573 | parts = append(parts, p) |
| 574 | } |
| 575 | } |
| 576 | n.Op.Parts = parts |
| 577 | } |
| 578 | n.Op.Tgt = next |
| 579 | n.Op.Parts = addUniq(n.Op.Parts, next) |
| 580 | n.Op.Phase = phase |
| 581 | return true |
| 582 | } |
| 583 | |
| 584 | // markStuck is MarkStuck: a journal entry, never a phase; every relocation |
| 585 | // step refuses afterwards until the operator resolves it. It never stops |
| 586 | // an operation between installed and a probe while P is unopened: that |
| 587 | // live target's safe point is its probe. |
| 588 | func (w *world) markStuck(c *claim, why string) { |
| 589 | id := c.Op.ID |
| 590 | if w.cas(c, func(n *claim) bool { |
| 591 | if !sameOp(n.Op.ID, id) || n.Op.Kind != "relocating" || n.Op.Stuck != "" { |
| 592 | return false |
| 593 | } |
| 594 | if n.Op.Phase == "installed" && n.Op.Receipt != nil && !preparedOpened(n) { |
| 595 | return false |
| 596 | } |
| 597 | n.Op.Stuck = why |
| 598 | return true |
| 599 | }) { |
| 600 | w.alerts++ |
| 601 | w.stats["relocation stuck"]++ |
| 602 | w.logf("relocation %s stuck: %s", id, why) |
| 603 | } |
| 604 | } |
| 605 | |
| 606 | // relocationLeftovers queues, inside the serving CAS (CompleteRelocation), |
| 607 | // the park-local chore every abandoned installation owes (claim §4: "every |
| 608 | // abandoned local installation receives an exact park-local"): the |
| 609 | // source, when it is alive and its fence did not discharge it, and every |
| 610 | // other box this operation handed op.exec to. |
| 611 | func (w *world) relocationLeftovers(n *claim, op opState) { |
| 612 | // every prepared generation the operation abandoned unopened |
| 613 | // (abandoned_prepared) is a dead generation of holder.location: its |
| 614 | // remote build is retired by a park-generation queued here, beside the |
| 615 | // park-local chores (1b-next ruling 27, CompleteRelocation; plan ruling |
| 616 | // 16). An opened P is part of the writer lineage and never listed |
| 617 | if h := n.Holder; h != nil { |
| 618 | for _, g := range op.Abandoned { |
| 619 | if g == h.Gen || w.ancestorGen(h.Location, h.Gen, g) { |
| 620 | continue |
| 621 | } |
| 622 | n.Obl = append(n.Obl, w.scoped(obligation{ID: w.newID("ob"), Op: op.ID, Kind: "park-generation", Region: op.SrcRegion, Box: -1, Prefix: h.Location, Ret: g, Quiet: w.now + horizonH}, w.now, h.Inc)) |
| 623 | w.stats["relocation: abandoned prepared generation parked"]++ |
| 624 | } |
| 625 | } |
| 626 | // the pinned manifest of a superseded generation (rev 5): the files |
| 627 | // live set kept it while the operation ran, so a park-generation of |
| 628 | // that generation that ran meanwhile left it; once the operation |
| 629 | // completes it is a dead generation's key, parked here. This is the |
| 630 | // model's abstraction, not a CompleteRelocation guarantee (rev 6, |
| 631 | // Astra round 3 finding 2): in 1b-next rev 19 completion queues only |
| 632 | // the local chores and the abandoned prepared generations; an obsolete |
| 633 | // files manifest is the at-rest sweep's (claim §6.2a), and the |
| 634 | // pinner's retained control object is the storage follow-up's (1b-next |
| 635 | // ruling 21). The model parks it at once so INV-10 can check the |
| 636 | // residue at completion |
| 637 | if h := n.Holder; h != nil && op.Pin != nil && op.Pin.ManKey != "" && op.Pin.ManGen != h.Gen { |
| 638 | ret := op.Pin.ManGen |
| 639 | if ret == "" { |
| 640 | ret = "@root" |
| 641 | } |
| 642 | queued := false |
| 643 | for _, o := range n.Obl { |
| 644 | if o.Kind == "park-generation" && o.Region == op.SrcRegion && o.Ret == ret && o.Prefix == h.Location { |
| 645 | queued = true |
| 646 | } |
| 647 | } |
| 648 | if !queued { |
| 649 | n.Obl = append(n.Obl, w.scoped(obligation{ID: w.newID("ob"), Op: op.ID, Kind: "park-generation", Region: op.SrcRegion, Box: -1, Prefix: h.Location, Ret: ret, Quiet: w.now + horizonH}, w.now, h.Inc)) |
| 650 | w.stats["relocation: the pinned manifest's generation parked"]++ |
| 651 | } |
| 652 | } |
| 653 | owed := append([]int{op.Src}, op.InstalledOn...) |
| 654 | for _, id := range owed { |
| 655 | if id < 0 || id == op.Exec || contains(op.Discharged, id) { |
| 656 | continue |
| 657 | } |
| 658 | if b := w.boxByID(id); b == nil || w.owesLocal(n, b) { |
| 659 | continue |
| 660 | } |
| 661 | n.Obl = append(n.Obl, obligation{ID: w.newID("ob"), Op: op.ID, Kind: "park-local", Box: id, Region: -1, Ret: op.ID}) |
| 662 | } |
| 663 | } |
| 664 | |
| 665 | // relocStarting: the barrier (claim D6, requireAcks in ToReleasing). A |
| 666 | // drain leaves starting for releasing once both participants' execution |
| 667 | // leases are current (role() writes them on every read of the claim), and |
| 668 | // the CAS records releasing_since for the release deadline; a failover |
| 669 | // never enters starting in this model (it starts at pinning, Task 3, and |
| 670 | // PinFailover's barrier is checked there before the pin is selected: |
| 671 | // rule reloc-failover-ack-before-pin, rev 5 and 6). The mutation of |
| 672 | // reloc-exec-ack-barrier moves on with no acknowledgement. |
| 673 | func (w *world) relocStarting(b *box, c *claim) { |
| 674 | op := c.Op |
| 675 | if on("reloc-exec-ack-barrier") && !acked(c) { |
| 676 | return |
| 677 | } |
| 678 | w.cas(c, func(n *claim) bool { |
| 679 | if !sameOp(n.Op.ID, op.ID) || n.Op.Kind != "relocating" || n.Op.Phase != "starting" { |
| 680 | return false |
| 681 | } |
| 682 | n.Op.Phase, n.Op.ReleasingSince = "releasing", w.now |
| 683 | return true |
| 684 | }) |
| 685 | } |
| 686 | |
| 687 | // relocReleasing: the source runs the release sequence and journals it |
| 688 | // (JournalRelease); then a worker's CAS (PinAfterRelease) cites the entry |
| 689 | // and leaves for pinning with op.exec none and the stamp kept (claim |
| 690 | // R6-4), once every participant has acknowledged (rule |
| 691 | // reloc-ack-before-pinning: a target replaced in releasing acknowledges |
| 692 | // before the operation leaves the phase; 1b-next rev 4 N6). A source that |
| 693 | // has not journaled its release releaseDeadline after releasing began is |
| 694 | // stuck (rule reloc-release-deadline, Worker.releaseOverdue): nothing |
| 695 | // moves away from a live box without its stop evidence. The mutation of |
| 696 | // reloc-release-first pins without the entry: the source is never |
| 697 | // stopped. |
| 698 | func (w *world) relocReleasing(b *box, c *claim) { |
| 699 | op := c.Op |
| 700 | if op.Release == nil { |
| 701 | if on("reloc-release-deadline") && w.now > op.ReleasingSince+releaseDeadline { |
| 702 | w.markStuck(c, fmt.Sprintf("box %d has not released the host %d steps after releasing", op.Exec, releaseDeadline)) |
| 703 | return |
| 704 | } |
| 705 | if b.ID != op.Exec { |
| 706 | if !on("reloc-release-first") { |
| 707 | w.cas(c, func(n *claim) bool { |
| 708 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "releasing" |
| 709 | if ok { |
| 710 | n.Op.Phase, n.Op.Exec = "pinning", -1 |
| 711 | } |
| 712 | return ok |
| 713 | }) |
| 714 | } |
| 715 | return |
| 716 | } |
| 717 | w.relocJournalRelease(b, c, false) |
| 718 | return |
| 719 | } |
| 720 | if on("reloc-ack-before-pinning") && !acked(c) { |
| 721 | w.stats["pin after release: waits for the acknowledgements"]++ |
| 722 | return |
| 723 | } |
| 724 | w.cas(c, func(n *claim) bool { |
| 725 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "releasing" && n.Op.Release != nil && n.Op.Release.Box == n.Op.Exec |
| 726 | if ok { |
| 727 | n.Op.Phase, n.Op.Exec = "pinning", -1 |
| 728 | } |
| 729 | return ok |
| 730 | }) |
| 731 | } |
| 732 | |
| 733 | // relocPinning: a worker selects the pin against its observation and |
| 734 | // journals it (JournalPin) in a CAS bound to that observation's claim |
| 735 | // version (rule reloc-pin-claim-gen, ruling 16: a pin selected against an |
| 736 | // older claim is abandoned, and the next pass selects afresh); a failover |
| 737 | // pins no earlier than the fence's observed_at plus failoverPinDelay (rule |
| 738 | // reloc-failover-pin-delay, a loss bound). Then it names a target when the |
| 739 | // operation has none (NameTarget: a converted move, a resolved stuck |
| 740 | // operation, a target lost with no spare); then, with every participant's |
| 741 | // lease current (rule reloc-exec-ack-barrier), hands op.exec to the |
| 742 | // target for a new installation (ToInstalling): a new stamp, journaled as |
| 743 | // install_stamp, and a fresh prepared generation id (claim R6-4). The |
| 744 | // target must owe no park-local for the host. |
| 745 | // |
| 746 | // An initial failover (op.PinFailover: 1b-next's starting after |
| 747 | // StartRelocation with a fence) first passes PinFailover's barrier (rev |
| 748 | // 5, Astra round 2 finding 1; narrowed in rev 6, round 3 finding 1): the |
| 749 | // target is named, every participant has acknowledged, and the PinFailover |
| 750 | // CAS clears the flag, all before any pin is selected; so a manifest that |
| 751 | // lands while the target's controller is unavailable is in the pin (rule |
| 752 | // reloc-failover-ack-before-pin). The flag is the model's starting |
| 753 | // (ruling 4). A converted move, a nested failover after a replacement and |
| 754 | // an operator's replay enter pinning directly, as in 1b-next, where |
| 755 | // Worker.pinning selects and journals the pin first and checks the |
| 756 | // acknowledgements only for ToInstalling. The mutation selects the pin |
| 757 | // before the barrier, as rev 4 did. |
| 758 | func (w *world) relocPinning(b *box, c *claim) { |
| 759 | op := c.Op |
| 760 | if op.PinFailover && on("reloc-failover-ack-before-pin") { |
| 761 | if op.Tgt < 0 { |
| 762 | w.relocNameTarget(c) |
| 763 | return |
| 764 | } |
| 765 | if !acked(c) { |
| 766 | w.stats["failover pin: waits for the acknowledgements"]++ |
| 767 | return |
| 768 | } |
| 769 | if w.cas(c, func(n *claim) bool { |
| 770 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "pinning" && n.Op.PinFailover && n.Op.Tgt == op.Tgt && n.Op.Stuck == "" |
| 771 | if ok { |
| 772 | n.Op.PinFailover = false |
| 773 | } |
| 774 | return ok |
| 775 | }) { |
| 776 | w.logf("relocation %s: PinFailover, box %d acknowledged", op.ID, op.Tgt) |
| 777 | } |
| 778 | return |
| 779 | } |
| 780 | if op.Tgt < 0 && on("reloc-name-before-pin") { |
| 781 | // Worker.pinning names a missing target before it selects any pin |
| 782 | // (1b-next rev 19; rev 7, Astra round 4 finding 1), on every path |
| 783 | // that enters pinning without one: a converted move, an operator's |
| 784 | // replay. With no eligible box it waits, and a manifest that lands |
| 785 | // meanwhile is in the later pin. The mutation pins first, as rev 6 |
| 786 | // did for these paths |
| 787 | w.relocNameTarget(c) |
| 788 | return |
| 789 | } |
| 790 | if op.Pin == nil { |
| 791 | if on("reloc-failover-pin-delay") && (op.Release == nil || on("reloc-pin-delay-every-fence")) && w.now < latestObservation(op)+failoverPinDelay { |
| 792 | return |
| 793 | } |
| 794 | pin := w.selectPin(b, c) |
| 795 | if pin == nil { |
| 796 | return |
| 797 | } |
| 798 | if w.cas(c, func(n *claim) bool { |
| 799 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "pinning" && n.Op.Pin == nil && n.Op.Stuck == "" |
| 800 | if ok && on("reloc-pin-claim-gen") && n.Ver != pin.ClaimVer { |
| 801 | w.stats["pin: selected against an older claim, abandoned"]++ |
| 802 | ok = false |
| 803 | } |
| 804 | if ok { |
| 805 | n.Op.Pin = pin |
| 806 | } |
| 807 | return ok |
| 808 | }) { |
| 809 | w.logf("relocation %s: pinned %q at TXID %d (selected against claim version %d)", op.ID, pin.Gen, pin.TX, pin.ClaimVer) |
| 810 | } |
| 811 | return |
| 812 | } |
| 813 | if op.Tgt < 0 { |
| 814 | w.relocNameTarget(c) |
| 815 | return |
| 816 | } |
| 817 | if on("reloc-exec-ack-barrier") && !acked(c) { |
| 818 | return |
| 819 | } |
| 820 | to := op.Tgt |
| 821 | if w.cas(c, func(n *claim) bool { |
| 822 | if !sameOp(n.Op.ID, op.ID) || n.Op.Kind != "relocating" || n.Op.Phase != "pinning" || n.Op.Tgt != to || n.Op.Pin == nil || n.Op.Stuck != "" || contains(n.Lost, to) { |
| 823 | return false |
| 824 | } |
| 825 | if t := w.boxByID(to); t == nil || w.owesLocal(n, t) { |
| 826 | return false |
| 827 | } |
| 828 | w.handExec(n, to, "installing", w.newID("P")) |
| 829 | n.Op.BaseKey, n.Op.BaseVer, n.Op.BaseTX = "", 0, 0 |
| 830 | return true |
| 831 | }) { |
| 832 | w.logf("relocation %s: box %d installs with stamp %d", op.ID, to, w.claim.Holder.Stamp) |
| 833 | } |
| 834 | } |
| 835 | |
| 836 | // relocNameTarget is NameTarget at pinning: choose() in the source region |
| 837 | // over the eligible boxes, the excluded ones left out. A converted move's |
| 838 | // own target (1b-next's move_target_box, in the region the move was |
| 839 | // going to) leaves the participants in the same CAS (rule |
| 840 | // reloc-name-drops-move-target, 1b-next 151261fe): readiness binds to the |
| 841 | // box named here (moves §7.10) and a failover's participant is its |
| 842 | // target alone (§5.8); the move's target never held op.exec on the |
| 843 | // converted operation. A relocation's participants are otherwise all in |
| 844 | // its region, so the model drops every participant outside op.SrcRegion, |
| 845 | // which only a converted move leaves there. The mutation keeps it, and |
| 846 | // the barrier waits for a box that may never acknowledge: a stopped move |
| 847 | // target holds the relocation at pinning with no deadline and no stuck |
| 848 | // (a liveness guard: TestConvertedMoveDropsTheMoveTargetFromTheBarrier). |
| 849 | func (w *world) relocNameTarget(c *claim) { |
| 850 | op := c.Op |
| 851 | to := w.chooseBoxExcept(op.SrcRegion, c.Holder.App, relocExcept(op)...) |
| 852 | if to == nil { |
| 853 | return // no eligible box: the recovery pass alerts once (moves §7.10) |
| 854 | } |
| 855 | dropped := false |
| 856 | if w.cas(c, func(n *claim) bool { |
| 857 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "pinning" && n.Op.Tgt < 0 |
| 858 | if ok && on("reloc-name-drops-move-target") { |
| 859 | var parts []int |
| 860 | for _, p := range n.Op.Parts { |
| 861 | if x := w.boxByID(p); x == nil || x.Region == n.Op.SrcRegion { |
| 862 | parts = append(parts, p) |
| 863 | } |
| 864 | } |
| 865 | dropped = len(parts) < len(n.Op.Parts) |
| 866 | n.Op.Parts = parts |
| 867 | } |
| 868 | if ok { |
| 869 | ok = w.nameReplacement(n, to.ID, "pinning") |
| 870 | } |
| 871 | return ok |
| 872 | }) && dropped { |
| 873 | w.stats["name target: the move's target left the participants"]++ |
| 874 | } |
| 875 | } |
| 876 | |
| 877 | // abandonPrepared is abandonPrepared(c) in hostclaim (1b-next ruling |
| 878 | // 27, rev 18 finding 3): before an unopened P is replaced or dropped, it |
| 879 | // is journaled in abandoned_prepared, so the serving CAS parks it. Every |
| 880 | // path that replaces or drops P calls it: install, the replacement |
| 881 | // table's nested branch, the nested relocation after probe-failed and the |
| 882 | // operator's replay. The mutation of reloc-abandoned-prepared records |
| 883 | // nothing, which reopens finding F3. |
| 884 | func (w *world) abandonPrepared(n *claim) { |
| 885 | p := n.Op.Prepared |
| 886 | if p == "" || preparedOpened(n) || hasStr(n.Op.Abandoned, p) || !on("reloc-abandoned-prepared") { |
| 887 | return |
| 888 | } |
| 889 | n.Op.Abandoned = append(n.Op.Abandoned, p) |
| 890 | } |
| 891 | |
| 892 | // handExec is install() in hostclaim: op.exec to box at phase with |
| 893 | // prepared generation P, a new stamp (this CAS's), journaled as |
| 894 | // install_stamp, no receipt, the probe attempts reset for the new |
| 895 | // executor (BeginProbeAttempt counts per box), and the box noted as owing |
| 896 | // a park-local should it be abandoned. |
| 897 | func (w *world) handExec(n *claim, box int, phase, prepared string) { |
| 898 | if prepared != n.Op.Prepared { |
| 899 | w.abandonPrepared(n) // install replaces an unopened P (1b-next ruling 27) |
| 900 | } |
| 901 | n.Op.Exec, n.Op.Phase, n.Op.Prepared = box, phase, prepared |
| 902 | n.Holder.Stamp = w.issueStamp(n, box) |
| 903 | n.Op.InstallStamp = n.Holder.Stamp |
| 904 | if on("reloc-install-receipt") { |
| 905 | n.Op.Receipt = nil |
| 906 | } |
| 907 | n.Op.Attempts = 0 |
| 908 | n.Op.ProbeStop, n.Op.ProbePassedAt = nil, 0 |
| 909 | n.Op.InstalledOn = addUniq(n.Op.InstalledOn, box) |
| 910 | n.Boxes = addUniq(n.Boxes, box) |
| 911 | } |
| 912 | |
| 913 | // relocInstalling: the target builds P, the remote prepared generation, |
| 914 | // exactly as a landing builds its generation (moves §7.6, L16): the |
| 915 | // snapshot [1, N] of the pinned generation, create-only and acknowledged, |
| 916 | // then the level-0 anchor [N, N], acknowledged, then the installed CAS |
| 917 | // (ToInstalled, R7-4). A pin at N = 0 builds nothing: the probe is an |
| 918 | // empty bootstrap (claim P6-D3). Only op.exec installs (rule |
| 919 | // reloc-exec-only-installs, HA2: being in the region is never enough). |
| 920 | func (w *world) relocInstalling(b *box, c *claim) { |
| 921 | op, h := c.Op, c.Holder |
| 922 | if on("reloc-exec-only-installs") && op.Exec != b.ID { |
| 923 | return |
| 924 | } |
| 925 | loc, g := h.Location, op.Prepared |
| 926 | bk := w.buckets[b.Region] |
| 927 | installed := func(n *claim) bool { |
| 928 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "installing" && n.Op.Exec == op.Exec && n.Op.Prepared == g |
| 929 | if ok && on("reloc-exec-ack-barrier") && !acked(n) { |
| 930 | ok = false // ToInstalled's requireAcks (rev 4, Astra R1-4) |
| 931 | } |
| 932 | if ok { |
| 933 | n.Op.Phase = "installed" |
| 934 | } |
| 935 | return ok |
| 936 | } |
| 937 | if !w.buildPreparedFiles(b, c) { |
| 938 | return // Task 4: P's prepared files manifest first |
| 939 | } |
| 940 | if op.Pin.TX == 0 { |
| 941 | w.cas(c, installed) |
| 942 | return |
| 943 | } |
| 944 | if op.BaseKey == "" { |
| 945 | db := w.restoreUpTo(b.Region, h.Inc, loc, op.Pin.Gen, op.Pin.TX) |
| 946 | if db == nil || db.Pos != op.Pin.TX { |
| 947 | w.stats["build: the pinned set cannot be restored yet"]++ |
| 948 | return |
| 949 | } |
| 950 | key := snapKey(loc, g, op.Pin.TX) |
| 951 | if !w.ackedPut(b, key, snapshotOf(db, h.Inc), true) { |
| 952 | return |
| 953 | } |
| 954 | v := bk.current(key) |
| 955 | if v == nil { |
| 956 | return |
| 957 | } |
| 958 | w.cas(c, func(n *claim) bool { |
| 959 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "installing" && n.Op.Exec == op.Exec && n.Op.Prepared == g && n.Op.BaseKey == "" |
| 960 | if ok { |
| 961 | n.Op.BaseKey, n.Op.BaseVer, n.Op.BaseTX = key, v.ID, op.Pin.TX |
| 962 | } |
| 963 | return ok |
| 964 | }) |
| 965 | return |
| 966 | } |
| 967 | if !w.anchorAcked(b, loc, g, op.BaseKey, op.BaseVer, op.BaseTX) { |
| 968 | return |
| 969 | } |
| 970 | if w.cas(c, installed) { |
| 971 | w.logf("relocation %s: P %q built on box %d at TXID %d", op.ID, g, b.ID, op.BaseTX) |
| 972 | } |
| 973 | } |
| 974 | |
| 975 | // buildPreparedFiles writes P's prepared files manifest before anything |
| 976 | // of the database (1b-next Task 8b; CopyPinnedManifest; claim rev 6.30 |
| 977 | // R18-1, moves rev 7.22 §7.6): create-only at sequence 0, bound to |
| 978 | // {location, P, op id, install_stamp}, with the pinned manifest's files |
| 979 | // and wake state; the installed CAS waits for its acknowledgement |
| 980 | // (ToInstalled requires files_seq 0 and the pin's source sequence as |
| 981 | // provenance). The first admission consumes it as the session |
| 982 | // publisher's baseline, and the publisher starts at sequence 1 (rule |
| 983 | // files-prepared-seq-zero); the mutation keeps the source manifest's |
| 984 | // sequence, as rev 14 did, so the session's first manifests sort below |
| 985 | // it. In the one-box shape there is no publisher (ruling 11) and nothing |
| 986 | // to write. |
| 987 | func (w *world) buildPreparedFiles(b *box, c *claim) bool { |
| 988 | if w.perRegion <= 1 { |
| 989 | return true |
| 990 | } |
| 991 | op, h := c.Op, c.Holder |
| 992 | seq := 0 |
| 993 | if !on("files-prepared-seq-zero") && op.Pin != nil { |
| 994 | seq = op.Pin.ManSeq |
| 995 | } |
| 996 | bl := &blob{Kind: "files", Inc: h.Inc, Gen: op.Prepared, Seq: seq, Files: w.relocFiles(c).clone(), Timers: append([]int(nil), w.relocTimers(c)...), Text: fmt.Sprintf("%s/%d", op.ID, op.InstallStamp), Base: w.issued} |
| 997 | return w.ackedPut(b, manifestKey(h.Location, op.Prepared, seq), bl, true) |
| 998 | } |
| 999 | |
| 1000 | // localInstall advances the executor's local install by one step (§5.8, |
| 1001 | // B1-2; 1b-next Task 8c's Install) and reports whether it is whole: park |
| 1002 | // what an earlier stay left as retire-artifacts of its stamp (claim §6.4), |
| 1003 | // extract the files, replace the timers with the pinned wake state (the |
| 1004 | // files Done marker last), render and materialize with current inputs, |
| 1005 | // and write the box's install journal (its receipt) under install_stamp. |
| 1006 | // EnsureLocal reuses it to restore a retired install under the same |
| 1007 | // stamp. |
| 1008 | func (w *world) localInstall(b *box, c *claim) bool { |
| 1009 | op, h := c.Op, c.Holder |
| 1010 | g := op.Prepared |
| 1011 | if b.Files == nil || b.Files.Gen != g { |
| 1012 | for _, st := range []int{dbStamp(b.DB), filesStamp(b.Files)} { |
| 1013 | if st != 0 && st != b.stampOf(h.Inc) { |
| 1014 | w.retireArtifacts(b, h.Inc, st) |
| 1015 | } |
| 1016 | } |
| 1017 | if b.DB != nil { |
| 1018 | b.DB = nil // never reused (claim R6); the probe's admission restores |
| 1019 | b.Parked++ |
| 1020 | } |
| 1021 | src := w.relocFiles(c) |
| 1022 | if src == nil { |
| 1023 | w.nextID++ |
| 1024 | src = &filesDir{Inc: h.Inc, Key: w.nextID, Env: w.nextID} |
| 1025 | } |
| 1026 | f := src.clone() |
| 1027 | f.Inc, f.Gen, f.Done, f.Stamp = h.Inc, g, false, b.stampOf(h.Inc) |
| 1028 | if !on("markers-last") { |
| 1029 | f.Done = true |
| 1030 | } |
| 1031 | b.Files = f |
| 1032 | b.pendingTimers = append([]int(nil), w.relocTimers(c)...) |
| 1033 | return false |
| 1034 | } |
| 1035 | if !b.Files.Done { |
| 1036 | b.Timers = append([]int(nil), b.pendingTimers...) // ReplaceTimers: the pinned wake state |
| 1037 | b.Attempt, b.History = 0, nil |
| 1038 | b.Files.Done = true |
| 1039 | b.installEpoch = b.Epoch |
| 1040 | return false |
| 1041 | } |
| 1042 | if b.HasKey { |
| 1043 | b.Config, b.CfgVer = h.Inc, w.cfgVer[h.App] |
| 1044 | } |
| 1045 | b.Live, b.LiveInc = h.App, h.Inc |
| 1046 | b.receipt = &receiptEntry{Op: op.ID, Box: b.ID, Stamp: op.InstallStamp} |
| 1047 | return true |
| 1048 | } |
| 1049 | |
| 1050 | // relocInstalled: the executor's local install, then its receipt by CAS |
| 1051 | // (RecordLocalInstall), bound to this box and install_stamp; probing |
| 1052 | // needs that exact receipt (ToProbing; rule reloc-install-receipt), and |
| 1053 | // the install is verified, and restored if a lost lease's scoped cleanup |
| 1054 | // retired it, before the probing CAS (EnsureLocal; rule |
| 1055 | // reloc-ensure-local). The mutation of reloc-install-receipt lets a |
| 1056 | // replacement inherit its predecessor's receipt and skip its install. |
| 1057 | func (w *world) relocInstalled(b *box, c *claim) { |
| 1058 | op := c.Op |
| 1059 | if on("reloc-exec-only-installs") && op.Exec != b.ID { |
| 1060 | return |
| 1061 | } |
| 1062 | probing := func(n *claim) bool { |
| 1063 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "installed" && n.Op.Exec == op.Exec && n.Op.Receipt != nil |
| 1064 | if ok && on("reloc-exec-ack-barrier") && !acked(n) { |
| 1065 | ok = false // ToProbing's requireAcks (rev 4, Astra R1-4) |
| 1066 | } |
| 1067 | if ok { |
| 1068 | n.Op.Phase, n.Op.ProbingSince = "probing", w.now |
| 1069 | } |
| 1070 | return ok |
| 1071 | } |
| 1072 | if r := op.Receipt; r != nil { |
| 1073 | mine := r.Box == b.ID && r.Stamp == op.InstallStamp |
| 1074 | if !mine && !on("reloc-install-receipt") { |
| 1075 | w.cas(c, probing) // the mutation: any receipt will do, the install is skipped |
| 1076 | return |
| 1077 | } |
| 1078 | if mine { |
| 1079 | if w.ensureLocal(b, c) { |
| 1080 | w.cas(c, probing) |
| 1081 | } |
| 1082 | return |
| 1083 | } |
| 1084 | } |
| 1085 | if op.Exec != b.ID || !w.localInstall(b, c) { |
| 1086 | return |
| 1087 | } |
| 1088 | w.cas(c, func(n *claim) bool { |
| 1089 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "installed" && n.Op.Exec == b.ID && n.Op.InstallStamp == op.InstallStamp && !preparedOpened(n) |
| 1090 | if ok { |
| 1091 | n.Op.Receipt = &receiptEntry{Op: op.ID, Box: b.ID, Stamp: op.InstallStamp} |
| 1092 | } |
| 1093 | return ok |
| 1094 | }) |
| 1095 | } |
| 1096 | |
| 1097 | // ensureLocal is EnsureLocal (1b-next rev 15 finding 2): before P's first |
| 1098 | // admission the executor's local install must be whole, so it is |
| 1099 | // verified before the probing CAS and before every attempt, and a |
| 1100 | // retired one (the scoped cleanup of a lost controller lease retires an |
| 1101 | // unopened P's local install, role()) is restored under the same |
| 1102 | // install_stamp. It never touches an opened P. It reports whether the |
| 1103 | // install is whole now. The mutation goes on without verifying, so P's |
| 1104 | // first admission can meet a missing install. |
| 1105 | func (w *world) ensureLocal(b *box, c *claim) bool { |
| 1106 | op := c.Op |
| 1107 | if preparedOpened(c) || !on("reloc-ensure-local") { |
| 1108 | return true |
| 1109 | } |
| 1110 | if b.Files != nil && b.Files.Gen == op.Prepared && b.Files.Done && b.receipt != nil && b.receipt.Op == op.ID && b.receipt.Stamp == op.InstallStamp { |
| 1111 | return true |
| 1112 | } |
| 1113 | if w.localInstall(b, c) { |
| 1114 | w.stats["local install restored before the first admission"]++ |
| 1115 | w.logf("relocation %s: box %d restored its retired local install under stamp %d", op.ID, b.ID, op.InstallStamp) |
| 1116 | return true |
| 1117 | } |
| 1118 | return false |
| 1119 | } |
| 1120 | |
| 1121 | // verifyReceipt is VerifyReceipt (1b-next rev 16 finding 1): once P is |
| 1122 | // opened the local install is never remade, and before every attempt the |
| 1123 | // box's durable receipt (its install journal) must still be the one the |
| 1124 | // claim journaled; a missing or different one marks the operation stuck |
| 1125 | // and nothing is admitted. Files, env and timers the admitted writers |
| 1126 | // changed are never compared. The mutation goes on to the next attempt. |
| 1127 | func (w *world) verifyReceipt(b *box, c *claim) bool { |
| 1128 | op := c.Op |
| 1129 | if r := b.receipt; r != nil && r.Op == op.ID && r.Box == b.ID && r.Stamp == op.InstallStamp { |
| 1130 | return true |
| 1131 | } |
| 1132 | if !on("reloc-verify-receipt") { |
| 1133 | w.stats["receipt missing after P opened, attempt made anyway (mutation)"]++ |
| 1134 | return true |
| 1135 | } |
| 1136 | w.markStuck(c, fmt.Sprintf("box %d's local install receipt is missing after P was admitted", b.ID)) |
| 1137 | return false |
| 1138 | } |
| 1139 | |
| 1140 | // preparedOpened is PreparedOpened: an admission has named P (B2-5). The |
| 1141 | // admission that opens P journals prepared_opened in the same CAS |
| 1142 | // (admitCommit), so the fact outlives holder.admissions' trim to eight |
| 1143 | // (1b-next rev 12 S1; rule reloc-prepared-opened-journal); the mutation |
| 1144 | // reads the retained admissions alone, as rev 11 did. |
| 1145 | func preparedOpened(c *claim) bool { |
| 1146 | if c.Op.Prepared == "" || c.Holder == nil { |
| 1147 | return false |
| 1148 | } |
| 1149 | if on("reloc-prepared-opened-journal") && c.Op.PreparedOpened == c.Op.Prepared { |
| 1150 | return true |
| 1151 | } |
| 1152 | for _, e := range c.Holder.Admissions { |
| 1153 | if e.Gen == c.Op.Prepared { |
| 1154 | return true |
| 1155 | } |
| 1156 | } |
| 1157 | return false |
| 1158 | } |
| 1159 | |
| 1160 | // relocProbing: each attempt is counted in the claim before it is made |
| 1161 | // (BeginProbeAttempt, ruling 19; rule reloc-probe-attempts: a crash, a |
| 1162 | // lost response or a restart never buys a fourth), then one cold |
| 1163 | // AdmitWriter by op.exec into P (the first) or a fresh generation |
| 1164 | // restored from the previous attempt's (claim §6.4a; spawn and admitWriter |
| 1165 | // do both), then the probe. The fourth is refused and the target enters |
| 1166 | // probe-failed, as it does past the probe deadline. Before P's first |
| 1167 | // admission the local install is verified and restored (ensureLocal); |
| 1168 | // after it, the receipt alone is verified (verifyReceipt) and the install |
| 1169 | // is never remade (rule reloc-no-reinstall; its mutation reapplies the |
| 1170 | // pinned wake state on every agent restart, as rev 4.2's install marker |
| 1171 | // did, over the timers the admitted session registered). |
| 1172 | func (w *world) relocProbing(b *box, c *claim) { |
| 1173 | op, h := c.Op, c.Holder |
| 1174 | failed := func(n *claim) bool { |
| 1175 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "probing" && n.Op.Exec == op.Exec |
| 1176 | if ok { |
| 1177 | n.Op.Phase, n.Op.FailedAt, n.Op.Release = "probe-failed", w.now, nil |
| 1178 | } |
| 1179 | return ok |
| 1180 | } |
| 1181 | if op.Exec != b.ID { |
| 1182 | if w.now > op.ProbingSince+probeDeadline && op.Exec >= 0 { |
| 1183 | w.stats["probe-failed: deadline"]++ |
| 1184 | w.cas(c, failed) // the worker's: ToProbeFailed past the probe deadline |
| 1185 | } |
| 1186 | return |
| 1187 | } |
| 1188 | if b.Row == nil || b.Row.Status == "ready" { |
| 1189 | w.holdRow(b, "relocating", op.ID, c) |
| 1190 | } |
| 1191 | opened := preparedOpened(c) |
| 1192 | if !opened { |
| 1193 | if !w.ensureLocal(b, c) { |
| 1194 | return |
| 1195 | } |
| 1196 | } else if !w.verifyReceipt(b, c) { |
| 1197 | return |
| 1198 | } |
| 1199 | if !on("reloc-no-reinstall") && opened && b.installEpoch != b.Epoch { |
| 1200 | b.Timers = append([]int(nil), w.relocTimers(c)...) // the mutation: the install marker re-applied after a restart |
| 1201 | b.installEpoch = b.Epoch |
| 1202 | w.stats["local install remade after admission (mutation)"]++ |
| 1203 | } |
| 1204 | if !w.admits(b, c, op.ID) { |
| 1205 | return |
| 1206 | } |
| 1207 | if b.HasKey { |
| 1208 | b.Config, b.CfgVer, b.Live, b.LiveInc = h.Inc, w.cfgVer[h.App], h.App, h.Inc |
| 1209 | } |
| 1210 | probed := func(n *claim) bool { |
| 1211 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Phase == "probing" && n.Op.Exec == b.ID |
| 1212 | if ok { |
| 1213 | n.Op.Phase, n.Op.ProbePassedAt = "probed", w.now // RecordProbe's At (ruling 67's bound) |
| 1214 | } |
| 1215 | return ok |
| 1216 | } |
| 1217 | if hasMain(b) { |
| 1218 | w.probeAccepted(b, c) |
| 1219 | w.cas(c, probed) |
| 1220 | return |
| 1221 | } |
| 1222 | if w.now > op.ProbingSince+probeDeadline { |
| 1223 | w.stats["probe-failed: deadline"]++ |
| 1224 | w.cas(c, failed) |
| 1225 | return |
| 1226 | } |
| 1227 | tag := fmt.Sprintf("%s/%d", op.ID, op.Attempts) |
| 1228 | if b.open == nil { |
| 1229 | // a new attempt: counted before it is made. attemptTag is the |
| 1230 | // attempt this agent counted (tag) or made (tag+"+"); one made and |
| 1231 | // ended, or none counted, is counted now |
| 1232 | if on("reloc-probe-attempts") && b.attemptTag != tag { |
| 1233 | if preparedOpened(c) && w.c11Fallback() { |
| 1234 | // Preflight's ErrLaterAttemptUnlaunchable (ruling 58, until |
| 1235 | // claim ask C11): root pins every relocating launch to |
| 1236 | // install_stamp, so a cold attempt after P's opening is not |
| 1237 | // made; probe-failed now, before any attempt is counted |
| 1238 | w.stats["probe-failed: a later attempt root cannot launch (C11 open)"]++ |
| 1239 | w.cas(c, failed) |
| 1240 | return |
| 1241 | } |
| 1242 | if op.Attempts >= probeTries { |
| 1243 | w.stats["probe-failed: attempts"]++ |
| 1244 | w.cas(c, failed) |
| 1245 | return |
| 1246 | } |
| 1247 | if w.cas(c, func(n *claim) bool { |
| 1248 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Phase == "probing" && n.Op.Exec == b.ID && n.Op.Attempts == op.Attempts |
| 1249 | if ok { |
| 1250 | n.Op.Attempts++ |
| 1251 | } |
| 1252 | return ok |
| 1253 | }) { |
| 1254 | b.attemptTag = fmt.Sprintf("%s/%d", op.ID, op.Attempts+1) |
| 1255 | } |
| 1256 | return |
| 1257 | } |
| 1258 | if !on("reloc-probe-attempts") && op.ProbeTries >= probeTries { |
| 1259 | w.stats["probe-failed: attempts"]++ |
| 1260 | w.cas(c, failed) // the mutation: failures counted after the fact, as rev 1 did |
| 1261 | return |
| 1262 | } |
| 1263 | } |
| 1264 | if b.open == nil { |
| 1265 | b.attemptTag = tag + "+" // made now |
| 1266 | } |
| 1267 | if w.spawn(b, c, op.Prepared, true) { |
| 1268 | w.probeAccepted(b, c) |
| 1269 | w.cas(c, probed) |
| 1270 | } |
| 1271 | } |
| 1272 | |
| 1273 | // relocProbeFailed: the live target runs its own release sequence (§5.8 |
| 1274 | // probe-failed; JournalRelease), then a worker starts the nested |
| 1275 | // relocation from pinning (PinAfterRelease with a next target), pinned to |
| 1276 | // at least the target's final TXID (the release entry, which |
| 1277 | // relocPinning reads), the failed target excluded and counted as a |
| 1278 | // replacement. Exhaustion is checked before a replacement is looked for |
| 1279 | // (1b-next R1-16): with the replacements used the operation is stuck. A |
| 1280 | // target that cannot finish its release within releaseDeadline goes to |
| 1281 | // stuck (ruling 14): nothing moves away from a live box without its stop |
| 1282 | // evidence. With no eligible box for releaseDeadline the operation is |
| 1283 | // stuck too (model ruling 14: 1b-next waits for a spare with no bound, |
| 1284 | // and in a two-box region the source is never a spare). |
| 1285 | func (w *world) relocProbeFailed(b *box, c *claim) { |
| 1286 | op, h := c.Op, c.Holder |
| 1287 | if op.Release == nil || op.Release.Box != op.Exec { |
| 1288 | if on("reloc-release-deadline") && w.now > op.FailedAt+releaseDeadline { |
| 1289 | w.markStuck(c, fmt.Sprintf("box %d cannot finish its release sequence", op.Exec)) |
| 1290 | return |
| 1291 | } |
| 1292 | if b.ID != op.Exec { |
| 1293 | return |
| 1294 | } |
| 1295 | w.relocJournalRelease(b, c, false) |
| 1296 | return |
| 1297 | } |
| 1298 | if on("reloc-replacement-cap") && op.Replacements >= maxReplacements { |
| 1299 | w.markStuck(c, fmt.Sprintf("two replacements of the target used; a third was needed at %s", op.Phase)) |
| 1300 | return |
| 1301 | } |
| 1302 | next := w.chooseBoxExcept(op.SrcRegion, h.App, relocExcept(op, op.Exec)...) |
| 1303 | if next == nil { |
| 1304 | // 1b-next ruling 26 (rev 17, finding F1; rule reloc-no-spare-bound): |
| 1305 | // with no spare ReleaseDeadline after probe_failed_since, stuck; |
| 1306 | // never the source, which the operator's resolve alone may name. |
| 1307 | // The mutation waits for a spare for ever, rev 16's behaviour |
| 1308 | if on("reloc-no-spare-bound") && w.now > op.FailedAt+releaseDeadline { |
| 1309 | w.markStuck(c, "no eligible box for the nested relocation") |
| 1310 | } |
| 1311 | return |
| 1312 | } |
| 1313 | w.cas(c, func(n *claim) bool { |
| 1314 | if !sameOp(n.Op.ID, op.ID) || n.Op.Kind != "relocating" || n.Op.Phase != "probe-failed" || n.Op.Release == nil || n.Op.Release.Box != n.Op.Exec { |
| 1315 | return false |
| 1316 | } |
| 1317 | if !w.nameReplacement(n, next.ID, "pinning") { |
| 1318 | return n.Op.Stuck != "" // stuck: the CAS records it |
| 1319 | } |
| 1320 | w.abandonPrepared(n) |
| 1321 | n.Op.Exec = -1 |
| 1322 | n.Op.Pin, n.Op.Prepared, n.Op.Receipt = nil, "", nil |
| 1323 | n.Op.BaseKey, n.Op.BaseVer, n.Op.BaseTX = "", 0, 0 |
| 1324 | n.Op.ProbeTries, n.Op.Attempts = 0, 0 |
| 1325 | w.stats["nested relocation after probe-failed"]++ |
| 1326 | return true |
| 1327 | }) |
| 1328 | } |
| 1329 | |
| 1330 | // relocSettle is the end of a relocation as 1b-next builds it (L18 joint |
| 1331 | // round finding 2; regionha_completion_test.go): at probed (1b-next's |
| 1332 | // probing with the probe journaled) op.exec stops the probe's session with |
| 1333 | // root's receipt (relocStopProbe, ruling 49), bounded by ruling 67 (no |
| 1334 | // stop ReleaseDeadline after the probe passed is stuck for an operator); |
| 1335 | // then any worker of the region makes the serving CAS (relocComplete, |
| 1336 | // CompleteRelocation), which records the completion. The row stays held: |
| 1337 | // the completion pass releases it after the discharge, and the first |
| 1338 | // serving wake is a separate admission (N6). Rev 7.20's settle window and |
| 1339 | // open are gone: a probe that dies after it passed is stopped all the same. |
| 1340 | func (w *world) relocSettle(b *box, c *claim) { |
| 1341 | op := c.Op |
| 1342 | if op.Phase != "probed" { |
| 1343 | return |
| 1344 | } |
| 1345 | if op.ProbeStop == nil && on("reloc-probe-stop") { |
| 1346 | if w.now > op.ProbePassedAt+releaseDeadline { |
| 1347 | w.markStuck(c, fmt.Sprintf("box %d passed its probe but has no root stop receipt for session %d", op.Exec, c.Holder.Session)) |
| 1348 | return |
| 1349 | } |
| 1350 | if b.ID == op.Exec { |
| 1351 | w.relocStopProbe(b, c) |
| 1352 | } |
| 1353 | return |
| 1354 | } |
| 1355 | w.relocComplete(c) |
| 1356 | } |
| 1357 | |
| 1358 | // scopedCleanup is the relocation controller's scoped cleanup when its |
| 1359 | // lease was lost (1b-next rev 14 finding 1, Task 7's live): an unopened |
| 1360 | // P's local install is retired and its receipt (the install journal) |
| 1361 | // kept, so EnsureLocal restores it before the first admission. role() |
| 1362 | // calls it when the executor's execution lease expired before it renewed |
| 1363 | // it (a freeze, a pause, a partition longer than the TTL). |
| 1364 | func (w *world) scopedCleanup(b *box, c *claim) { |
| 1365 | if c == nil || c.Holder == nil || c.Op.Kind != "relocating" || c.Op.Exec != b.ID || preparedOpened(c) { |
| 1366 | return |
| 1367 | } |
| 1368 | if f := b.Files; f != nil && f.Gen == c.Op.Prepared && f.Gen != "" { |
| 1369 | w.retireArtifacts(b, c.Holder.Inc, f.Stamp) |
| 1370 | w.stats["relocation controller lost: unopened local install retired"]++ |
| 1371 | w.logf("box %d: relocation %s's controller lease was lost; the scoped cleanup retired its unopened local install", b.ID, c.Op.ID) |
| 1372 | } |
| 1373 | } |
| 1374 | |
| 1375 | // ---------- termination, enrollment and the fence (region HA §5.9; H1, H10, H11, H16, H17, H19) ---------- |
| 1376 | |
| 1377 | // terminate is the cloud terminating a box (H1): nothing on it runs |
| 1378 | // again, it never returns (a stopped box may; heal skips a terminated |
| 1379 | // one), and the requests it issued still complete at their own time, |
| 1380 | // which is the late-PUT-after-termination case. The two-box step draws |
| 1381 | // it; it never leaves the world without an enrolled live box, since H17's |
| 1382 | // cross-region confirmer must exist for a replacement to enroll. |
| 1383 | func (w *world) terminate(b *box) { |
| 1384 | if b.Destroyed { |
| 1385 | return |
| 1386 | } |
| 1387 | enrolled := 0 |
| 1388 | for _, x := range w.liveBoxes() { |
| 1389 | if x.Enrolled && x != b { |
| 1390 | enrolled++ |
| 1391 | } |
| 1392 | } |
| 1393 | if enrolled == 0 { |
| 1394 | return |
| 1395 | } |
| 1396 | b.Destroyed, b.Terminated, b.TerminatedAt = true, true, w.now |
| 1397 | b.Procs, b.plans = nil, nil |
| 1398 | w.stats["terminated"]++ |
| 1399 | w.logf("box %d terminated by the cloud (region %d)", b.ID, b.Region) |
| 1400 | } |
| 1401 | |
| 1402 | // launchReplacements: the group keeps the region at its minimum (H5): a |
| 1403 | // replacement launches replaceDelay steps after the termination that left |
| 1404 | // the region short, unenrolled (H16), with its volume flag drawn (two-box |
| 1405 | // sweeps only: a witness world's quiet source draws nothing). |
| 1406 | func (w *world) launchReplacements() { |
| 1407 | for r := 0; r < nRegions; r++ { |
| 1408 | if len(w.regionBoxes(r)) >= w.regionMin(r) { |
| 1409 | continue |
| 1410 | } |
| 1411 | last := 0 |
| 1412 | for _, b := range w.boxes { |
| 1413 | if b.Region == r && b.Terminated && b.TerminatedAt > last { |
| 1414 | last = b.TerminatedAt |
| 1415 | } |
| 1416 | } |
| 1417 | if last == 0 || w.now < last+replaceDelay { |
| 1418 | continue |
| 1419 | } |
| 1420 | w.nextID++ |
| 1421 | nb := &box{ID: 100 + w.nextID, Region: r, HasKey: true, EnrollAt: w.now, DeleteOnTermination: w.rng.Intn(8) != 0} |
| 1422 | nb.forget() |
| 1423 | w.boxes = append(w.boxes, nb) |
| 1424 | w.stats["replacements launched"]++ |
| 1425 | w.logf("box %d launched in region %d, unenrolled", nb.ID, r) |
| 1426 | } |
| 1427 | } |
| 1428 | |
| 1429 | // enrollStep is automatic enrollment (H16, L19): a launched box's boxid |
| 1430 | // evidence verifies and an enrolled box confirms its group membership |
| 1431 | // enrollDelay steps after launch. Any enrolled box of any region may |
| 1432 | // confirm (H17: the first box of an emptied region is confirmed from |
| 1433 | // another region); with none, nothing enrolls. |
| 1434 | func (w *world) enrollStep() { |
| 1435 | confirmer := false |
| 1436 | for _, b := range w.liveBoxes() { |
| 1437 | if b.Enrolled { |
| 1438 | confirmer = true |
| 1439 | } |
| 1440 | } |
| 1441 | if !confirmer { |
| 1442 | return |
| 1443 | } |
| 1444 | for _, b := range w.liveBoxes() { |
| 1445 | if !b.Enrolled && w.now >= b.EnrollAt+enrollDelay { |
| 1446 | b.Enrolled = true |
| 1447 | w.stats["boxes enrolled automatically"]++ |
| 1448 | w.logf("box %d enrolled (region %d)", b.ID, b.Region) |
| 1449 | } |
| 1450 | } |
| 1451 | } |
| 1452 | |
| 1453 | // twoBoxStep is the two-box shape's extra step (ruling 1): replacements |
| 1454 | // launch and enroll; during the chaos a box is terminated now and then, |
| 1455 | // and a drain is asked for now and then. |
| 1456 | func (w *world) twoBoxStep() { |
| 1457 | w.launchReplacements() |
| 1458 | w.enrollStep() |
| 1459 | for _, b := range w.liveBoxes() { |
| 1460 | w.renewController(b) |
| 1461 | w.publishFiles(b) |
| 1462 | } |
| 1463 | if !w.calm && w.fail["terminate"] && w.rng.Intn(200) == 0 { |
| 1464 | bs := w.liveBoxes() |
| 1465 | if b := bs[w.rng.Intn(len(bs))]; w.regionHAOwnsLoss(b) { |
| 1466 | w.terminate(b) |
| 1467 | } |
| 1468 | } |
| 1469 | if !w.calm && w.rng.Intn(60) == 0 { |
| 1470 | w.userRelocate() |
| 1471 | } |
| 1472 | if !w.calm && w.rng.Intn(400) == 0 { |
| 1473 | bs := w.liveBoxes() |
| 1474 | w.updateExecutable(bs[w.rng.Intn(len(bs))]) // claim §5.10: refused while a controller record exists (rev 4) |
| 1475 | } |
| 1476 | } |
| 1477 | |
| 1478 | // regionHAOwnsLoss: the loss of b is region HA's to recover (model ruling |
| 1479 | // 15): the claim holds the host and names b, if at all, only as a serving |
| 1480 | // executor, a relocation's participant or a chore's owner; a box holding |
| 1481 | // a legacy host's state before its adoption is never terminated. A participant of a move, |
| 1482 | // a landing, a rollback, a revert, a push, a create or a delete is |
| 1483 | // recovered from a lost box by moves' and the claim's declare-lost paths |
| 1484 | // (L14), which the model walks for a destroyed box that is stopped and |
| 1485 | // may return; the sweep never terminates one. |
| 1486 | func (w *world) regionHAOwnsLoss(b *box) bool { |
| 1487 | c := w.claim |
| 1488 | if c == nil || c.Holder == nil { |
| 1489 | return b.Row == nil && b.DB == nil && b.Files == nil // a legacy host not yet adopted is not region HA's |
| 1490 | } |
| 1491 | switch c.Op.Kind { |
| 1492 | case "serving", "relocating": |
| 1493 | if c.Op.Exec == b.ID && w.perRegion > 1 { |
| 1494 | // the L21 window (plan ruling 17): an executor whose newest |
| 1495 | // landed manifest carries no instance.key (none landed yet, or |
| 1496 | // only the pre-start copy) is unrecoverable by any relocation: |
| 1497 | // its failover is stuck by design (1b-next ruling 25) and never |
| 1498 | // converges, so the sweep leaves it alone; the witness for |
| 1499 | // files-pin-needs-key and TestFailoverBeforeTheFirstManifest |
| 1500 | // show it fail safe |
| 1501 | _, _, _, _, m := w.newestManifest(b.Region, c.Holder.Location, c.Holder.Gen) |
| 1502 | return manifestHasKey(m) |
| 1503 | } |
| 1504 | return true |
| 1505 | } |
| 1506 | return !(c.Op.Exec == b.ID || contains(c.Op.Parts, b.ID) || c.Op.Src == b.ID || c.Op.Tgt == b.ID || c.Op.NewBox == b.ID) |
| 1507 | } |
| 1508 | |
| 1509 | // fenceStep is the regional worker's termination fence (§5.9; claim L2 |
| 1510 | // to L5; Worker.Step's "evidence first"): for a terminated, enrolled box |
| 1511 | // of its own region that the claim still names, it writes the fence in |
| 1512 | // one CAS (fence). A stopped box is never fenced (rule |
| 1513 | // fence-terminated-only: "stopped and stopping do not count, because a |
| 1514 | // stopped box can start again with its disk"); the mutation fences any |
| 1515 | // destroyed box. An unenrolled box has no roster entry to bind the fence |
| 1516 | // to (HA7) and is never fenced. The source's fence is consumed before the |
| 1517 | // target's, as Worker.Step orders them. |
| 1518 | func (w *world) fenceStep(b *box, c *claim) (changed, pending bool) { |
| 1519 | if c == nil || !b.Enrolled || contains(c.Lost, b.ID) || w.haOff { |
| 1520 | return false, false |
| 1521 | } |
| 1522 | var cands []*box |
| 1523 | for _, x := range w.boxes { |
| 1524 | if x.Region != b.Region || x == b || !x.Destroyed || !x.Enrolled || contains(c.Lost, x.ID) { |
| 1525 | continue |
| 1526 | } |
| 1527 | if !x.Terminated && on("fence-terminated-only") { |
| 1528 | continue |
| 1529 | } |
| 1530 | if !w.claimNames(c, x) { |
| 1531 | continue |
| 1532 | } |
| 1533 | cands = append(cands, x) |
| 1534 | } |
| 1535 | if len(cands) == 0 { |
| 1536 | return false, false |
| 1537 | } |
| 1538 | // Worker.Step's order (rev 8): the source's fence, then the target's, |
| 1539 | // then any other box the claim names (a chore's owner) |
| 1540 | rank := func(x *box) int { |
| 1541 | switch { |
| 1542 | case c.Op.Kind == "relocating" && x.ID == c.Op.Src: |
| 1543 | return 0 |
| 1544 | case c.Op.Kind == "relocating" && x.ID == c.Op.Tgt: |
| 1545 | return 1 |
| 1546 | } |
| 1547 | return 2 |
| 1548 | } |
| 1549 | sort.SliceStable(cands, func(i, j int) bool { return rank(cands[i]) < rank(cands[j]) }) |
| 1550 | if x := cands[0]; x.FenceObservedAt == 0 { |
| 1551 | // hostfence.TerminationWorker: the first worker to observe the |
| 1552 | // terminated state writes the box-fences record with that |
| 1553 | // observed_at; every deadline and delay runs from it, never from |
| 1554 | // the termination itself, which no worker saw (rev 4, Astra R1-5) |
| 1555 | x.FenceObservedAt = w.now |
| 1556 | w.logf("box %d observed box %d terminated: fence record, observed_at %d", b.ID, x.ID, w.now) |
| 1557 | } |
| 1558 | before := w.claim.Ver |
| 1559 | w.fence(b, c, cands[0]) |
| 1560 | if w.claim.Ver != before { |
| 1561 | return true, false |
| 1562 | } |
| 1563 | // nothing written: for a relocation's source or target the fence is |
| 1564 | // pending (rule reloc-fence-pending-blocks) |
| 1565 | return false, c.Op.Kind == "relocating" && rank(cands[0]) < 2 |
| 1566 | } |
| 1567 | |
| 1568 | // claimNames: the claim gives x a role, so a fence changes it. |
| 1569 | func (w *world) claimNames(c *claim, x *box) bool { |
| 1570 | if c.Op.Exec == x.ID || contains(c.Op.Parts, x.ID) || c.Op.Kind == "creating" && c.Op.NewBox == x.ID || c.Op.Kind == "moving" && c.Op.Src == x.ID { |
| 1571 | return true |
| 1572 | } |
| 1573 | if c.Op.Kind == "relocating" && (c.Op.Src == x.ID || c.Op.Tgt == x.ID) { |
| 1574 | return true |
| 1575 | } |
| 1576 | for _, o := range c.Obl { |
| 1577 | if o.Owner == boxOwner(x.ID) { |
| 1578 | return true |
| 1579 | } |
| 1580 | } |
| 1581 | return w.owesLocal(c, x) |
| 1582 | } |
| 1583 | |
| 1584 | // boxOwner is an obligation's owner box:<b> (claim §2): the legacy form a |
| 1585 | // regional chore may carry; "" is region:<r>, any eligible worker of the |
| 1586 | // region (P6-D5). |
| 1587 | func boxOwner(id int) string { return fmt.Sprintf("box:%d", id) } |
| 1588 | |
| 1589 | // fence is the fence CAS for x, observed terminated by worker b. Moves' |
| 1590 | // cases keep their owners: a pre-flip move's source converts (moves |
| 1591 | // §7.10, the existing CAS); a move's target before the flip, a landing's |
| 1592 | // or a rollback's target are moves' declare-lost path, which this model |
| 1593 | // gives the operator (L14); a create's box is cancelled by the recovery |
| 1594 | // pass (Task 1). Everything else is one CAS here: the box goes on lost |
| 1595 | // with the fence reference and observed_at, its chores are discharged if |
| 1596 | // its volume evidence qualifies, its regional work passes to its region, |
| 1597 | // and the operation's rule applies (StartRelocation's failover, |
| 1598 | // FenceSource, ReplaceTarget, FenceExhaustedTarget). A target with no |
| 1599 | // spare to name is fenced on a later pass (Worker.next: "no eligible box |
| 1600 | // yet"). |
| 1601 | func (w *world) fence(b *box, c *claim, x *box) { |
| 1602 | op := c.Op |
| 1603 | switch { |
| 1604 | case op.Kind == "moving" && op.Src == x.ID: |
| 1605 | if on("fence-converts-moving") && !op.Committed && !(w.rec != nil && w.rec.MoveID == op.ID && w.rec.MoveState == "landing") { |
| 1606 | w.convertToRelocating(c, x) |
| 1607 | } |
| 1608 | return |
| 1609 | case op.Kind == "landing" && op.Committed && op.Tgt == x.ID && on("fence-converts-committed-landing"): |
| 1610 | // finding F5 (rev 4): the record committed the landing, so no |
| 1611 | // rollback is possible, and its dead target can never finish it. |
| 1612 | // The fence makes it a failover in the target's region, as a |
| 1613 | // serving executor's fence does (plan ruling 20); the mutation |
| 1614 | // leaves it to the operator's path, which has nothing to do. The |
| 1615 | // failover keeps the landing's op id, as hostclaim |
| 1616 | // ConvertCommittedLanding does (and as convertToRelocating does |
| 1617 | // before the flip): the move's admission registration is the move |
| 1618 | // id, and judgeRegistration drains it 90 seconds in unless the |
| 1619 | // claim's operation carries that id (hostops RegisterMove) |
| 1620 | id := op.ID |
| 1621 | if w.cas(c, func(n *claim) bool { |
| 1622 | if !sameOp(n.Op.ID, id) || n.Op.Kind != "landing" || !n.Op.Committed || contains(n.Lost, x.ID) { |
| 1623 | return false |
| 1624 | } |
| 1625 | w.startFailover(n, x, nil, id) // the model's conversion (plan ruling 20): its target is named at pinning |
| 1626 | w.fenceBox(n, x) |
| 1627 | return true |
| 1628 | }) { |
| 1629 | w.stats["fences written"]++ |
| 1630 | w.stats["relocation started: failover of a committed landing"]++ |
| 1631 | w.logf("box %d fenced box %d, the committed landing %s's dead target: failover in region %d", b.ID, x.ID, id, x.Region) |
| 1632 | w.declareTimers(x, w.claim.Holder) |
| 1633 | } |
| 1634 | return |
| 1635 | case (op.Kind == "landing" || op.Kind == "restoring") && op.Tgt == x.ID: |
| 1636 | // claim D2's automatic termination for a move's target (rev 4, |
| 1637 | // claim §11 "automatic target loss racing the record commit CAS"): |
| 1638 | // the same record-first declared loss the operator makes |
| 1639 | w.declareTargetLost(c) |
| 1640 | return |
| 1641 | case op.Kind == "moving" && op.Tgt == x.ID, op.Kind == "creating": |
| 1642 | return |
| 1643 | case op.Kind == "relocating" && op.Stuck != "": |
| 1644 | return // Worker.Step consumes no fence of a stuck operation: the operator's resolve does |
| 1645 | } |
| 1646 | id := op.ID |
| 1647 | var to *box |
| 1648 | if c.Holder != nil && op.Kind == "serving" && op.Exec == x.ID { |
| 1649 | var ok bool |
| 1650 | if to, ok = w.failoverStart(c, x); !ok { |
| 1651 | return // no start yet: the claim, and the fence, wait for a later pass |
| 1652 | } |
| 1653 | } |
| 1654 | if w.cas(c, func(n *claim) bool { |
| 1655 | if !sameOp(n.Op.ID, id) || n.Op.Kind != op.Kind || contains(n.Lost, x.ID) { |
| 1656 | return false |
| 1657 | } |
| 1658 | switch { |
| 1659 | case n.Holder != nil && n.Op.Kind == "serving" && n.Op.Exec == x.ID: |
| 1660 | if !w.failoverStartAllowed(n, x, to) { |
| 1661 | w.stats["failover start: refused by the start CAS"]++ |
| 1662 | return false |
| 1663 | } |
| 1664 | w.startFailover(n, x, to, w.newID("rl")) |
| 1665 | case n.Op.Kind == "relocating" && n.Op.Stuck != "": |
| 1666 | return false |
| 1667 | case n.Op.Kind == "relocating" && n.Op.Src == x.ID: |
| 1668 | w.fenceSource(n, x) |
| 1669 | if n.Op.Tgt == x.ID && !w.replaceTarget(n, x) { |
| 1670 | // an operator's replay made the source eligible and named it |
| 1671 | // the target (model ruling 13): its fence is a target loss |
| 1672 | // too, or pinning waits on a lost target for ever |
| 1673 | return false // no spare yet |
| 1674 | } |
| 1675 | case n.Op.Kind == "relocating" && n.Op.Tgt == x.ID: |
| 1676 | if !w.replaceTarget(n, x) { |
| 1677 | return false // no spare yet |
| 1678 | } |
| 1679 | } |
| 1680 | w.fenceBox(n, x) |
| 1681 | return true |
| 1682 | }) { |
| 1683 | w.stats["fences written"]++ |
| 1684 | w.logf("box %d fenced box %d (terminated at %d) under %s/%s", b.ID, x.ID, x.TerminatedAt, w.claim.Op.Kind, w.claim.Op.ID) |
| 1685 | w.declareTimers(x, w.claim.Holder) |
| 1686 | } |
| 1687 | } |
| 1688 | |
| 1689 | // fenceBox is plan 1b-next's fenceBox: lost with the fence reference |
| 1690 | // (FenceBox, FenceAt); the box's regional work (a park-generation or |
| 1691 | // park-replica owned by box:<x>) passed to region:<r> keeping its |
| 1692 | // identity (rule fence-reassigns-regional, 1b-next rev 6 finding 2, as |
| 1693 | // DeclareLost does for a declared loss); and the box's park-local chores |
| 1694 | // discharged only when its volume evidence qualifies (L3, H19: every |
| 1695 | // attached volume DeleteOnTermination while it ran); otherwise they stay |
| 1696 | // open for the operator path, and nothing about recovery waits on them. |
| 1697 | // The mutation of fence-discharge-qualified discharges regardless (INV-26 |
| 1698 | // is the oracle); the mutation of fence-reassigns-regional leaves the |
| 1699 | // regional work with the dead box, where nothing runs it. |
| 1700 | func (w *world) fenceBox(n *claim, x *box) { |
| 1701 | n.Lost = addUniq(n.Lost, x.ID) |
| 1702 | at := x.FenceObservedAt |
| 1703 | if at == 0 { |
| 1704 | at = w.now |
| 1705 | } |
| 1706 | n.Op.FenceBox, n.Op.FenceAt = x.ID, at |
| 1707 | n.Op.Observed = append(n.Op.Observed, at) // fence.<box>.observed_at, retained for every later pin (JournalPin) |
| 1708 | var parts []int |
| 1709 | for _, p := range n.Op.Parts { |
| 1710 | if p != x.ID { |
| 1711 | parts = append(parts, p) |
| 1712 | } |
| 1713 | } |
| 1714 | n.Op.Parts = parts |
| 1715 | if on("fence-reassigns-regional") { |
| 1716 | for i := range n.Obl { |
| 1717 | if o := &n.Obl[i]; o.Kind != "park-local" && o.Owner == boxOwner(x.ID) { |
| 1718 | o.Owner = "" // region:<r>, the chore's id, operation, retirement and scope kept |
| 1719 | w.stats["fence: regional work passed to the region"]++ |
| 1720 | } |
| 1721 | } |
| 1722 | } |
| 1723 | // a fence exists only for a terminated box (TerminationWorker verifies |
| 1724 | // terminated), and only a terminated box's volume evidence says |
| 1725 | // anything: a stopped box keeps its disk, so its park-local chores |
| 1726 | // stay owed whatever DeleteOnTermination says (claim §7.3 D5, H19; |
| 1727 | // rev 10, Astra round 7). The operator's declared loss of a stopped |
| 1728 | // box reaches this function too, and discharges nothing |
| 1729 | qualified := x.Terminated && x.DeleteOnTermination |
| 1730 | if !on("fence-discharge-qualified") { |
| 1731 | qualified = true |
| 1732 | } |
| 1733 | if !qualified { |
| 1734 | w.stats["fence: park-local chores left open (volume evidence)"]++ |
| 1735 | return |
| 1736 | } |
| 1737 | var keep []obligation |
| 1738 | for _, o := range n.Obl { |
| 1739 | if o.Kind == "park-local" && o.Box == x.ID { |
| 1740 | if !x.Terminated || !x.DeleteOnTermination { |
| 1741 | w.violate("INV-26: box %d's park-local chore %s discharged without qualifying volume evidence (§5.9, H19; terminated %v)", x.ID, o.ID, x.Terminated) |
| 1742 | } |
| 1743 | w.stats["fence: park-local chores discharged"]++ |
| 1744 | continue |
| 1745 | } |
| 1746 | keep = append(keep, o) |
| 1747 | } |
| 1748 | n.Obl = keep |
| 1749 | if !x.Terminated || !x.DeleteOnTermination { |
| 1750 | // INV-26 (rev 10): a discharge recorded for the operation's |
| 1751 | // leftovers is a discharge too, though no chore exists yet |
| 1752 | w.violate("INV-26: box %d's local retirement discharged without qualifying volume evidence (§5.9, H19, claim §7.3 D5; terminated %v)", x.ID, x.Terminated) |
| 1753 | } |
| 1754 | n.Op.Discharged = addUniq(n.Op.Discharged, x.ID) |
| 1755 | } |
| 1756 | |
| 1757 | // declareTimers: the timers acknowledged on a fenced box and not yet |
| 1758 | // delivered are declared lost with it and alerted (moves §7.10: no return |
| 1759 | // lane carries them; §5.9: the loss record's timer knowledge is partial). |
| 1760 | // Task 4 narrows it to the timers its last published manifest does not |
| 1761 | // carry. The mutation of fence-declares-timers drops them silently. |
| 1762 | func (w *world) declareTimers(x *box, h *holder) { |
| 1763 | if h == nil || !on("fence-declares-timers") { |
| 1764 | return |
| 1765 | } |
| 1766 | n := 0 |
| 1767 | for i := range w.acks { |
| 1768 | a := &w.acks[i] |
| 1769 | if a.Box == x.ID && a.Inc == h.Inc && a.Kind == "timer" && !a.Exempt && !w.delivered[a.W] && !w.timerCarried(x, h, a.W) { |
| 1770 | a.Exempt = true |
| 1771 | n++ |
| 1772 | } |
| 1773 | } |
| 1774 | if n > 0 { |
| 1775 | w.stats["timers declared lost"] += n |
| 1776 | w.alert("box %d fenced: %d acknowledged timers registered on it are lost with it", x.ID, n) |
| 1777 | } |
| 1778 | } |
| 1779 | |
| 1780 | // timerCarried: the newest landed manifest of the admitted chain carries |
| 1781 | // the timer (Task 4), so the install that follows restores it and the |
| 1782 | // fence does not declare it. INV-28 is the oracle beside it: a timer the |
| 1783 | // manifest published last of the generation carries is never declared, |
| 1784 | // whatever the sequences say. |
| 1785 | func (w *world) timerCarried(x *box, h *holder, id int) bool { |
| 1786 | _, _, _, _, m := w.newestManifest(x.Region, h.Location, h.Gen) |
| 1787 | if m != nil && contains(m.Timers, id) { |
| 1788 | return true |
| 1789 | } |
| 1790 | if last := w.lastLandedManifest(x.Region, h.Location, h.Gen); last != nil && contains(last.Timers, id) { |
| 1791 | w.violate("INV-28: box %d's timer %d is declared lost, but the manifest published last in generation %q carries it (seq %d)", x.ID, id, last.Gen, last.Seq) |
| 1792 | } |
| 1793 | return false |
| 1794 | } |
| 1795 | |
| 1796 | // deadBoxFiles is what a fenced box's files directory contributes to the |
| 1797 | // recovery: in the one-box shape, which has no files publisher (ruling |
| 1798 | // 11), its local directory stands in for what it published (L17), as |
| 1799 | // convertToRelocating's does; in the two-box shape nothing, because a |
| 1800 | // dead box's disk is never read: the files come from the pinned manifest |
| 1801 | // alone. |
| 1802 | func (w *world) deadBoxFiles(x *box) *filesDir { |
| 1803 | if w.perRegion > 1 { |
| 1804 | return nil |
| 1805 | } |
| 1806 | return x.Files.clone() |
| 1807 | } |
| 1808 | |
| 1809 | // startFailover is the fence's start of a relocation for a serving |
| 1810 | // executor (§5.8 reason failover, StartRelocation with a fence; spec 3's |
| 1811 | // placer supplies the decision): op.exec none, the stamp kept, at pinning |
| 1812 | // with op.PinFailover standing for 1b-next's starting (ruling 4). The |
| 1813 | // start names the target to and makes it the only participant (rev 7, |
| 1814 | // Astra round 4 finding 3); only the mutation of |
| 1815 | // reloc-failover-start-guard starts with none, and pinning names one. |
| 1816 | // The target acknowledges before the pin is selected (PinFailover's |
| 1817 | // barrier, rule reloc-failover-ack-before-pin, rev 5 and 6), and the pin |
| 1818 | // waits out failoverPinDelay from the fence. id is the operation's id: a |
| 1819 | // fresh one for a serving executor's failover, the landing's own for a |
| 1820 | // committed landing's conversion (ConvertCommittedLanding keeps it). |
| 1821 | func (w *world) startFailover(n *claim, x *box, to *box, id string) { |
| 1822 | n.Op = opState{Kind: "relocating", ID: id, Phase: "pinning", Exec: -1, Src: x.ID, Tgt: -1, SrcRegion: x.Region, |
| 1823 | RelocReason: "failover", ReqAt: w.now, App: n.Holder.App, SrcGen: n.Holder.Gen, SrcStamp: n.Holder.Stamp, StopFiles: w.deadBoxFiles(x), PinFailover: true} |
| 1824 | if to != nil { |
| 1825 | // StartRelocation names the target and makes it the only |
| 1826 | // participant (rev 7, Astra round 4 finding 3) |
| 1827 | n.Op.Tgt, n.Op.Parts = to.ID, []int{to.ID} |
| 1828 | } |
| 1829 | w.stats["relocation started: failover"]++ |
| 1830 | } |
| 1831 | |
| 1832 | // failoverStart is Worker.Start for a failover (1b-next rev 19; rev 7, |
| 1833 | // Astra round 4 finding 3), the checks before the start CAS in its order: |
| 1834 | // the claim serves on the fenced source, an eligible target exists |
| 1835 | // (Eligible), and the app admits the registration (Admission.Register). |
| 1836 | // It returns the target, or nil when the start must wait for a later |
| 1837 | // pass; the CAS rechecks the admission and applies RelocationStartGuard |
| 1838 | // with the source fenced (failoverStartAllowed). The mutation of |
| 1839 | // reloc-failover-start-guard starts at once with no target, as rev 6 did. |
| 1840 | func (w *world) failoverStart(c *claim, x *box) (*box, bool) { |
| 1841 | if !on("reloc-failover-start-guard") { |
| 1842 | return nil, true |
| 1843 | } |
| 1844 | to := w.chooseBoxExcept(x.Region, c.Holder.App, x.ID) |
| 1845 | if to == nil { |
| 1846 | w.stats["failover start: no eligible target yet"]++ |
| 1847 | return nil, false |
| 1848 | } |
| 1849 | if ad := w.admit[c.Holder.App]; ad != nil && ad.Closed { |
| 1850 | w.stats["failover start: app admission closed"]++ |
| 1851 | return nil, false |
| 1852 | } |
| 1853 | return to, true |
| 1854 | } |
| 1855 | |
| 1856 | // failoverStartAllowed is the start CAS's own checks for a failover: |
| 1857 | // the admission recheck (Admission.Recheck), the target not on lost, and |
| 1858 | // RelocationStartGuard with the source fenced, so the source's own |
| 1859 | // park-local does not block (L11) and a park-replica in the region does. |
| 1860 | func (w *world) failoverStartAllowed(n *claim, x, to *box) bool { |
| 1861 | if !on("reloc-failover-start-guard") { |
| 1862 | return true |
| 1863 | } |
| 1864 | if ad := w.admit[n.Holder.App]; ad != nil && ad.Closed { |
| 1865 | return false |
| 1866 | } |
| 1867 | return to != nil && !contains(n.Lost, to.ID) && w.relocationStartGuard(n, x, to, true) |
| 1868 | } |
| 1869 | |
| 1870 | // fenceSource is §5.8's source loss (B1-6, FenceSource): a drain whose |
| 1871 | // source is terminated before its release is journaled becomes a |
| 1872 | // failover in the same CAS, op.exec none, the stamp kept, the source off |
| 1873 | // the barrier, its files directory the publisher's stand-in; after its |
| 1874 | // release only its liveness changes. (1b-next keeps a source lost in |
| 1875 | // starting at starting until the remaining participant acknowledges, |
| 1876 | // then PinFailover; the model goes to pinning at once with |
| 1877 | // op.PinFailover set for a source lost in starting, so the target |
| 1878 | // acknowledges before the pin is selected, rule |
| 1879 | // reloc-failover-ack-before-pin, and the pin waits out the delay.) The |
| 1880 | // mutation of reloc-source-loss takes a source lost in releasing as |
| 1881 | // released at what the remote holds, so its unuploaded writes go |
| 1882 | // undeclared. |
| 1883 | func (w *world) fenceSource(n *claim, x *box) { |
| 1884 | switch { |
| 1885 | case n.Op.Phase == "starting", n.Op.Phase == "releasing" && n.Op.Release == nil: |
| 1886 | if !on("reloc-source-loss") && n.Op.Phase == "releasing" { |
| 1887 | h := n.Holder |
| 1888 | n.Op.Release = &releaseEntry{Box: x.ID, Session: h.Session, Gen: h.Gen, FinalTX: w.remoteMax(w.buckets[x.Region], dataKey(h.Location, h.Gen, "")), Flushed: true, Files: w.deadBoxFiles(x), Timers: append([]int(nil), x.Timers...), At: w.now} |
| 1889 | n.Op.Exec, n.Op.Phase = -1, "pinning" |
| 1890 | return |
| 1891 | } |
| 1892 | n.Op.PinFailover = n.Op.Phase == "starting" // 1b-next keeps it in starting until PinFailover (rev 6) |
| 1893 | n.Op.RelocReason, n.Op.Exec, n.Op.Phase = "failover", -1, "pinning" |
| 1894 | n.Op.Release, n.Op.StopFiles = nil, w.deadBoxFiles(x) |
| 1895 | w.stats["relocation: source lost before its release, converted to failover"]++ |
| 1896 | default: |
| 1897 | w.stats["relocation: source lost after its release"]++ |
| 1898 | } |
| 1899 | } |
| 1900 | |
| 1901 | // replaceTarget is §5.8's replacement table (ReplaceTarget) for a target |
| 1902 | // terminated by its fence, within the fence's CAS: |
| 1903 | // |
| 1904 | // starting, releasing, pinning -> the same phase, to' named |
| 1905 | // installing -> installing: to' is op.exec, a new stamp, a fresh P' |
| 1906 | // installed, or probing before P -> installed: to' is op.exec, a new stamp, the same P, no receipt |
| 1907 | // after P's first admission, or -> pinning: a nested failover from holder.generation as observed |
| 1908 | // probe-failed |
| 1909 | // |
| 1910 | // Exhaustion is checked first (Worker.replace, 1b-next R1-16): with the |
| 1911 | // replacements used the fence goes to fenceExhaustedTarget and no spare |
| 1912 | // is looked for. With no eligible spare it reports false and the fence |
| 1913 | // waits for a later pass (Worker.next). The mutation of |
| 1914 | // reloc-replacement-table resumes every late loss at installed with the |
| 1915 | // same P. |
| 1916 | func (w *world) replaceTarget(n *claim, x *box) bool { |
| 1917 | phase := n.Op.Phase |
| 1918 | if on("reloc-replacement-cap") && n.Op.Replacements >= maxReplacements { |
| 1919 | w.fenceExhaustedTarget(n, x) |
| 1920 | return true |
| 1921 | } |
| 1922 | next := w.chooseBoxExcept(n.Op.SrcRegion, n.Holder.App, relocExcept(n.Op, x.ID)...) |
| 1923 | if next == nil { |
| 1924 | if on("reloc-dead-target-bound") && x.FenceObservedAt > 0 && w.now > observedAt(x)+releaseDeadline { |
| 1925 | // FenceTargetWithoutSpare (1b-next rev 18 finding 2): the fence |
| 1926 | // is consumed ReleaseDeadline after its observed_at and the |
| 1927 | // operation is stuck; the mutation waits for a spare for ever |
| 1928 | w.stats["relocation: dead target with no spare fenced"]++ |
| 1929 | w.fenceTargetStuck(n, x, fmt.Sprintf("box %d terminated and no spare in %d steps", x.ID, releaseDeadline)) |
| 1930 | return true |
| 1931 | } |
| 1932 | w.stats["relocation: target lost, no spare yet"]++ |
| 1933 | return false |
| 1934 | } |
| 1935 | if !w.nameReplacement(n, next.ID, phase) { |
| 1936 | return n.Op.Stuck != "" |
| 1937 | } |
| 1938 | w.stats["relocation: target replaced at "+phase]++ |
| 1939 | late := preparedOpened(n) || phase == "probe-failed" || phase == "probed" || phase == "opened" |
| 1940 | switch { |
| 1941 | case phase == "starting" || phase == "releasing" || phase == "pinning": |
| 1942 | case phase == "installing": |
| 1943 | n.Op.BaseKey, n.Op.BaseVer, n.Op.BaseTX = "", 0, 0 |
| 1944 | w.handExec(n, next.ID, "installing", w.newID("P")) |
| 1945 | case !late || !on("reloc-replacement-table"): |
| 1946 | w.handExec(n, next.ID, "installed", n.Op.Prepared) // P was never opened: to' makes its own local install and opens it |
| 1947 | default: |
| 1948 | w.abandonPrepared(n) |
| 1949 | n.Op.Exec, n.Op.Phase = -1, "pinning" |
| 1950 | n.Op.Pin, n.Op.Prepared, n.Op.Receipt, n.Op.Release = nil, "", nil, nil |
| 1951 | n.Op.BaseKey, n.Op.BaseVer, n.Op.BaseTX = "", 0, 0 |
| 1952 | n.Op.ProbeTries, n.Op.Attempts = 0, 0 |
| 1953 | w.stats["nested failover after P was admitted"]++ |
| 1954 | } |
| 1955 | return true |
| 1956 | } |
| 1957 | |
| 1958 | // fenceExhaustedTarget is FenceExhaustedTarget (1b-next ruling 23 as rev |
| 1959 | // 9 amended it; rule reloc-exhausted-fence): a target terminated after |
| 1960 | // the operation used its replacements is fenced (the caller's fenceBox) |
| 1961 | // and the operation is stuck, with no third target chosen. While another |
| 1962 | // box is op.exec (the live source in starting or releasing) only stuck is |
| 1963 | // new: the target's fence is no evidence against the source, whose |
| 1964 | // execution identity, phase and release stay, so its own stop evidence |
| 1965 | // still decides. Otherwise (op.exec the dead target, or none) the dead |
| 1966 | // installation's authority is revoked: op.exec none at pinning, its |
| 1967 | // receipt and release dropped, and P's pin and remote build kept when P |
| 1968 | // was never opened, dropped for a nested pinning when it was. The |
| 1969 | // mutation is rev 8's: always op.exec none at pinning, which revokes a |
| 1970 | // live source without its stop evidence. |
| 1971 | func (w *world) fenceExhaustedTarget(n *claim, x *box) { |
| 1972 | w.stats["relocation: exhausted target fenced"]++ |
| 1973 | w.fenceTargetStuck(n, x, fmt.Sprintf("box %d terminated after %d target replacements", x.ID, maxReplacements)) |
| 1974 | } |
| 1975 | |
| 1976 | // fenceTargetStuck is 1b-next's fenceTargetStuck, the body |
| 1977 | // FenceExhaustedTarget and FenceTargetWithoutSpare share (rev 18): phase |
| 1978 | // sensitive as above; an unopened P is kept for the operator. |
| 1979 | func (w *world) fenceTargetStuck(n *claim, x *box, why string) { |
| 1980 | if on("reloc-exhausted-fence") && n.Op.Exec >= 0 && n.Op.Exec != x.ID { |
| 1981 | n.Op.Stuck = why |
| 1982 | return |
| 1983 | } |
| 1984 | opened := preparedOpened(n) |
| 1985 | n.Op.Exec, n.Op.Phase = -1, "pinning" |
| 1986 | n.Op.Receipt, n.Op.Release = nil, nil |
| 1987 | n.Op.ProbeTries, n.Op.Attempts = 0, 0 |
| 1988 | if opened { |
| 1989 | n.Op.Pin, n.Op.Prepared = nil, "" |
| 1990 | n.Op.BaseKey, n.Op.BaseVer, n.Op.BaseTX = "", 0, 0 |
| 1991 | } |
| 1992 | n.Op.Stuck = why |
| 1993 | } |
| 1994 | |
| 1995 | // ---------- the files publisher and the files pin (region HA §5.6, L13, L17) ---------- |
| 1996 | |
| 1997 | // manifestKey is a files manifest of generation gen within loc, by |
| 1998 | // sequence; isManifestKey recognizes one. genOf reads files/ under the |
| 1999 | // origin generation as "". |
| 2000 | func manifestKey(loc, gen string, seq int) string { |
| 2001 | return dataKey(loc, gen, fmt.Sprintf("files/m-%06d", seq)) |
| 2002 | } |
| 2003 | |
| 2004 | func isManifestKey(k string) bool { return strings.Contains(k, "files/m-") } |
| 2005 | |
| 2006 | // filesState is what a manifest carries, as the publisher compares it. |
| 2007 | func filesState(timers []int, f *filesDir) string { |
| 2008 | if f == nil { |
| 2009 | return fmt.Sprint(timers) |
| 2010 | } |
| 2011 | return fmt.Sprint(timers, f.Key, f.Env) |
| 2012 | } |
| 2013 | |
| 2014 | // publishFiles is the files publisher (§5.6, H6: platform-managed files |
| 2015 | // and wake state sync continuously on every plan): the executor |
| 2016 | // publishes a manifest, files and timers as they stand, whenever that |
| 2017 | // state changed since its last one, at the generation's next sequence |
| 2018 | // (the publisher of a generation starts at 1; a relocation's P has its |
| 2019 | // prepared manifest at 0). The PUT goes through issue, so it may complete |
| 2020 | // late, after the box is terminated: the late-file-upload case. A |
| 2021 | // relocation's target publishes its probe session's state under P |
| 2022 | // (B2-4). Only the two-box shape runs it (ruling 11). |
| 2023 | func (w *world) publishFiles(b *box) { |
| 2024 | c := w.claim |
| 2025 | if c == nil || c.Holder == nil || !w.agentUp(b) || b.Row == nil || b.Files == nil || b.Files.Inc != c.Holder.Inc { |
| 2026 | return |
| 2027 | } |
| 2028 | if c.Op.Exec != b.ID || !(c.Op.Kind == "serving" || c.Op.Kind == "relocating" && (b.ID == c.Op.Src || preparedOpened(c))) { |
| 2029 | return // a relocation's target publishes only once P is opened, into its own generations |
| 2030 | } |
| 2031 | h := c.Holder |
| 2032 | if b.pubGen != h.Gen { |
| 2033 | b.pubGen, b.pubSeq, b.releaseSeq = h.Gen, 0, 0 |
| 2034 | } |
| 2035 | state := filesState(b.Timers, b.Files) |
| 2036 | if state == b.published && w.objectCurrent(b, b.Files) { |
| 2037 | return |
| 2038 | } |
| 2039 | if !w.ensureObject(b, h, b.Files) { |
| 2040 | return // the object first: a manifest names only durable objects (§5.6) |
| 2041 | } |
| 2042 | b.pubSeq++ |
| 2043 | w.issued++ |
| 2044 | bl := &blob{Kind: "files", Inc: h.Inc, Gen: h.Gen, Seq: b.pubSeq, Files: b.Files.clone(), Timers: append([]int(nil), b.Timers...), Base: w.issued} |
| 2045 | w.issue(b.ID, &request{kind: rPut, region: b.Region, key: manifestKey(h.Location, h.Gen, b.pubSeq), b: bl}) |
| 2046 | b.published = state |
| 2047 | w.stats["files manifests published"]++ |
| 2048 | } |
| 2049 | |
| 2050 | // newestManifest is the files pin's choice (§5.8, B2-4; StorePinner): the |
| 2051 | // newest landed manifest in the admitted chain, holder.generation first, |
| 2052 | // then each generation it superseded (genParent: the files live set's |
| 2053 | // chain, which no trim of holder.admissions shortens; 1b-next rev 12 |
| 2054 | // finding 1), by sequence; not the database baseline's generation. |
| 2055 | // Nothing landed: "" and nil. |
| 2056 | func (w *world) newestManifest(r int, loc, gen string) (key string, ver, seq int, mgen string, m *blob) { |
| 2057 | bk := w.buckets[r] |
| 2058 | seen := map[string]bool{} |
| 2059 | for p := dataKey(loc, gen, ""); !seen[p]; { |
| 2060 | seen[p] = true |
| 2061 | var best *blob |
| 2062 | for _, k := range bk.keys(p + "files/") { |
| 2063 | v := bk.current(k) |
| 2064 | if v == nil || v.B == nil || v.B.Kind != "files" { |
| 2065 | continue |
| 2066 | } |
| 2067 | if best == nil || v.B.Seq > seq { |
| 2068 | key, ver, seq, mgen, best = k, v.ID, v.B.Seq, v.B.Gen, v.B |
| 2069 | } |
| 2070 | } |
| 2071 | if best != nil { |
| 2072 | return key, ver, seq, mgen, best |
| 2073 | } |
| 2074 | next, ok := w.genParent[p] |
| 2075 | if !ok { |
| 2076 | break |
| 2077 | } |
| 2078 | p = next |
| 2079 | } |
| 2080 | return "", 0, 0, "", nil |
| 2081 | } |
| 2082 | |
| 2083 | // lastLandedManifest is INV-28's ground truth: of the first generation in |
| 2084 | // the chain that holds a landed manifest, the one published last (the |
| 2085 | // issue order the model records in the blob), whatever its sequence. |
| 2086 | func (w *world) lastLandedManifest(r int, loc, gen string) *blob { |
| 2087 | _, _, _, mg, m := w.newestManifest(r, loc, gen) |
| 2088 | if m == nil { |
| 2089 | return nil |
| 2090 | } |
| 2091 | bk := w.buckets[r] |
| 2092 | last := m |
| 2093 | for _, k := range bk.keys(dataKey(loc, mg, "files/")) { |
| 2094 | if v := bk.current(k); v != nil && v.B != nil && v.B.Kind == "files" && v.B.Gen == mg && v.B.Base > last.Base { |
| 2095 | last = v.B |
| 2096 | } |
| 2097 | } |
| 2098 | return last |
| 2099 | } |
| 2100 | |
| 2101 | // filesLive is the files live set (L13; claim §6.2a "Files in the live |
| 2102 | // set"): a manifest of holder.generation, of a relocation's prepared |
| 2103 | // generation, the one a running relocation's pin names, and the newest |
| 2104 | // landed manifest of the admitted chain (the live manifest, which an |
| 2105 | // unchanged file keeps referencing generations after it was published) |
| 2106 | // is never archived by a park-generation or read as residue. It is |
| 2107 | // region-qualified (rule files-live-region; joint round 1 finding 1): k |
| 2108 | // is the key in region r's bucket. The mutation of files-live-set |
| 2109 | // protects nothing, so a park-generation of an older generation archives |
| 2110 | // the live or the pinned manifest. |
| 2111 | func (w *world) filesLive(c *claim, r int, k string) bool { |
| 2112 | return on("files-live-set") && w.filesLiveIn(c, r, k, on("files-live-region") && on("live-set-region")) |
| 2113 | } |
| 2114 | |
| 2115 | // filesLiveSet is the files live set itself, region-qualified, which the |
| 2116 | // residue oracle reads whatever is mutated. |
| 2117 | func (w *world) filesLiveSet(c *claim, r int, k string) bool { |
| 2118 | return w.filesLiveIn(c, r, k, true) |
| 2119 | } |
| 2120 | |
| 2121 | // filesHome: does region r hold the files live set? Only the |
| 2122 | // authoritative region does (claim rev 6.5 §6.2a, R4-3): the region of |
| 2123 | // the holder's current installation, as the moves spec's §5.1 table |
| 2124 | // ("The authoritative region", the claim state by claim state table) |
| 2125 | // names it, united with op.exec's region, whose box runs moves' own parks |
| 2126 | // (F4's keepFilesLive): |
| 2127 | // |
| 2128 | // - serving, pushing, reverting: op.exec's region; |
| 2129 | // - relocating: its source region, for the whole operation (a |
| 2130 | // relocation never leaves it; while op.exec is none, at a failover's |
| 2131 | // pinning or a replay, the chain's newest manifest there is the pin's |
| 2132 | // candidate: two-box seed 1215, stuck for ever under a narrower |
| 2133 | // reading, INV-3); |
| 2134 | // - moving (before the flip, every phase, op.exec none from stopped on): |
| 2135 | // the source's region; |
| 2136 | // - landing: the target's region, op.exec's, throughout; and the |
| 2137 | // source's region until the claim follows the move's commit |
| 2138 | // (op.Committed, followRecord's "committed" case), not only until the |
| 2139 | // target's probe is admitted as §5.1's row for holder.generation |
| 2140 | // reads: the probe's admission commits nothing, and until the commit |
| 2141 | // a clean rollback names op.SrcGen again in the source region |
| 2142 | // (moves_test.go's return CAS, Holder.Gen = op.SrcGen), whose next |
| 2143 | // failover pins the source's chain there (Astra joint round 2). The |
| 2144 | // record's commit (landTarget's MoveState = "committed", the CAS on |
| 2145 | // the field rollback also takes) rules the rollback out; the claim |
| 2146 | // follows it in the CAS that queues the source region's |
| 2147 | // whole-location park-superseded, so the source stays home until |
| 2148 | // that park owns it; |
| 2149 | // - restoring: the source's region while the target releases |
| 2150 | // (releasing, released, the dirty lane's returned: SrcPhase waiting, |
| 2151 | // op.exec the target or none), until the source's return CAS (moves |
| 2152 | // a5e8c13f, L18 finding 1). After a clean return the source is op.exec |
| 2153 | // and names op.SrcGen again. After a dirty return the newest admission |
| 2154 | // (the target's) decides until the source's probe is admitted into the |
| 2155 | // return generation: the source region is not the files home then, |
| 2156 | // even as op.exec's, so the return's install park (moves' D6, |
| 2157 | // install-park) retires the pre-move generations with their files |
| 2158 | // (dirtyReturned). The target region is retired by the release's |
| 2159 | // whole-location park, which never consults the files live set; |
| 2160 | // - creating, deleting, parking, purging, free: none (op.exec is none |
| 2161 | // in each, and no case below names them). |
| 2162 | // |
| 2163 | // The mutation of files-home-move-phases keeps op.exec's region and a |
| 2164 | // relocation's source only, so from a move's stopped phase to the |
| 2165 | // commit, and in a rollback's released phase, a park-generation queued |
| 2166 | // during the move archives the live manifest in the source region. |
| 2167 | func (w *world) filesHome(c *claim, r int) bool { |
| 2168 | if dirtyReturned(c, on("dirty-return-needs-admission")) && r == c.Op.SrcRegion && on("files-home-move-phases") { |
| 2169 | return false // D6: the dirty return's install parks the pre-move generations |
| 2170 | } |
| 2171 | if e := w.execBox(c); e != nil && e.Region == r { |
| 2172 | return true |
| 2173 | } |
| 2174 | if c.Op.Kind == "relocating" && r == c.Op.SrcRegion { |
| 2175 | return true |
| 2176 | } |
| 2177 | if !on("files-home-move-phases") || r != c.Op.SrcRegion { |
| 2178 | return false |
| 2179 | } |
| 2180 | switch c.Op.Kind { |
| 2181 | case "moving", "restoring": |
| 2182 | return true |
| 2183 | case "landing": |
| 2184 | return !c.Op.Committed |
| 2185 | } |
| 2186 | return false |
| 2187 | } |
| 2188 | |
| 2189 | // pinWatch is what INV-30 remembers of one region: the manifest version |
| 2190 | // its next install would pin, and the object that manifest names when |
| 2191 | // the object was current in the region's bucket. |
| 2192 | type pinWatch struct { |
| 2193 | inc int |
| 2194 | loc string |
| 2195 | key string |
| 2196 | ver, base int |
| 2197 | obj string |
| 2198 | } |
| 2199 | |
| 2200 | // servableRegions are the regions where the host could still be served |
| 2201 | // next (INV-30's own reading of claim §6.2a and moves §5.1, written |
| 2202 | // apart from filesHome so that it can catch filesHome): op.exec's |
| 2203 | // region; a relocation's source region (it never leaves it); a move's |
| 2204 | // source region before the flip; a landing's source region until the |
| 2205 | // record commits (landTarget's MoveState = "committed", the CAS on the |
| 2206 | // field a rollback also takes: until then a clean rollback names |
| 2207 | // op.SrcGen again there); a rollback's source region. Creating, |
| 2208 | // deleting, parking and purging have no executor and name none. |
| 2209 | func (w *world) servableRegions(c *claim) []int { |
| 2210 | if c == nil || c.Holder == nil || dirtyReturned(c, true) { |
| 2211 | // after a dirty return of a move its target admitted, the source's |
| 2212 | // next install comes from the return lane, never from a pinned |
| 2213 | // manifest of the pre-move generations, which D6 retires by design |
| 2214 | // (moves a5e8c13f). The oracle reads the claim's admissions itself, |
| 2215 | // whatever the rules say: a dirty return the target never admitted |
| 2216 | // keeps the pre-move generation as the source's live state |
| 2217 | return nil |
| 2218 | } |
| 2219 | var out []int |
| 2220 | if e := w.execBox(c); e != nil { |
| 2221 | out = append(out, e.Region) |
| 2222 | } |
| 2223 | src := false |
| 2224 | switch c.Op.Kind { |
| 2225 | case "relocating", "moving", "restoring": |
| 2226 | src = true |
| 2227 | case "landing": |
| 2228 | src = !c.Op.Committed && !(w.rec != nil && w.rec.MoveID == c.Op.ID && w.rec.MoveState == "committed") |
| 2229 | } |
| 2230 | if src && (len(out) == 0 || out[0] != c.Op.SrcRegion) { |
| 2231 | out = append(out, c.Op.SrcRegion) |
| 2232 | } |
| 2233 | return out |
| 2234 | } |
| 2235 | |
| 2236 | // checkFilesPreserved is INV-30 (joint round 2; claim §6.2a "Files in the |
| 2237 | // live set", moves §5.1): in every region where the host could still be |
| 2238 | // served next, the manifest that region's next install would pin (the |
| 2239 | // newest landed manifest of the admitted chain there, newestManifest, |
| 2240 | // which the failover's pin takes) and the object it names survive: once |
| 2241 | // seen, the version is never archived or deleted while the region can |
| 2242 | // still serve, unless a newer manifest has landed in its place. It reads |
| 2243 | // the bucket each step and remembers the last step's choice, so an |
| 2244 | // archive is seen as it happens, before any later read of the bucket |
| 2245 | // takes the shortened chain for the truth (the fence's timerCarried and |
| 2246 | // INV-28 read the bucket after the archive; a failover's stuck pin is |
| 2247 | // INV-3 only much later, and only if a failover happens). |
| 2248 | func (w *world) checkFilesPreserved() { |
| 2249 | c := w.claim |
| 2250 | next := map[int]pinWatch{} |
| 2251 | for _, r := range w.servableRegions(c) { |
| 2252 | loc := c.Holder.Location |
| 2253 | bk := w.buckets[r] |
| 2254 | mk, mv, _, _, m := w.newestManifest(r, loc, c.Holder.Gen) |
| 2255 | if p, ok := w.pinWatch[r]; ok && p.inc == c.Holder.Inc && p.loc == loc { |
| 2256 | gone := bk.current(p.key) == nil || bk.byID(p.key, p.ver) == nil |
| 2257 | superseded := m != nil && m.Base > p.base |
| 2258 | if gone && !superseded { |
| 2259 | w.violate("INV-30: region %d could still serve inc %d (%s %s), and the manifest its next install would pin, %s@%d, is gone from its bucket (now %q)", r, c.Holder.Inc, c.Op.Kind, c.Op.ID, p.key, p.ver, mk) |
| 2260 | return |
| 2261 | } |
| 2262 | if !gone && mk == p.key && p.obj != "" && bk.current(p.obj) == nil { |
| 2263 | w.violate("INV-30: region %d could still serve inc %d (%s %s), and the object %s its pinned manifest %s names is gone from its bucket", r, c.Holder.Inc, c.Op.Kind, c.Op.ID, p.obj, p.key) |
| 2264 | return |
| 2265 | } |
| 2266 | } |
| 2267 | if m == nil { |
| 2268 | continue |
| 2269 | } |
| 2270 | pw := pinWatch{inc: c.Holder.Inc, loc: loc, key: mk, ver: mv, base: m.Base} |
| 2271 | if m.Files != nil && m.Files.Obj != "" && bk.current(m.Files.Obj) != nil { |
| 2272 | pw.obj = m.Files.Obj |
| 2273 | } |
| 2274 | next[r] = pw |
| 2275 | } |
| 2276 | w.pinWatch = next |
| 2277 | } |
| 2278 | |
| 2279 | // filesLiveIn is the files live set of region r. regional false is the |
| 2280 | // region-blind set the rule files-live-region replaced (and every region |
| 2281 | // authoritative, as live-set-region's mutation reads the whole live set): |
| 2282 | // the newest manifest of the chain in any region is live in every region, |
| 2283 | // so a manifest PUT that lands in a former region after a move's cleanup |
| 2284 | // protects itself there for ever. |
| 2285 | func (w *world) filesLiveIn(c *claim, r int, k string, regional bool) bool { |
| 2286 | if c == nil || c.Holder == nil || !isFilesKey(k) { |
| 2287 | return false |
| 2288 | } |
| 2289 | if !isManifestKey(k) { |
| 2290 | return w.objectReferenced(c, r, k, regional) |
| 2291 | } |
| 2292 | loc := c.Holder.Location |
| 2293 | if !strings.HasPrefix(k, loc) { |
| 2294 | return false |
| 2295 | } |
| 2296 | if regional && !w.filesHome(c, r) { |
| 2297 | return false |
| 2298 | } |
| 2299 | g := genOf(loc, k) |
| 2300 | if g == c.Holder.Gen { |
| 2301 | return true |
| 2302 | } |
| 2303 | if c.Op.Kind == "relocating" { |
| 2304 | if c.Op.Pin != nil && k == c.Op.Pin.ManKey { |
| 2305 | return true |
| 2306 | } |
| 2307 | if c.Op.Prepared != "" && g == c.Op.Prepared { |
| 2308 | return true |
| 2309 | } |
| 2310 | } |
| 2311 | for _, x := range w.filesRegions(r, regional) { |
| 2312 | if mk, _, _, _, _ := w.newestManifest(x, loc, c.Holder.Gen); mk == k { |
| 2313 | return true |
| 2314 | } |
| 2315 | } |
| 2316 | return false |
| 2317 | } |
| 2318 | |
| 2319 | // filesRegions is where the live set's newest manifest is read for a key |
| 2320 | // of region r: region r alone; the region-blind set reads every region. |
| 2321 | func (w *world) filesRegions(r int, regional bool) []int { |
| 2322 | if regional { |
| 2323 | return []int{r} |
| 2324 | } |
| 2325 | all := make([]int, nRegions) |
| 2326 | for i := range all { |
| 2327 | all[i] = i |
| 2328 | } |
| 2329 | return all |
| 2330 | } |
| 2331 | |
| 2332 | // pinnedManifest is the manifest the local install applies: the exact |
| 2333 | // version the pin names (rule files-pin-exact; the model's versioned |
| 2334 | // bucket reads it by version id, where the real store, which has none, |
| 2335 | // reads the pinner's retained copy, 1b-next ruling 21); the mutation |
| 2336 | // reads the newest manifest at install time, which a late PUT from the |
| 2337 | // dead box may have changed since the pin. |
| 2338 | func (w *world) pinnedManifest(c *claim) *blob { |
| 2339 | p := c.Op.Pin |
| 2340 | if p == nil || p.ManKey == "" || c.Holder == nil { |
| 2341 | return nil |
| 2342 | } |
| 2343 | if !on("files-pin-exact") { |
| 2344 | _, _, _, _, m := w.newestManifest(c.Op.SrcRegion, c.Holder.Location, c.Holder.Gen) |
| 2345 | return m |
| 2346 | } |
| 2347 | if v := w.buckets[c.Op.SrcRegion].byID(p.ManKey, p.ManVer); v != nil && !v.Marker { |
| 2348 | return v.B |
| 2349 | } |
| 2350 | return nil |
| 2351 | } |
| 2352 | |
| 2353 | // regionMin is the region's minimum box count (H5): the shape's, or one |
| 2354 | // for a witness's one-box region. |
| 2355 | func (w *world) regionMin(r int) int { |
| 2356 | if w.oneBoxRegion[r] { |
| 2357 | return 1 |
| 2358 | } |
| 2359 | return w.perRegion |
| 2360 | } |
| 2361 | |
| 2362 | // ---------- acknowledgements (rev 4, Astra R1-4) ---------- |
| 2363 | |
| 2364 | // acked is requireAcks: every current participant has a durable |
| 2365 | // exec_ack.<box> entry in the journal. It reads only the claim; whether a |
| 2366 | // participant's controller is ready now is the participant's own check |
| 2367 | // before its effects (recordAck, and role()'s lease), never the barrier. |
| 2368 | func acked(c *claim) bool { |
| 2369 | for _, p := range c.Op.Parts { |
| 2370 | if !contains(c.Op.Acks, p) { |
| 2371 | return false |
| 2372 | } |
| 2373 | } |
| 2374 | return true |
| 2375 | } |
| 2376 | |
| 2377 | // recordAck is AcknowledgeExecution: a participant whose controller is |
| 2378 | // ready (its execution lease current, role() renews it first) journals |
| 2379 | // exec_ack.<box> once; the entry outlives a lost lease or an agent |
| 2380 | // restart (1b-next rev 2 finding 10: the claim keeps the first |
| 2381 | // acknowledgement). A box replaced out of the participants keeps its |
| 2382 | // entry, which the barrier no longer reads. |
| 2383 | func (w *world) recordAck(b *box, c *claim) { |
| 2384 | if c == nil || c.Op.Kind != "relocating" || !contains(c.Op.Parts, b.ID) || contains(c.Op.Acks, b.ID) || contains(c.Lost, b.ID) { |
| 2385 | return |
| 2386 | } |
| 2387 | if b.ExecUntil == 0 || w.localNow(b) > b.ExecUntil { |
| 2388 | return // no ready controller: nothing to acknowledge with |
| 2389 | } |
| 2390 | id := c.Op.ID |
| 2391 | // the controller's durable record exists before the acknowledgement |
| 2392 | // (1b-next Task 7: journaled before the open), and stays until a |
| 2393 | // successful scoped cleanup clears it (finishParticipation) |
| 2394 | b.execRecord, b.cleanupPending = id, true |
| 2395 | if w.cas(c, func(n *claim) bool { |
| 2396 | ok := sameOp(n.Op.ID, id) && n.Op.Kind == "relocating" && contains(n.Op.Parts, b.ID) && !contains(n.Op.Acks, b.ID) |
| 2397 | if ok { |
| 2398 | n.Op.Acks = append(n.Op.Acks, b.ID) |
| 2399 | } |
| 2400 | return ok |
| 2401 | }) { |
| 2402 | w.logf("relocation %s: box %d acknowledged execution", id, b.ID) |
| 2403 | } |
| 2404 | } |
| 2405 | |
| 2406 | // finishParticipation is the participant's Sweep and Finish (1b-next |
| 2407 | // Task 6 and 7, rulings 20): once the claim no longer runs the operation |
| 2408 | // with this box as a live participant, the scoped cleanup runs (it |
| 2409 | // retires an unopened P's local install of that operation, and is a |
| 2410 | // no-op on a box that admitted a writer or serves the host); only when it |
| 2411 | // succeeds is the durable record cleared (rule reloc-clear-after-cleanup; |
| 2412 | // the mutation clears it after a failed cleanup, as rev 4 of 1b-next |
| 2413 | // did). A box whose cleanup fails keeps the record, and root refuses an |
| 2414 | // executable update while it exists (claim §5.10). |
| 2415 | func (w *world) finishParticipation(b *box, c *claim) { |
| 2416 | if b.execRecord == "" || c == nil { |
| 2417 | return |
| 2418 | } |
| 2419 | if d := c.Op.Completed; c.Op.Kind == "serving" && d != nil && !d.Discharged && d.Box == b.ID && b.execRecord == d.Op { |
| 2420 | return // the completion box's participation is C10's to discharge (completionPass), first |
| 2421 | } |
| 2422 | if c.Op.Kind == "relocating" && c.Op.ID == b.execRecord && contains(c.Op.Parts, b.ID) && !contains(c.Lost, b.ID) { |
| 2423 | return // still a participant |
| 2424 | } |
| 2425 | if b.cleanupFails { |
| 2426 | w.stats["relocation controller: scoped cleanup failed"]++ |
| 2427 | if on("reloc-clear-after-cleanup") { |
| 2428 | return // the record stays, and so does the refusal of updates |
| 2429 | } |
| 2430 | b.execRecord = "" // the mutation: cleared anyway |
| 2431 | return |
| 2432 | } |
| 2433 | if f := b.Files; f != nil && f.Gen != "" && c.Holder != nil && f.Inc == c.Holder.Inc && strings.HasPrefix(f.Gen, "P") && !(c.Op.Kind == "serving" && c.Op.Exec == b.ID) && f.Gen != c.Holder.Gen && !w.ancestorGen(c.Holder.Location, c.Holder.Gen, f.Gen) { |
| 2434 | w.retireArtifacts(b, f.Inc, f.Stamp) // an unopened P's local install of the finished operation |
| 2435 | } |
| 2436 | b.execRecord, b.cleanupPending = "", false |
| 2437 | w.stats["relocation controller: record cleared"]++ |
| 2438 | } |
| 2439 | |
| 2440 | // updateExecutable is an executable update on b (claim §5.10, region HA |
| 2441 | // §5.10): root refuses it while a relocation controller's record exists. |
| 2442 | // INV-29 is the oracle: an update never runs on a box whose relocation |
| 2443 | // participation's scoped cleanup has not succeeded. |
| 2444 | func (w *world) updateExecutable(b *box) bool { |
| 2445 | if b.Destroyed { |
| 2446 | return false |
| 2447 | } |
| 2448 | if b.execRecord != "" { |
| 2449 | w.stats["executable update refused: controller record"]++ |
| 2450 | return false |
| 2451 | } |
| 2452 | if b.cleanupPending { |
| 2453 | w.violate("INV-29: box %d updated its executable while a relocation's scoped cleanup had not succeeded (claim §5.10)", b.ID) |
| 2454 | } |
| 2455 | w.stats["executable updated"]++ |
| 2456 | return true |
| 2457 | } |
| 2458 | |
| 2459 | // latestObservation is the newest observed_at the operation's journal |
| 2460 | // retains (JournalPin checks every fence.<box>.observed_at, a release or |
| 2461 | // not: rev 4, Astra R1-5); FenceAt covers a converted move's fence. |
| 2462 | func latestObservation(op opState) int { |
| 2463 | at := op.FenceAt |
| 2464 | for _, o := range op.Observed { |
| 2465 | if o > at { |
| 2466 | at = o |
| 2467 | } |
| 2468 | } |
| 2469 | return at |
| 2470 | } |
| 2471 | |
| 2472 | // observedAt is when the fence record says the box was observed |
| 2473 | // terminated (FenceTargetWithoutSpare's deadline basis); the mutation of |
| 2474 | // reloc-deadline-from-observed-at uses the termination itself, which no |
| 2475 | // worker may have seen yet. |
| 2476 | func observedAt(x *box) int { |
| 2477 | if !on("reloc-deadline-from-observed-at") { |
| 2478 | return x.TerminatedAt |
| 2479 | } |
| 2480 | return x.FenceObservedAt |
| 2481 | } |
| 2482 | |
| 2483 | // objectReferenced: a live manifest (the chain's newest, the pinned one, |
| 2484 | // holder.generation's or P's) names the object, or a live box's files |
| 2485 | // directory does (its next manifest will): the files live set is every |
| 2486 | // object the live manifests reference (claim §6.2a). Region-qualified as |
| 2487 | // the manifests are: k is the key in region r's bucket, region r must be |
| 2488 | // the files home (filesHome), only region r's manifests are read, and a |
| 2489 | // box's directory names k only when it names region r's object. The |
| 2490 | // region-blind set (regional false) reads every region's manifests and |
| 2491 | // every box's directory, for a key of any region. |
| 2492 | func (w *world) objectReferenced(c *claim, r int, k string, regional bool) bool { |
| 2493 | loc := c.Holder.Location |
| 2494 | names := func(m *blob) bool { return m != nil && m.Files != nil && m.Files.Obj == k } |
| 2495 | if regional && !w.filesHome(c, r) { |
| 2496 | return false |
| 2497 | } |
| 2498 | for _, x := range w.filesRegions(r, regional) { |
| 2499 | if _, _, _, _, m := w.newestManifest(x, loc, c.Holder.Gen); names(m) { |
| 2500 | return true |
| 2501 | } |
| 2502 | bk := w.buckets[x] |
| 2503 | for _, g := range []string{c.Holder.Gen, c.Op.Prepared} { |
| 2504 | if g == "" && g != c.Holder.Gen { |
| 2505 | continue |
| 2506 | } |
| 2507 | for _, mk := range bk.keys(dataKey(loc, g, "files/m-")) { |
| 2508 | if v := bk.current(mk); v != nil && names(v.B) { |
| 2509 | return true |
| 2510 | } |
| 2511 | } |
| 2512 | } |
| 2513 | } |
| 2514 | if c.Op.Kind == "relocating" && c.Op.Pin != nil && c.Op.Pin.ManKey != "" && (!regional || r == c.Op.SrcRegion) { |
| 2515 | if names(w.pinnedManifestOf(c.Op.SrcRegion, c.Op.Pin)) { |
| 2516 | return true |
| 2517 | } |
| 2518 | } |
| 2519 | for _, b := range w.liveBoxes() { |
| 2520 | if b.Files != nil && b.Files.Obj == k && (!regional || b.Files.ObjRegion == r) { |
| 2521 | return true |
| 2522 | } |
| 2523 | } |
| 2524 | return false |
| 2525 | } |
| 2526 | |
| 2527 | // keepFilesLive wraps a moves park's keep (a push's whole-location park, |
| 2528 | // a landing's install park, a rollback's return park) so it never |
| 2529 | // archives the files live set (finding F4, rev 4: claim §6.2a's live set |
| 2530 | // binds every park; moves' parks predate region HA's files tree). Only |
| 2531 | // the two-box shape has files keys, so the one-box shape is unchanged. |
| 2532 | // The mutation of files-live-set-moves-parks archives them with the rest. |
| 2533 | func (w *world) keepFilesLive(r int, keep func(string) bool) func(string) bool { |
| 2534 | return func(k string) bool { |
| 2535 | if keep(k) { |
| 2536 | return true |
| 2537 | } |
| 2538 | if on("files-live-set-moves-parks") && w.filesLive(w.claim, r, k) { |
| 2539 | w.markSweepOwed(r, k) |
| 2540 | return true |
| 2541 | } |
| 2542 | return false |
| 2543 | } |
| 2544 | } |
| 2545 | |
| 2546 | // markSweepOwed: a park kept k only because the files live set named it; |
| 2547 | // once obsolete it is the at-rest sweep's (claim §6.2a, 1b-next ruling 21), |
| 2548 | // so INV-10 at an operation's end exempts it as it exempts a straggler. |
| 2549 | func (w *world) markSweepOwed(r int, k string) { |
| 2550 | for i := range w.buckets { |
| 2551 | if r >= 0 && i != r { |
| 2552 | continue |
| 2553 | } |
| 2554 | for _, v := range w.buckets[i].objs[k] { |
| 2555 | v.SweepOwed = true |
| 2556 | } |
| 2557 | } |
| 2558 | } |
| 2559 | |
| 2560 | // renewController is a relocation controller's own renewal loop (hostexec |
| 2561 | // renews its lease on its own clock, not when the regional worker happens |
| 2562 | // to run; rev 4): an agent that is up and holds a current lease for a |
| 2563 | // relocation it participates in keeps it current. A lease is still lost |
| 2564 | // when the agent is down, paused, frozen or partitioned past the TTL, |
| 2565 | // which is the scoped cleanup's case. Two-box shape only. |
| 2566 | func (w *world) renewController(b *box) { |
| 2567 | c := w.claim |
| 2568 | if c == nil || c.Op.Kind != "relocating" || !contains(c.Op.Parts, b.ID) || !w.agentUp(b) || b.Partitioned { |
| 2569 | return |
| 2570 | } |
| 2571 | if b.ExecUntil > 0 && w.localNow(b) <= b.ExecUntil && b.ExecUntil-w.localNow(b) < execRenew { |
| 2572 | b.ExecUntil = w.localNow(b) + execTTL |
| 2573 | w.execLease[b.ID] = execLease{Op: c.Op.ID, Expires: w.now + execTTL} |
| 2574 | } |
| 2575 | } |
| 2576 | |
| 2577 | // preparedManifestHere is moves rev 7.22 §7.6's one install routine (L16, |
| 2578 | // L17; claim R18-1) for a landing or a return in the two-box shape (rev |
| 2579 | // 4, claim §11 "move landing, source cleanup, then target failure with an |
| 2580 | // unchanged file"): the generation's prepared files manifest, create-only |
| 2581 | // at sequence 0 in this region, naming objects in this region's bucket |
| 2582 | // (an unchanged file the host carried from another region is uploaded |
| 2583 | // here first), acknowledged before the install's CAS. Without it a |
| 2584 | // target lost right after its commit would leave no files in its region. |
| 2585 | func (w *world) preparedManifestHere(b *box, c *claim, gen string) bool { |
| 2586 | h := c.Holder |
| 2587 | if !w.ensureObject(b, h, b.Files) { |
| 2588 | return false |
| 2589 | } |
| 2590 | bl := &blob{Kind: "files", Inc: h.Inc, Gen: gen, Seq: 0, Files: b.Files.clone(), Timers: append([]int(nil), b.Timers...), Text: c.Op.ID, Base: w.issued} |
| 2591 | return w.ackedPut(b, manifestKey(h.Location, gen, 0), bl, true) |
| 2592 | } |
| 2593 | |
| 2594 | // relocJournalRelease is the participant's release (1b-next Task 7): the |
| 2595 | // live executor runs its release sequence and journals it |
| 2596 | // (JournalRelease), the source in releasing, a failed target in |
| 2597 | // probe-failed. relocReleasing and relocProbeFailed call it for their |
| 2598 | // release branch, and relocate calls it by itself while a fence is |
| 2599 | // pending (rev 8; pending marks the trace line). The CAS is the union of |
| 2600 | // the two phases' conditions: in releasing no release is journaled yet; |
| 2601 | // in probe-failed the journaled release, if any, is another box's. |
| 2602 | func (w *world) relocJournalRelease(b *box, c *claim, pending bool) { |
| 2603 | op := c.Op |
| 2604 | phase := op.Phase |
| 2605 | if phase == "probe-failed" && w.c1Fallback() { |
| 2606 | // QueueRelocationRelease needs the source in releasing: with claim |
| 2607 | // ask C1 open there is no failed-target obligation, so nothing is |
| 2608 | // queued, stopped or journaled, and the release deadline makes the |
| 2609 | // operation stuck (finding 4) |
| 2610 | w.stats["release refused: no failed-target obligation (C1 open)"]++ |
| 2611 | return |
| 2612 | } |
| 2613 | r := w.relocRelease(b, c) |
| 2614 | if r == nil { |
| 2615 | return |
| 2616 | } |
| 2617 | if w.cas(c, func(n *claim) bool { |
| 2618 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == phase && n.Op.Exec == b.ID && (n.Op.Release == nil || phase == "probe-failed" && n.Op.Release.Box != b.ID) |
| 2619 | if ok { |
| 2620 | n.Op.Release = r |
| 2621 | } |
| 2622 | return ok |
| 2623 | }) { |
| 2624 | if phase == "releasing" { |
| 2625 | w.stats["release journaled: source"]++ |
| 2626 | } else { |
| 2627 | w.stats["release journaled: failed target"]++ |
| 2628 | } |
| 2629 | note := "" |
| 2630 | if pending { |
| 2631 | note = ", a fence pending" |
| 2632 | } |
| 2633 | w.logf("relocation %s: box %d released the host at TXID %d (flushed=%v)%s", op.ID, b.ID, r.FinalTX, r.Flushed, note) |
| 2634 | } |
| 2635 | } |
| 2636 | |
| 2637 | // dirtyReturned: a rollback whose source has taken the host back dirty |
| 2638 | // (its return CAS written: SrcPhase past waiting) and whose probe has not |
| 2639 | // yet been admitted into the return generation. From that CAS the newest |
| 2640 | // admission decides (moves' movesAuthoritative, a5e8c13f): the target's, |
| 2641 | // so the source region is not authoritative, and the return's install |
| 2642 | // park retires the pre-move generations (D6). |
| 2643 | func dirtyReturned(c *claim, needAdmission bool) bool { |
| 2644 | if c == nil || c.Holder == nil || c.Op.Kind != "restoring" || !c.Op.Dirty || c.Op.SrcPhase == "waiting" || c.Holder.Gen == c.Op.RollbackID { |
| 2645 | return false |
| 2646 | } |
| 2647 | return !needAdmission || targetAdmitted(c) |
| 2648 | } |
| 2649 | |
| 2650 | // targetAdmitted is movesinstall.go's targetAdmitted (Astra round 4, item |
| 2651 | // 2): the claim's recorded admissions hold one of this move's generation |
| 2652 | // (the landing's, installGen: the move's op id). A dirty return can come |
| 2653 | // without one: src_covered false makes the return dirty even when the |
| 2654 | // target never admitted (releaseTarget's seal-return), and then |
| 2655 | // holder.generation is still the pre-move one, the source's live state, |
| 2656 | // which the return's install keeps and parks nothing of. Decided from the |
| 2657 | // claim's admissions only, never from the landing phase. |
| 2658 | func targetAdmitted(c *claim) bool { |
| 2659 | if c == nil || c.Holder == nil { |
| 2660 | return false |
| 2661 | } |
| 2662 | for _, a := range c.Holder.Admissions { |
| 2663 | if a.Gen == c.Op.ID { |
| 2664 | return true |
| 2665 | } |
| 2666 | } |
| 2667 | return false |
| 2668 | } |
| 2669 | |
amadan