| 1 | package hostlife |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "strings" |
| 6 | ) |
| 7 | |
| 8 | // ---------- the built completion protocol (L18 joint round, findings 2 to 4) ---------- |
| 9 | // |
| 10 | // Finding 2: a relocation does not end with the probe still running. The |
| 11 | // built sequence (1b-next 596fa8b1; hostrelocate Participant and Worker, |
| 12 | // host relocationStopProbe and relocationReleaseHold, hostclaim |
| 13 | // CompleteRelocation, completionBarrier) is: |
| 14 | // |
| 15 | // 1. probe stop (ruling 49): with the probe journaled (the model's |
| 16 | // probed), the target persists the owner of the fence it will install |
| 17 | // (probe-fence.json), installs that owned activator fence on an |
| 18 | // unfenced host or meets its own (ruling 75; any other owner's fence |
| 19 | // refuses with nothing stopped), stops the session's whole tree with |
| 20 | // root's receipt and journals the stop (RecordProbeStop). Ruling 67 |
| 21 | // bounds it: no stop ReleaseDeadline after the probe passed is stuck; |
| 22 | // 2. the worker's serving CAS (CompleteRelocation) requires that stop and |
| 23 | // records completed_relocation on the serving op; |
| 24 | // 3. the completion pass on the box it completed on (relocationReleaseHold, |
| 25 | // rulings 66, 76, 80): C10's discharge of the participation, then the |
| 26 | // row's hold released, then the owned fence lifted, then the mark |
| 27 | // (MarkCompletedDischarged) last; found from the claim record, never |
| 28 | // from the row, so a failure at any step is retried; run whenever claim |
| 29 | // mode is on, region HA on or off (ruling 82); |
| 30 | // 4. the first serving wake after it is a separate admission that hands |
| 31 | // over from the probe's receipt (N6), never main's stop; |
| 32 | // 5. until the mark, every start from serving refuses (ruling 77), except |
| 33 | // with the completion box on lost or cited as a failover's fenced |
| 34 | // source. |
| 35 | // |
| 36 | // Finding 3 (ruling 85, 1b-next a092ddc2): the source's row hold is |
| 37 | // reconcile's, not the release's (ruling 28 fences admission at the |
| 38 | // release's end), so a source whose reconcile runs only after the |
| 39 | // completion catches up: serving on another box, a completed relocation |
| 40 | // that did not complete here, and a park-local this box owes for exactly |
| 41 | // that op hold the row until the park-local retires it. |
| 42 | // |
| 43 | // Finding 4: today's fallbacks for claim asks C1 and C11. A failed target |
| 44 | // cannot release in probe-failed (QueueRelocationRelease needs the |
| 45 | // source's releasing; no failed-target obligation), so it reaches stuck at |
| 46 | // the release deadline; and after P's opening a later cold probe attempt |
| 47 | // is refused before it is counted (root pins every relocating launch to |
| 48 | // install_stamp), which is probe-failed. world.claimC1 and claimC11 are |
| 49 | // the switches that flip them when the asks land. |
| 50 | |
| 51 | // probeOwner is the owner a probe stop persists and installs its fence |
| 52 | // under: the operation and the probe's session. |
| 53 | func probeOwner(op string, session int) string { return fmt.Sprintf("probe:%s/%d", op, session) } |
| 54 | |
| 55 | // probeKey is the stop evidence key of op's probe stop: root's completed |
| 56 | // writer-session receipt in the box's guard journal (durable). |
| 57 | func probeKey(op string) string { return "probe:" + op } |
| 58 | |
| 59 | // relocStopProbe is relocationStopProbe and RecordProbeStop (ruling 49, |
| 60 | // ruling 75): op.exec stops its passed probe's session while the op is |
| 61 | // still relocating. The fence's owner is persisted first; the owned fence |
| 62 | // is installed only on an unfenced host or met again (rule |
| 63 | // reloc-probe-fence-owner; the mutation installs over any fence); then |
| 64 | // the whole tree is stopped and the session's database closed cleanly, and |
| 65 | // with root's receipt the stop is journaled. |
| 66 | func (w *world) relocStopProbe(b *box, c *claim) { |
| 67 | op, h := c.Op, c.Holder |
| 68 | if h == nil || h.WriterBox != b.ID || op.Exec != b.ID { |
| 69 | return |
| 70 | } |
| 71 | // ruling 87 (1b-next 6a4eadde, EndProbe): the release's own steps first |
| 72 | // (flushStopped): the writers stopped, the database and the files |
| 73 | // flushed, the positions recorded; only then the activator's fence |
| 74 | // and root's stop. A restart before the record leaves no replicator, |
| 75 | // so the flush refuses until ruling 67's deadline makes the op stuck, |
| 76 | // as a release does. The mutation of reloc-probe-stop-flush stops at |
| 77 | // once with nothing flushed (joint round 2's model) |
| 78 | if !w.evidence(b, "probe-writers:"+op.ID) { |
| 79 | w.stats["probe stop: a process survives"]++ |
| 80 | return |
| 81 | } |
| 82 | if on("reloc-probe-stop-flush") { |
| 83 | if r := b.probeRecord; r == nil || r.Op != op.ID || r.Session != h.Session { |
| 84 | tx, seq, ok := w.flushProbe(b, c) |
| 85 | if !ok { |
| 86 | return |
| 87 | } |
| 88 | b.probeRecord = &probeRecordEntry{Op: op.ID, Session: h.Session, TX: tx, Seq: seq} |
| 89 | w.stats["probe stop: flushed and recorded"]++ |
| 90 | } |
| 91 | } |
| 92 | owner := probeOwner(op.ID, h.Session) |
| 93 | if strings.HasPrefix(b.probeFence, probeKey(op.ID)+"/") && b.probeFence != owner { |
| 94 | w.stats["probe stop: the persisted fence owner names another session"]++ |
| 95 | return |
| 96 | } |
| 97 | b.probeFence = owner // probe-fence.json, persisted before anything is asked of the activator |
| 98 | if b.actFence != "" && b.actFence != owner { |
| 99 | if on("reloc-probe-fence-owner") { |
| 100 | // ErrFenceHeld (ruling 75 as ruling 87 amends it): the writers |
| 101 | // are stopped and flushed, root is not asked, and the other |
| 102 | // owner's fence is left as it is |
| 103 | w.stats["probe stop: another owner's fence, root not asked"]++ |
| 104 | w.logf("relocation %s: box %d's probe stop meets %s's fence; flushed, root not asked", op.ID, b.ID, b.actFence) |
| 105 | return |
| 106 | } |
| 107 | w.stats["probe stop: another owner's fence replaced (mutation)"]++ |
| 108 | } |
| 109 | b.actFence = owner |
| 110 | // root's writer-session stop and its receipt |
| 111 | if !w.evidence(b, probeKey(op.ID)) { |
| 112 | w.stats["probe stop: a process survives"]++ |
| 113 | return |
| 114 | } |
| 115 | if w.cas(c, func(n *claim) bool { |
| 116 | ok := sameOp(n.Op.ID, op.ID) && n.Op.Kind == "relocating" && n.Op.Phase == "probed" && n.Op.Exec == b.ID && n.Holder.Session == h.Session && n.Op.ProbeStop == nil |
| 117 | if ok { |
| 118 | n.Op.ProbeStop = &probeStopEntry{Box: b.ID, Session: h.Session} |
| 119 | } |
| 120 | return ok |
| 121 | }) { |
| 122 | w.stats["probe stop journaled"]++ |
| 123 | w.logf("relocation %s: box %d stopped its probe session %d with root's receipt", op.ID, b.ID, h.Session) |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | // relocComplete is the worker's CompleteRelocation: the serving CAS |
| 128 | // requires the journaled stop of exactly the probe's session (rule |
| 129 | // reloc-probe-stop; the mutation completes over the running probe, which |
| 130 | // strands its session across the kind change: INV-31). |
| 131 | func (w *world) relocComplete(c *claim) { |
| 132 | op := c.Op |
| 133 | if on("reloc-probe-stop") && (op.ProbeStop == nil || op.ProbeStop.Box != op.Exec || op.ProbeStop.Session != c.Holder.Session) { |
| 134 | return |
| 135 | } |
| 136 | w.release(c, op.Exec) |
| 137 | } |
| 138 | |
| 139 | // completionPass is relocationReleaseHold for every serving claim whose |
| 140 | // completion record names this box and is not marked (ruling 80). Each |
| 141 | // step is idempotent; a failure or a crash leaves the record unmarked and |
| 142 | // the next pass resumes. Rules: reloc-completion-claim-mode (ruling 82), |
| 143 | // reloc-completion-from-claim (ruling 80's finding), reloc-completion- |
| 144 | // discharge-first (ruling 66), reloc-completion-mark-last (ruling 80) and |
| 145 | // reloc-probe-fence-owner (the owned lift). |
| 146 | func (w *world) completionPass(b *box, c *claim) { |
| 147 | d := c.Op.Completed |
| 148 | if c.Op.Kind != "serving" || d == nil || d.Discharged || d.Box != b.ID || c.Holder == nil || c.Op.Exec != b.ID || c.Holder.WriterBox != b.ID { |
| 149 | return |
| 150 | } |
| 151 | if w.haOff && !on("reloc-completion-claim-mode") { |
| 152 | return // the mutation: the pass runs only with region HA on |
| 153 | } |
| 154 | if !on("reloc-completion-from-claim") && (b.Row == nil || b.Row.HoldOp != d.Op) { |
| 155 | return // the mutation: the completion is found from the row's hold, which a finished release no longer shows |
| 156 | } |
| 157 | if b.Row == nil { |
| 158 | w.stats["completion: no row, refused"]++ |
| 159 | return |
| 160 | } |
| 161 | if b.Row.HoldOp != "" && b.Row.HoldOp != d.Op { |
| 162 | w.stats["completion: the row is held by another op, refused"]++ |
| 163 | return |
| 164 | } |
| 165 | owner := probeOwner(d.Op, d.Session) |
| 166 | if b.probeFence != owner { |
| 167 | w.stats["completion: no probe-fence owner, refused"]++ |
| 168 | return |
| 169 | } |
| 170 | held := b.Row.HoldOp == d.Op |
| 171 | if !on("reloc-completion-mark-last") && !w.markCompleted(b, c) { |
| 172 | return // the mutation: the mark first |
| 173 | } |
| 174 | releaseHold := func() { |
| 175 | if b.Row != nil && b.Row.HoldOp == d.Op { |
| 176 | b.Row.Status, b.Row.HoldOp = "ready", "" |
| 177 | w.logf("box %d: completion of %s released the row's hold", b.ID, d.Op) |
| 178 | } |
| 179 | } |
| 180 | lift := func() { |
| 181 | if b.actFence == owner || !on("reloc-probe-fence-owner") { |
| 182 | b.actFence = "" // LiftFenceAs: only the owner's fence (the mutation lifts any) |
| 183 | } |
| 184 | } |
| 185 | if !on("reloc-completion-discharge-first") { |
| 186 | // the mutation: the hold released and the fence lifted before the |
| 187 | // discharge, so a serving admission can move the holder first |
| 188 | releaseHold() |
| 189 | lift() |
| 190 | if !w.dischargeCompleted(b, c, false) { |
| 191 | return |
| 192 | } |
| 193 | } else { |
| 194 | if held && c.Holder.Session != d.Session { |
| 195 | w.stats["completion: the holder is not the stopped probe session, refused"]++ |
| 196 | return |
| 197 | } |
| 198 | if !w.dischargeCompleted(b, c, held) { |
| 199 | return |
| 200 | } |
| 201 | if w.crashed(b, "discharge") { |
| 202 | return |
| 203 | } |
| 204 | releaseHold() |
| 205 | if w.crashed(b, "release") { |
| 206 | return |
| 207 | } |
| 208 | lift() |
| 209 | if w.crashed(b, "lift") { |
| 210 | return |
| 211 | } |
| 212 | } |
| 213 | if on("reloc-completion-mark-last") { |
| 214 | w.markCompleted(b, c) |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | // crashed is a witness's agent crash after the named step of the |
| 219 | // completion pass (a restart: the activator's fences are gone). |
| 220 | func (w *world) crashed(b *box, step string) bool { |
| 221 | if b.crashAfter != step { |
| 222 | return false |
| 223 | } |
| 224 | b.crashAfter = "" |
| 225 | b.forget() |
| 226 | w.logf("witness: box %d's agent crashed after the completion's %s", b.ID, step) |
| 227 | return true |
| 228 | } |
| 229 | |
| 230 | // dischargeCompleted is C10's discharge (completedRelocationDischarge, |
| 231 | // DischargeCompleted): keyed by the persisted binding, it needs, while the |
| 232 | // row is still held for the op, root's receipt for the stopped probe |
| 233 | // session; it closes main's participation slot and clears the .hostexec |
| 234 | // record (the model's execRecord). A replay after the hold's release finds |
| 235 | // no slot, which is positive evidence. dischargeFails is a witness's |
| 236 | // failing discharge. |
| 237 | func (w *world) dischargeCompleted(b *box, c *claim, held bool) bool { |
| 238 | d := c.Op.Completed |
| 239 | if b.dischargeFails { |
| 240 | w.stats["completion: discharge failed"]++ |
| 241 | return false |
| 242 | } |
| 243 | if held && !b.StopEvidence[probeKey(d.Op)] { |
| 244 | w.stats["completion: no receipt for the probe's stop, discharge refused"]++ |
| 245 | return false |
| 246 | } |
| 247 | if b.execRecord == d.Op { |
| 248 | b.execRecord, b.cleanupPending = "", false |
| 249 | } |
| 250 | if b.owedDischarge == d.Op { |
| 251 | b.owedDischarge = "" |
| 252 | w.stats["completion: participation discharged"]++ |
| 253 | } |
| 254 | return true |
| 255 | } |
| 256 | |
| 257 | // markCompleted is MarkCompletedDischarged: the serving op's record is |
| 258 | // marked, which lets any successor start. markFails is a witness's failed |
| 259 | // mark (the CAS lost, or the store refused it). |
| 260 | func (w *world) markCompleted(b *box, c *claim) bool { |
| 261 | d := c.Op.Completed |
| 262 | if b.markFails { |
| 263 | b.markFails = false |
| 264 | w.stats["completion: mark failed"]++ |
| 265 | return false |
| 266 | } |
| 267 | if w.cas(c, func(n *claim) bool { |
| 268 | x := n.Op.Completed |
| 269 | ok := n.Op.Kind == "serving" && x != nil && x.Op == d.Op && !x.Discharged |
| 270 | if ok { |
| 271 | cp := *x |
| 272 | cp.Discharged = true |
| 273 | n.Op.Completed = &cp |
| 274 | } |
| 275 | return ok |
| 276 | }) { |
| 277 | w.stats["completion marked discharged"]++ |
| 278 | w.logf("box %d: completion of %s marked discharged", b.ID, d.Op) |
| 279 | return true |
| 280 | } |
| 281 | return false |
| 282 | } |
| 283 | |
| 284 | // completionBarrier is completionBarrier (ruling 77; rule |
| 285 | // reloc-completion-barrier): true refuses a start from serving while the |
| 286 | // serving op's completed relocation is not marked, unless its box is on |
| 287 | // lost or is the source the start cites as fenced (fenced, -1 none). |
| 288 | func (w *world) completionBarrier(c *claim, fenced int) bool { |
| 289 | d := c.Op.Completed |
| 290 | if c.Op.Kind != "serving" || d == nil || d.Discharged || !on("reloc-completion-barrier") { |
| 291 | return false |
| 292 | } |
| 293 | if contains(c.Lost, d.Box) || fenced >= 0 && fenced == d.Box { |
| 294 | return false |
| 295 | } |
| 296 | w.stats["start refused: a completed relocation is not discharged"]++ |
| 297 | return true |
| 298 | } |
| 299 | |
| 300 | // completedHandover is N6 (wireCompletedHandover): the first serving wake |
| 301 | // on the box a relocation completed on, while the holder is still the |
| 302 | // stopped probe session, hands over from the probe's receipt and stops |
| 303 | // nothing. It reports whether the wake may go on. The mutation of |
| 304 | // reloc-completed-handover routes it to main's stop, which installs a |
| 305 | // serving-shaped fence over the probe's and re-stops S, whose stop root |
| 306 | // already completed under the probe's fence: the handover refuses for |
| 307 | // good, and the fence is never the probe's to lift. |
| 308 | func (w *world) completedHandover(b *box, c *claim) bool { |
| 309 | d := c.Op.Completed |
| 310 | if c.Op.Kind != "serving" || d == nil || d.Box != b.ID || c.Holder == nil || c.Holder.Session != d.Session || c.Holder.WriterBox != b.ID { |
| 311 | return true // not the completed prior: main's path |
| 312 | } |
| 313 | if !on("reloc-completed-handover") { |
| 314 | if b.actFence == "" || strings.HasPrefix(b.actFence, "probe:") { |
| 315 | b.actFence = fmt.Sprintf("main:%d", d.Session) |
| 316 | } |
| 317 | w.stats["serving wake: main's stop of the completed prior refused (mutation)"]++ |
| 318 | return false |
| 319 | } |
| 320 | if !b.StopEvidence[probeKey(d.Op)] { |
| 321 | w.stats["serving wake: no receipt for the completed probe, refused"]++ |
| 322 | return false // absence is never evidence |
| 323 | } |
| 324 | w.stats["serving wake: handed over from the probe's receipt"]++ |
| 325 | return true |
| 326 | } |
| 327 | |
| 328 | // completionHolds: the row's hold is the unmarked completion's own, which |
| 329 | // only the completion pass releases, after the discharge (ruling 66); the |
| 330 | // mutation of reloc-completion-discharge-first lets reconcile release it |
| 331 | // as a stale hold. |
| 332 | func (w *world) completionHolds(c *claim, b *box) bool { |
| 333 | d := c.Op.Completed |
| 334 | return d != nil && !d.Discharged && d.Box == b.ID && b.Row != nil && b.Row.HoldOp == d.Op && on("reloc-completion-discharge-first") |
| 335 | } |
| 336 | |
| 337 | // catchUpHold is reconcile's completedHold (finding 3, ruling 85): with |
| 338 | // region HA on, a box that is not the serving executor, not on lost, and |
| 339 | // owes a park-local for exactly the serving op's completed relocation |
| 340 | // (which completed elsewhere) holds its row for that op until the |
| 341 | // park-local retires it. It reports whether the case applied. The |
| 342 | // mutation drops the hold, and the row a delayed source's release left |
| 343 | // ready stays ready (INV-33: the edge routes requests to a box that does |
| 344 | // not serve). |
| 345 | func (w *world) catchUpHold(b *box, c *claim) bool { |
| 346 | d := c.Op.Completed |
| 347 | if w.haOff || c.Op.Kind != "serving" || d == nil || d.Box == b.ID || c.Op.Exec == b.ID || contains(c.Lost, b.ID) || b.Row == nil { |
| 348 | return false |
| 349 | } |
| 350 | owes := false |
| 351 | for _, o := range c.Obl { |
| 352 | if o.Kind == "park-local" && o.Box == b.ID && o.Op == d.Op { |
| 353 | owes = true |
| 354 | } |
| 355 | } |
| 356 | if !owes { |
| 357 | return false |
| 358 | } |
| 359 | if b.Row.Status == "ready" && on("reloc-completed-catchup-hold") { |
| 360 | b.Row.Status, b.Row.HoldOp = "relocating", d.Op |
| 361 | w.stats["catch-up hold: a delayed source held its row"]++ |
| 362 | w.logf("box %d: reconcile caught up: its row is held for completed relocation %s until its park-local", b.ID, d.Op) |
| 363 | } |
| 364 | if b.Row.Status == "ready" { |
| 365 | w.violate("INV-33: box %d's reconcile left its row ready after relocation %s completed on box %d, while it owes that op's park-local (the edge routes requests to a box that does not serve)", b.ID, d.Op, d.Box) |
| 366 | } |
| 367 | return true |
| 368 | } |
| 369 | |
| 370 | // checkCompletion is INV-31 (rulings 49, 66, 77): while a box has not |
| 371 | // discharged its participation in a completed relocation, nothing of the |
| 372 | // host runs on it (the stopped probe session stays the holder until C10 |
| 373 | // discharges it, and a session the kind change stranded is never |
| 374 | // discharged), and no operation names it as a participant (main's Prepare |
| 375 | // answers ErrLost). A box on lost, or destroyed, holds nothing a |
| 376 | // successor needs. |
| 377 | func (w *world) checkCompletion() { |
| 378 | c := w.claim |
| 379 | if c != nil && c.Holder != nil && c.Op.Kind == "serving" { |
| 380 | if d := c.Op.Completed; d != nil && !d.Discharged && c.Holder.Session != d.Session { |
| 381 | if b := w.boxByID(d.Box); b != nil && b.owedDischarge == d.Op && !b.Destroyed && !contains(c.Lost, b.ID) { |
| 382 | w.violate("INV-31: a writer was admitted (session %d, box %d) before box %d discharged its participation in completed relocation %s, whose holder must still be the stopped probe session %d (ruling 66)", c.Holder.Session, c.Holder.WriterBox, b.ID, d.Op, d.Session) |
| 383 | return |
| 384 | } |
| 385 | } |
| 386 | } |
| 387 | for _, b := range w.boxes { |
| 388 | if b.owedDischarge == "" || b.Destroyed || c != nil && contains(c.Lost, b.ID) { |
| 389 | continue |
| 390 | } |
| 391 | if len(b.Procs) > 0 { |
| 392 | w.violate("INV-31: the host runs on box %d before its participation in completed relocation %s was discharged (rulings 49, 66)", b.ID, b.owedDischarge) |
| 393 | return |
| 394 | } |
| 395 | if c != nil && c.Op.Kind != "serving" && c.Op.Kind != "" && opNames(c.Op, b.ID) { |
| 396 | w.violate("INV-31: %s %s names box %d before its participation in completed relocation %s was discharged (ruling 77: main's Prepare answers ErrLost)", c.Op.Kind, c.Op.ID, b.ID, b.owedDischarge) |
| 397 | return |
| 398 | } |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | // opNames: op names box id as its executor or a participant, or as its |
| 403 | // source or target in the kinds that set them. |
| 404 | func opNames(op opState, id int) bool { |
| 405 | if op.Exec == id || contains(op.Parts, id) { |
| 406 | return true |
| 407 | } |
| 408 | switch op.Kind { |
| 409 | case "moving", "landing", "restoring", "reverting", "pushing", "relocating": |
| 410 | return op.Src == id || op.Tgt == id |
| 411 | } |
| 412 | return false |
| 413 | } |
| 414 | |
| 415 | // c1Fallback: claim ask C1 is open, so a failed target has no release |
| 416 | // obligation and its release in probe-failed is refused (rule |
| 417 | // reloc-c1-failed-release-refused; the mutation releases as if C1 had |
| 418 | // landed). |
| 419 | func (w *world) c1Fallback() bool { return !w.claimC1 && on("reloc-c1-failed-release-refused") } |
| 420 | |
| 421 | // c11Fallback: claim ask C11 is open, so root launches no cold probe |
| 422 | // attempt after P's opening (rule reloc-c11-later-attempt-refused; the |
| 423 | // mutation counts and makes it as if C11 had landed). |
| 424 | func (w *world) c11Fallback() bool { return !w.claimC11 && on("reloc-c11-later-attempt-refused") } |
| 425 | |
| 426 | // verifiedCompletionFence is ProductionAuthority.completionFence (ruling |
| 427 | // 86, 1b-next ca282378): a delete or a purge passes StartGuard the |
| 428 | // completion box whose termination fence it verified, as a failover |
| 429 | // passes its fenced source, or -1. Verified means the box is in the |
| 430 | // roster (enrolled), its box-fences record exists (a regional worker |
| 431 | // observed the termination: FenceObservedAt) and the cloud says it is |
| 432 | // terminated (VerifyFenceForBox); never inferred from a missing fence. |
| 433 | // The mutation of reloc-completion-fenced-exempt verifies nothing, as |
| 434 | // before ruling 86: a completion box terminated before its discharge |
| 435 | // while a purge holds admission closed blocks the purge, and the purge's |
| 436 | // closed admission blocks the failover (seed 2224 / 3,000, two boxes). |
| 437 | func (w *world) verifiedCompletionFence(c *claim) int { |
| 438 | d := c.Op.Completed |
| 439 | if c.Op.Kind != "serving" || d == nil || d.Discharged || contains(c.Lost, d.Box) || !on("reloc-completion-fenced-exempt") { |
| 440 | return -1 |
| 441 | } |
| 442 | x := w.boxByID(d.Box) |
| 443 | if x == nil || !x.Enrolled || !x.Terminated || x.FenceObservedAt == 0 { |
| 444 | return -1 |
| 445 | } |
| 446 | w.stats["start: the completion box's termination fence verified"]++ |
| 447 | return x.ID |
| 448 | } |
| 449 | |
| 450 | // completedPrior is relocationCompletedHandover.completedPrior: the claim |
| 451 | // serves on b with a completion recorded on b, and the holder is still the |
| 452 | // stopped probe session. |
| 453 | func (w *world) completedPrior(b *box, c *claim) bool { |
| 454 | d := c.Op.Completed |
| 455 | return c.Op.Kind == "serving" && d != nil && d.Box == b.ID && c.Holder != nil && c.Holder.Session == d.Session && c.Holder.WriterBox == b.ID |
| 456 | } |
| 457 | |
| 458 | // closeLaunches is StopWriters' first step: the current attempt's and |
| 459 | // session's launch journal is closed, so the stopped session is never |
| 460 | // relaunched (an admitted start still in progress is abandoned). It is |
| 461 | // not root's admission fence, which Collect writes later (relFence). |
| 462 | func (w *world) closeLaunches(b *box, op string, session int) { |
| 463 | if b.launchClosedOp != op || b.launchClosedSession != session { |
| 464 | b.launchClosedOp, b.launchClosedSession = op, session |
| 465 | w.logf("box %d: relocation %s closed session %d's launch journal", b.ID, op, session) |
| 466 | } |
| 467 | if b.open != nil && b.open.Session == session { |
| 468 | b.open = nil |
| 469 | w.stats["release: a start in progress abandoned with its session"]++ |
| 470 | } |
| 471 | } |
| 472 | |
| 473 | // probeRecordEntry is the probe stop's recorded positions (ruling 87, |
| 474 | // ReleasePositions under the probe's own stop request): durable, so a |
| 475 | // restart after the record stops without flushing again. |
| 476 | type probeRecordEntry struct { |
| 477 | Op string |
| 478 | Session, TX int |
| 479 | Seq int |
| 480 | } |
| 481 | |
| 482 | // flushProbe is flushStopped's database and files steps for the probe |
| 483 | // session (ruling 87, the release's FlushDB and FlushFiles): they need |
| 484 | // the session's own replicator and the box's own files directory; with |
| 485 | // either gone (a restart before the record) the flush refuses. The |
| 486 | // pending writes ship and the listing must reach the stopped database's |
| 487 | // position; the files and wake state are published as one manifest of the |
| 488 | // generation, acknowledged. It reports the positions once both landed. |
| 489 | func (w *world) flushProbe(b *box, c *claim) (tx, seq int, ok bool) { |
| 490 | h := c.Holder |
| 491 | db := b.DB |
| 492 | if db == nil || db.Inc != h.Inc || db.Gen != h.Gen || db.Epoch != b.Epoch || db.Session != h.Session || b.Files == nil || b.Files.Inc != h.Inc { |
| 493 | w.stats["probe stop: no replicator or files directory and no record, refused"]++ |
| 494 | return 0, 0, false |
| 495 | } |
| 496 | if len(db.Pending) > 0 { |
| 497 | bl := w.deltaOf(db, db.Inc) |
| 498 | db.FirstFull = false |
| 499 | db.Queue = append(db.Queue, bl) |
| 500 | db.Pos, db.Pending = bl.Max, nil |
| 501 | } |
| 502 | w.ship(b) |
| 503 | if len(db.Queue) > 0 || db.InFlight != nil || w.remoteMax(w.buckets[b.Region], dataKey(h.Location, h.Gen, "")) < db.Pos { |
| 504 | w.stats["probe stop: waits for the flush"]++ |
| 505 | return 0, 0, false |
| 506 | } |
| 507 | files := b.Files.clone() |
| 508 | if b.pubGen != h.Gen { |
| 509 | b.pubGen, b.pubSeq, b.releaseSeq = h.Gen, 0, 0 |
| 510 | } |
| 511 | if !w.ensureObject(b, h, files) { |
| 512 | w.stats["probe stop: waits for its files object"]++ |
| 513 | return 0, 0, false |
| 514 | } |
| 515 | if b.releaseSeq == 0 { |
| 516 | b.pubSeq++ |
| 517 | b.releaseSeq = b.pubSeq |
| 518 | } |
| 519 | w.issued++ |
| 520 | bl := &blob{Kind: "files", Inc: h.Inc, Gen: h.Gen, Seq: b.releaseSeq, Files: files.clone(), Timers: append([]int(nil), b.Timers...), Base: w.issued} |
| 521 | if !w.ackedPut(b, manifestKey(h.Location, h.Gen, b.releaseSeq), bl, true) { |
| 522 | w.stats["probe stop: waits for its manifest"]++ |
| 523 | return 0, 0, false |
| 524 | } |
| 525 | seq, b.releaseSeq = b.releaseSeq, 0 |
| 526 | b.published = filesState(b.Timers, files) |
| 527 | return db.Pos, seq, true |
| 528 | } |
| 529 | |
amadan