| 1 | package hostclaim |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "regexp" |
| 7 | "strconv" |
| 8 | "strings" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/regionname" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | // Move phases on the claim's op (moves spec §7.5, §19.1). The record |
| 16 | // carries the detailed journal; the claim carries only what the flip, |
| 17 | // the commit and a rollback compare. |
| 18 | const ( |
| 19 | PhaseMoving = "" // moving, before the seal |
| 20 | PhaseSealed = "sealed" // moving, after the cut's staged phase |
| 21 | PhaseAborting = "aborting" // moving, the abort has begun (Abort): the lane park is queued |
| 22 | ) |
| 23 | |
| 24 | // Journal keys the cut and the flip write on the claim's op (moves spec |
| 25 | // §7.4, §7.5). The record carries the detail; the claim carries what the |
| 26 | // flip, the commit and a rollback compare. |
| 27 | const ( |
| 28 | JournalSeal = "seal" // the sealed cut's summary |
| 29 | JournalSrcGen = "src_gen" // the source's generation, journalled at the flip |
| 30 | JournalSourceStamp = "source_stamp" // the source installation's stamp S, journalled at the flip |
| 31 | JournalCutSession = "cut_session" // the cut's session |
| 32 | JournalRevPhase = "rev_phase" // a revert's own phase |
| 33 | JournalEvidenceSource = "evidence_source" // the source's stop evidence (K3) |
| 34 | JournalEvidenceTarget = "evidence_target" // the target's stop evidence (K3) |
| 35 | ) |
| 36 | |
| 37 | // journalReleasedMove is the key under which a serving op remembers the |
| 38 | // move that released it (K8). |
| 39 | const journalReleasedMove = "released_move" |
| 40 | |
| 41 | // BeginMove is K2: the request door CASes serving → moving, with the move |
| 42 | // id, both participants and exec = the source box. The source must already |
| 43 | // be the executor; nothing moves exec here. Touched is not widened: K12 |
| 44 | // has every write site add its own region or box first (Touch). Every |
| 45 | // other claim field (holder, admissions, generation, chores) is left as |
| 46 | // found. |
| 47 | func BeginMove(opID, by, sourceBox, targetBox, targetRegion string, now time.Time) func(*Claim) error { |
| 48 | return func(c *Claim) error { |
| 49 | if err := StartGuard(c, []string{targetRegion}, []string{targetBox}, ""); err != nil { |
| 50 | return err |
| 51 | } |
| 52 | if opID == "" || sourceBox == "" || targetBox == "" || targetRegion == "" { |
| 53 | return fmt.Errorf("%w: a move needs an op id, two boxes and a target region", ErrMalformed) |
| 54 | } |
| 55 | if c.Op.Exec != sourceBox { |
| 56 | return fmt.Errorf("%w: exec is %q, the move's source is %q", ErrWrongState, c.Op.Exec, sourceBox) |
| 57 | } |
| 58 | c.Op = Op{Kind: KindMoving, ID: opID, Phase: PhaseMoving, By: by, StartedAt: now.UTC(), Exec: sourceBox, |
| 59 | Participants: []string{sourceBox, targetBox}, |
| 60 | Intent: map[string]string{"to": targetRegion, "source": sourceBox, "target": targetBox}} |
| 61 | return nil |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | // MoveAdmission is what a claim-mode request door proved before its start |
| 66 | // CAS (claim Plan 5 Task 7): the canonical app id, the id of the exact |
| 67 | // "move" registration the control writer minted for this claim observation |
| 68 | // (always the move's op id, so admission draining keeps it while the op |
| 69 | // holds the claim), and both boxes' enrollments from the signed roster. |
| 70 | // hostclaim only records them; hostops.BeginClaimMove does the proving. |
| 71 | type MoveAdmission struct { |
| 72 | HomeRegion string |
| 73 | AppID, AdmissionID, SourceEnrollment, TargetEnrollment string |
| 74 | } |
| 75 | |
| 76 | // BeginAdmittedMove is BeginMove plus the admission it ran under, in the same |
| 77 | // closure and so the same CAS. The move's own keys ("source", "target", "to") |
| 78 | // are unchanged; "app_id", "admission_id", "source_enrollment" and |
| 79 | // "target_enrollment" are additive. ConvertMovingToRelocating requires the |
| 80 | // first three. |
| 81 | func BeginAdmittedMove(opID, by, sourceBox, targetBox, targetRegion string, now time.Time, a MoveAdmission) func(*Claim) error { |
| 82 | return func(c *Claim) error { |
| 83 | if a.AppID == "" || a.AdmissionID == "" || a.AdmissionID != opID || !validEnrollment(a.SourceEnrollment) || !validEnrollment(a.TargetEnrollment) || a.SourceEnrollment == a.TargetEnrollment { |
| 84 | return fmt.Errorf("%w: an admitted move needs its app, its registration and both enrollments", ErrMalformed) |
| 85 | } |
| 86 | if a.HomeRegion != "" && regionname.PlacementRegion(a.HomeRegion) != a.HomeRegion { |
| 87 | return ErrMalformed |
| 88 | } |
| 89 | regions := []string{targetRegion} |
| 90 | if regionname.SameRegion(a.HomeRegion, targetRegion, regionname.HomeRegion) { |
| 91 | regions = append(regions, a.HomeRegion, regionname.HomeRegion) |
| 92 | } |
| 93 | if err := StartGuard(c, regions, []string{targetBox}, ""); err != nil { |
| 94 | return err |
| 95 | } |
| 96 | if err := BeginMove(opID, by, sourceBox, targetBox, targetRegion, now)(c); err != nil { |
| 97 | return err |
| 98 | } |
| 99 | if a.HomeRegion != "" { |
| 100 | c.Op.Intent["home_region"] = a.HomeRegion |
| 101 | } |
| 102 | c.Op.Intent["app_id"] = a.AppID |
| 103 | c.Op.Intent["admission_id"] = a.AdmissionID |
| 104 | c.Op.Intent["source_enrollment"] = a.SourceEnrollment |
| 105 | c.Op.Intent["target_enrollment"] = a.TargetEnrollment |
| 106 | return nil |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | // validEnrollment is the roster's enrollment token: 32 lowercase hex. |
| 111 | func validEnrollment(e string) bool { |
| 112 | if len(e) != 32 { |
| 113 | return false |
| 114 | } |
| 115 | for _, r := range e { |
| 116 | if (r < '0' || r > '9') && (r < 'a' || r > 'f') { |
| 117 | return false |
| 118 | } |
| 119 | } |
| 120 | return true |
| 121 | } |
| 122 | |
| 123 | // Touch is K12: a write site adds its region or box to touched before its |
| 124 | // first write there, under the op it serves. Idempotent. |
| 125 | func Touch(opID string, regions, boxes []string) func(*Claim) error { |
| 126 | return func(c *Claim) error { |
| 127 | if err := RequireOp(opID)(c); err != nil { |
| 128 | return err |
| 129 | } |
| 130 | c.Touched.Regions = union(c.Touched.Regions, regions) |
| 131 | c.Touched.Boxes = union(c.Touched.Boxes, boxes) |
| 132 | return nil |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | // RequireMoveOp is K5's revalidation in one closure: op id, kind and phase |
| 137 | // must be what the step expects. |
| 138 | func RequireMoveOp(opID, kind, phase string) func(*Claim) error { |
| 139 | return func(c *Claim) error { |
| 140 | if err := RequireOp(opID)(c); err != nil { |
| 141 | return err |
| 142 | } |
| 143 | if c.Op.Kind != kind || c.Op.Phase != phase { |
| 144 | return fmt.Errorf("%w: op is %s/%q, step expects %s/%q", ErrWrongState, c.Op.Kind, c.Op.Phase, kind, phase) |
| 145 | } |
| 146 | return nil |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | // ReleaseMove is K8: a move hands the host back to serving. It is legal |
| 151 | // from moving before the seal (phase "": the request door's compensation for |
| 152 | // a move whose record start failed), and from the end of each move outcome, |
| 153 | // read from op.phase (J1, J6): a committed landing, an opened rollback |
| 154 | // (restoring) and an opened revert (reverting). In those three the executor |
| 155 | // named must be the one executing. An aborting op is ReleaseAbort's (Plan 5 |
| 156 | // ruling K7), never this. It is legal only once no obligation under this op |
| 157 | // remains. The serving op keeps exec and records which move released it: |
| 158 | // operation-bound completion evidence for the abort's crash boundary |
| 159 | // (RequireReleasedMove). |
| 160 | func ReleaseMove(opID, execBox string) func(*Claim) error { |
| 161 | return func(c *Claim) error { |
| 162 | if err := RequireOp(opID)(c); err != nil { |
| 163 | return err |
| 164 | } |
| 165 | door := c.Op.Kind == KindMoving && c.Op.Phase == PhaseMoving |
| 166 | ended := (c.Op.Kind == KindLanding && c.Op.Phase == PhaseCommitted) || |
| 167 | ((c.Op.Kind == KindRestoring || c.Op.Kind == KindReverting) && c.Op.Phase == PhaseOpened) |
| 168 | if !door && !ended { |
| 169 | return fmt.Errorf("%w: op is %s/%q, a release needs moving before the seal, a committed landing or an opened recovery", ErrWrongState, c.Op.Kind, c.Op.Phase) |
| 170 | } |
| 171 | if execBox == "" { |
| 172 | return fmt.Errorf("%w: a release names the executor", ErrMalformed) |
| 173 | } |
| 174 | if ended && c.Op.Exec != execBox { |
| 175 | return fmt.Errorf("%w: exec is %q, the release names %q", ErrWrongState, c.Op.Exec, execBox) |
| 176 | } |
| 177 | return releaseToServing(c, opID, execBox) |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // releaseToServing is every move release's last step: legal only once no |
| 182 | // obligation under the op remains; the serving op keeps the executor and |
| 183 | // records the move it released. |
| 184 | func releaseToServing(c *Claim, opID, execBox string) error { |
| 185 | for _, o := range c.Obligations { |
| 186 | if o.Op == opID { |
| 187 | return fmt.Errorf("%w: %s %s%s is still owed under this move", ErrPendingChore, o.Kind, o.Region, o.Box) |
| 188 | } |
| 189 | } |
| 190 | c.Op = Op{Kind: KindServing, Exec: execBox, Journal: map[string]string{journalReleasedMove: opID}} |
| 191 | return nil |
| 192 | } |
| 193 | |
| 194 | // Abort is the claim half of a move's abort (moves spec §7.11, joint N13; |
| 195 | // Plan 5 Task 1): this move's moving op before the seal, with the source |
| 196 | // still executing, goes to moving/aborting, and the park of its own staging |
| 197 | // lane is queued in the same write for the target region, owned by that |
| 198 | // region (QueueStagingPark's chore, index D6). The source cannot read or |
| 199 | // park the target's moves/*, so the lane is the target region's to retire; |
| 200 | // execution stays with the source, which serves throughout. |
| 201 | // |
| 202 | // The scope is moves/<host>/<op> with the holder's incarnation and the |
| 203 | // journalled abort bound the caller computed (moves.AbortScope, ruling K5). |
| 204 | // A replay finds the op already aborting: it is a no-op when the same lane |
| 205 | // park is owed, or when none is (the target already acknowledged it), and |
| 206 | // is refused when a lane park with another scope is owed. It allocates no |
| 207 | // ids. A cut past requested cannot abort, and neither can an op whose |
| 208 | // execution was given up (that is a revert). |
| 209 | func Abort(opID, targetRegion string, scope ParkScope, newID func() string) func(*Claim) error { |
| 210 | return func(c *Claim) error { |
| 211 | if err := RequireOp(opID)(c); err != nil { |
| 212 | return err |
| 213 | } |
| 214 | if c.Op.Kind != KindMoving || (c.Op.Phase != PhaseMoving && c.Op.Phase != PhaseAborting) || c.Op.Exec == "" { |
| 215 | return fmt.Errorf("%w: op is %s/%q exec %q, an abort needs moving before the seal with the source executing", ErrWrongState, c.Op.Kind, c.Op.Phase, c.Op.Exec) |
| 216 | } |
| 217 | if targetRegion == "" || c.Op.Intent["to"] != targetRegion { |
| 218 | return fmt.Errorf("%w: %q is not the move's target region", ErrWrongState, targetRegion) |
| 219 | } |
| 220 | var err error |
| 221 | if scope, err = parkQueueGuard(c, opID, targetRegion, scope, newID); err != nil { |
| 222 | return err |
| 223 | } |
| 224 | // Restates moves.ScopeLocation(host, op), as QueueStagingPark does. |
| 225 | if scope.Location != "moves/"+c.Host+"/"+opID { |
| 226 | return fmt.Errorf("%w: an abort parks this move's own lane, moves/%s/%s", ErrMalformed, c.Host, opID) |
| 227 | } |
| 228 | if c.Op.Phase == PhaseAborting { |
| 229 | for _, o := range c.Obligations { |
| 230 | if o.Op == opID && o.Kind == "park-replica" && regionname.SameRegion(c.Op.Intent["home_region"], o.Region, targetRegion) && o.Park != nil && isStagingLocation(o.Park.Location) && !sameParkScope(*o.Park, scope) { |
| 231 | return fmt.Errorf("%w: %s already owes its lane park in %s with another scope", ErrWrongState, opID, targetRegion) |
| 232 | } |
| 233 | } |
| 234 | return nil |
| 235 | } |
| 236 | if _, err := queueReplicaPark(c, opID, targetRegion, scope, newID); err != nil { |
| 237 | return err |
| 238 | } |
| 239 | c.Op.Phase = PhaseAborting |
| 240 | return nil |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | // ReleaseAbort is the abort's release (ruling K7): from moving/aborting |
| 245 | // only, so always after Abort queued the lane park, with the named source |
| 246 | // still executing, and only once no obligation under the op remains (the |
| 247 | // target region acknowledged the lane park). The serving op keeps the |
| 248 | // source as its executor and records the move it released. |
| 249 | func ReleaseAbort(opID, sourceBox string) func(*Claim) error { |
| 250 | return func(c *Claim) error { |
| 251 | if err := RequireMoveOp(opID, KindMoving, PhaseAborting)(c); err != nil { |
| 252 | return err |
| 253 | } |
| 254 | if sourceBox == "" { |
| 255 | return fmt.Errorf("%w: a release names the executor", ErrMalformed) |
| 256 | } |
| 257 | if c.Op.Exec != sourceBox { |
| 258 | return fmt.Errorf("%w: exec is %q, the abort's release names %q", ErrWrongState, c.Op.Exec, sourceBox) |
| 259 | } |
| 260 | return releaseToServing(c, opID, sourceBox) |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | // RequireReleasedMove is the replay rule for a crashed abort: nil when |
| 265 | // the claim is serving and its op records that move opID released it, |
| 266 | // else ErrStaleOp. A plain serving claim, or one released by another |
| 267 | // move, does not satisfy it. |
| 268 | func RequireReleasedMove(opID string) func(*Claim) error { |
| 269 | return func(c *Claim) error { |
| 270 | if opID == "" || c.Op.Kind != KindServing || c.Op.Journal[journalReleasedMove] != opID { |
| 271 | return fmt.Errorf("%w: the claim does not record move %q as released", ErrStaleOp, opID) |
| 272 | } |
| 273 | return nil |
| 274 | } |
| 275 | } |
| 276 | |
| 277 | func setJournal(c *Claim, k, v string) { |
| 278 | if c.Op.Journal == nil { |
| 279 | c.Op.Journal = map[string]string{} |
| 280 | } |
| 281 | c.Op.Journal[k] = v |
| 282 | } |
| 283 | |
| 284 | // SetJournal records move journal entries under the op (D1): what the |
| 285 | // claim half must read with the same fresh read as an admission. It names |
| 286 | // the op only, so any kind may use it. |
| 287 | func SetJournal(opID string, kv map[string]string) func(*Claim) error { |
| 288 | return func(c *Claim) error { |
| 289 | if err := RequireOp(opID)(c); err != nil { |
| 290 | return err |
| 291 | } |
| 292 | for k, v := range kv { |
| 293 | setJournal(c, k, v) |
| 294 | } |
| 295 | return nil |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | // SetMovePhase moves the op's phase one step, under its kind. A replay |
| 300 | // finds the phase already moved and is refused (ErrWrongState). |
| 301 | func SetMovePhase(opID, kind, from, to string) func(*Claim) error { |
| 302 | return func(c *Claim) error { |
| 303 | if err := RequireMoveOp(opID, kind, from)(c); err != nil { |
| 304 | return err |
| 305 | } |
| 306 | c.Op.Phase = to |
| 307 | return nil |
| 308 | } |
| 309 | } |
| 310 | |
| 311 | // ExecToNone is K3: the participant that holds execution and has written |
| 312 | // stop evidence for this op gives it up, citing the evidence under |
| 313 | // journalKey (evidence_source or evidence_target). Nobody moves exec from |
| 314 | // one box to another. A replay after a crash finds exec already empty and |
| 315 | // the same evidence journalled: that is idempotent, not a conflict. |
| 316 | func ExecToNone(opID, box, evidenceKey, journalKey string) func(*Claim) error { |
| 317 | return func(c *Claim) error { |
| 318 | if err := RequireOp(opID)(c); err != nil { |
| 319 | return err |
| 320 | } |
| 321 | if box == "" || evidenceKey == "" || (journalKey != JournalEvidenceSource && journalKey != JournalEvidenceTarget) { |
| 322 | return fmt.Errorf("%w: execution is given up by a named executor, citing evidence", ErrMalformed) |
| 323 | } |
| 324 | if c.Op.Exec == "" && c.Op.Journal[journalKey] == evidenceKey { |
| 325 | return nil |
| 326 | } |
| 327 | if c.Op.Exec != box { |
| 328 | return fmt.Errorf("%w: exec is %q, not %q", ErrWrongState, c.Op.Exec, box) |
| 329 | } |
| 330 | c.Op.Exec = "" |
| 331 | setJournal(c, journalKey, evidenceKey) |
| 332 | return nil |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | // phaseLandingChecking is the landing's first phase, the one the flip writes. |
| 337 | const phaseLandingChecking = "checking" |
| 338 | |
| 339 | // Flip is the claim half of the record's flip (§7.5, K4): moving/sealed to |
| 340 | // landing/checking, exec none to the target, citing the source's evidence. |
| 341 | // holder.stamp becomes the gen this CAS produces (the target's installation |
| 342 | // stamp F; Update sets gen = observed + 1 after the transition runs, as |
| 343 | // CompleteCreate relies on); the source's stamp S and generation are |
| 344 | // journalled for a rollback's return. holder.generation is not touched. |
| 345 | // |
| 346 | // Either participant may write it (G12: the target after a source crash), |
| 347 | // so a second write of the same flip is a no-op: the claim is already |
| 348 | // landing/checking for this op and target with the same journalled S and |
| 349 | // generation. It does not restamp the holder. Any other flip over an |
| 350 | // already-flipped claim is refused. |
| 351 | func Flip(opID, targetBox string, sourceStamp uint64, srcGen string) func(*Claim) error { |
| 352 | return func(c *Claim) error { |
| 353 | if err := RequireOp(opID)(c); err != nil { |
| 354 | return err |
| 355 | } |
| 356 | if c.Op.Kind == KindLanding && c.Op.Phase == phaseLandingChecking && c.Op.Exec == targetBox && |
| 357 | len(c.Op.Participants) == 2 && c.Op.Participants[1] == targetBox && |
| 358 | c.Op.Journal[JournalSourceStamp] == strconv.FormatUint(sourceStamp, 10) && c.Op.Journal[JournalSrcGen] == srcGen { |
| 359 | return nil |
| 360 | } |
| 361 | if err := RequireMoveOp(opID, KindMoving, PhaseSealed)(c); err != nil { |
| 362 | return err |
| 363 | } |
| 364 | if c.Holder == nil { |
| 365 | return fmt.Errorf("%w: the flip needs the holder", ErrWrongState) |
| 366 | } |
| 367 | if c.Op.Exec != "" || c.Op.Journal[JournalEvidenceSource] == "" { |
| 368 | return fmt.Errorf("%w: the flip needs exec none and the source's evidence", ErrWrongState) |
| 369 | } |
| 370 | // The first flip names the source's own installation: its stamp S and |
| 371 | // generation are what a rollback returns to (§6.4, §7.8), so they |
| 372 | // must be what the claim holds now, before the stamp is overwritten. |
| 373 | if sourceStamp != c.Holder.Stamp || srcGen != c.Holder.Generation { |
| 374 | return fmt.Errorf("%w: the flip names stamp %d generation %q, the holder is at %d and %q", ErrWrongState, sourceStamp, srcGen, c.Holder.Stamp, c.Holder.Generation) |
| 375 | } |
| 376 | if len(c.Op.Participants) != 2 || targetBox == "" || c.Op.Participants[1] != targetBox { |
| 377 | return fmt.Errorf("%w: %q is not the move's target", ErrWrongState, targetBox) |
| 378 | } |
| 379 | c.Op.Kind, c.Op.Phase, c.Op.Exec = KindLanding, phaseLandingChecking, targetBox |
| 380 | setJournal(c, JournalSourceStamp, strconv.FormatUint(sourceStamp, 10)) |
| 381 | setJournal(c, JournalSrcGen, srcGen) |
| 382 | c.Holder.Stamp = c.Gen + 1 |
| 383 | return nil |
| 384 | } |
| 385 | } |
| 386 | |
| 387 | // Revert is §7.4 reverting: the source takes execution back with its own |
| 388 | // evidence, before the flip. The target region's staging park is queued in |
| 389 | // the same write (§7.11: the revert's `opened` waits for it), owned by the |
| 390 | // target region and scoped to this move's own lane, moves/<host>/<op>, with |
| 391 | // the holder's incarnation and the writers' horizon the caller computed |
| 392 | // (G7). Nothing here may park the live location, the host-wide staging |
| 393 | // prefix or another move's lane. A park already owed under this op for the |
| 394 | // region is not queued twice. |
| 395 | // |
| 396 | // A replay after a crash finds the claim already reverting, with this op's |
| 397 | // source, region, rev_phase "evidence" and the same staging park: that is |
| 398 | // a no-op and allocates no ids. |
| 399 | func Revert(opID, sourceBox, targetRegion string, stagingScope *ParkScope, newID func() string) func(*Claim) error { |
| 400 | return func(c *Claim) error { |
| 401 | if err := RequireOp(opID)(c); err != nil { |
| 402 | return err |
| 403 | } |
| 404 | if c.Op.Kind == KindReverting { |
| 405 | if c.Op.Exec == sourceBox && sourceBox != "" && len(c.Op.Participants) > 0 && c.Op.Participants[0] == sourceBox && |
| 406 | c.Op.Journal[JournalEvidenceSource] != "" && c.Op.Journal[JournalRevPhase] == RevPhaseEvidence && |
| 407 | targetRegion != "" && c.Op.Intent["to"] == targetRegion && stagingScope != nil { |
| 408 | for _, o := range c.Obligations { |
| 409 | if o.Op == opID && o.Kind == "park-replica" && regionname.SameRegion(c.Op.Intent["home_region"], o.Region, targetRegion) && o.Park != nil && sameParkScope(*o.Park, *stagingScope) { |
| 410 | return nil |
| 411 | } |
| 412 | } |
| 413 | } |
| 414 | return fmt.Errorf("%w: the op is reverting, and not with this revert's source, region and staging park", ErrWrongState) |
| 415 | } |
| 416 | if c.Op.Kind != KindMoving || c.Op.Exec != "" || c.Op.Journal[JournalEvidenceSource] == "" { |
| 417 | return fmt.Errorf("%w: revert needs a moving op with exec none and the source's evidence", ErrWrongState) |
| 418 | } |
| 419 | if len(c.Op.Participants) == 0 || sourceBox == "" || c.Op.Participants[0] != sourceBox { |
| 420 | return fmt.Errorf("%w: %q is not the move's source", ErrWrongState, sourceBox) |
| 421 | } |
| 422 | if targetRegion == "" || c.Op.Intent["to"] != targetRegion { |
| 423 | return fmt.Errorf("%w: %q is not the move's target region", ErrWrongState, targetRegion) |
| 424 | } |
| 425 | if stagingScope == nil || newID == nil || c.Holder == nil { |
| 426 | return fmt.Errorf("%w: a revert names the staging lane it leaves behind", ErrMalformed) |
| 427 | } |
| 428 | if err := stagingScope.Validate(c.Host); err != nil { |
| 429 | return err |
| 430 | } |
| 431 | if stagingScope.Location != "moves/"+c.Host+"/"+opID || stagingScope.Incarnation != c.Holder.Incarnation { |
| 432 | return fmt.Errorf("%w: the revert's park must be this move's own lane and this holder's incarnation", ErrMalformed) |
| 433 | } |
| 434 | setJournal(c, JournalSourceStamp, strconv.FormatUint(c.Holder.Stamp, 10)) |
| 435 | setJournal(c, JournalSrcGen, c.Holder.Generation) |
| 436 | c.Op.Kind, c.Op.Exec = KindReverting, sourceBox |
| 437 | if c.Op.Journal[JournalRevPhase] == "" { |
| 438 | setJournal(c, JournalRevPhase, RevPhaseEvidence) |
| 439 | } |
| 440 | for _, o := range c.Obligations { |
| 441 | if o.Op == opID && o.Kind == "park-replica" && regionname.SameRegion(c.Op.Intent["home_region"], o.Region, targetRegion) { |
| 442 | return nil |
| 443 | } |
| 444 | } |
| 445 | s := *stagingScope |
| 446 | c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-replica", Region: targetRegion, Retirement: newID(), Owner: "region:" + targetRegion, Park: &s}) |
| 447 | return nil |
| 448 | } |
| 449 | } |
| 450 | |
| 451 | func sameParkScope(a, b ParkScope) bool { |
| 452 | return a.Location == b.Location && a.Incarnation == b.Incarnation && a.EvidenceAt.Equal(b.EvidenceAt) && a.RequestLimit == b.RequestLimit |
| 453 | } |
| 454 | |
| 455 | // Park queue helpers for move retirements (moves spec §6.4, index D6). A |
| 456 | // move adds no obligation kind: a staging lane is a park-replica scoped to |
| 457 | // moves/<host>/<op>; a whole location retired where it is no longer |
| 458 | // authoritative is a park-replica scoped to instances/<host>[/<inc>]; a |
| 459 | // landing's earlier-stay residue is one park-generation per generation |
| 460 | // (RootGeneration for the origin); the box's own artifacts are a park-local. |
| 461 | // Claim.Validate accepts all of them unchanged, and the park engine archives |
| 462 | // them under parked/<incarnation>/<host>/<retirement>/ with the op in the |
| 463 | // manifest. |
| 464 | // |
| 465 | // Each helper changes obligations only, is idempotent per its key (a replay |
| 466 | // allocates no ids), and refuses a replay that names the same chore with a |
| 467 | // different scope rather than overwriting it. The scopes themselves come |
| 468 | // from internal/moves (SourceLocationScope, StagingScope, AbortScope, |
| 469 | // ReturnLaneScope), the one source for the box and the console; moves |
| 470 | // imports hostclaim, so they cannot be called from here. |
| 471 | |
| 472 | // parkQueueGuard is the check every helper shares: the op is current, the |
| 473 | // claim has its holder, the scope is valid for this host and names this |
| 474 | // holder's incarnation. |
| 475 | func parkQueueGuard(c *Claim, opID, region string, scope ParkScope, newID func() string) (ParkScope, error) { |
| 476 | if err := RequireOp(opID)(c); err != nil { |
| 477 | return scope, err |
| 478 | } |
| 479 | if c.Holder == nil || newID == nil || region == "" { |
| 480 | return scope, fmt.Errorf("%w: a move's park names its region and holder and has a way to mint ids", ErrMalformed) |
| 481 | } |
| 482 | scope.Location = strings.TrimSuffix(scope.Location, "/") |
| 483 | if err := scope.Validate(c.Host); err != nil { |
| 484 | return scope, err |
| 485 | } |
| 486 | if scope.Incarnation != c.Holder.Incarnation { |
| 487 | return scope, fmt.Errorf("%w: the park's incarnation is not this holder's", ErrMalformed) |
| 488 | } |
| 489 | return scope, nil |
| 490 | } |
| 491 | |
| 492 | // requireHolderLocation: location and residue parks name holder.location |
| 493 | // (§6.4), in whichever form the holder uses, not the other one. |
| 494 | func requireHolderLocation(c *Claim, scope ParkScope) error { |
| 495 | if isStagingLocation(scope.Location) { |
| 496 | return fmt.Errorf("%w: a location park is not a staging lane", ErrMalformed) |
| 497 | } |
| 498 | if scope.Location != strings.TrimSuffix(c.Holder.Location, "/") { |
| 499 | return fmt.Errorf("%w: the park names %q, the holder's location is %q", ErrMalformed, scope.Location, c.Holder.Location) |
| 500 | } |
| 501 | return nil |
| 502 | } |
| 503 | |
| 504 | func isStagingLocation(location string) bool { return strings.HasPrefix(location, "moves/") } |
| 505 | |
| 506 | // queueReplicaPark appends a park-replica for the region unless this op |
| 507 | // already owes one of the same class (a lane or a location) there. It |
| 508 | // reports whether it appended. |
| 509 | func queueReplicaPark(c *Claim, opID, region string, scope ParkScope, newID func() string) (bool, error) { |
| 510 | for _, o := range c.Obligations { |
| 511 | if o.Op == opID && o.Kind == "park-replica" && regionname.SameRegion(c.Op.Intent["home_region"], o.Region, region) && o.Park != nil && isStagingLocation(o.Park.Location) == isStagingLocation(scope.Location) { |
| 512 | if !sameParkScope(*o.Park, scope) { |
| 513 | return false, fmt.Errorf("%w: %s already owes a park in %s with another scope", ErrWrongState, opID, region) |
| 514 | } |
| 515 | return false, nil |
| 516 | } |
| 517 | } |
| 518 | s := scope |
| 519 | c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-replica", Region: region, Retirement: newID(), Owner: "region:" + region, Park: &s}) |
| 520 | return true, nil |
| 521 | } |
| 522 | |
| 523 | // queueLocalPark appends a park-local for the box unless this op has one. |
| 524 | func queueLocalPark(c *Claim, opID, box string, newID func() string) { |
| 525 | for _, o := range c.Obligations { |
| 526 | if o.Op == opID && o.Kind == "park-local" && o.Box == box { |
| 527 | return |
| 528 | } |
| 529 | } |
| 530 | c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-local", Box: box, Retirement: newID(), Owner: "box:" + box}) |
| 531 | } |
| 532 | |
| 533 | // PhaseCommitted is the landing after the commit: the source region's whole |
| 534 | // location is retired from here (§6.4, §7.9). |
| 535 | const PhaseCommitted = "committed" |
| 536 | |
| 537 | // requireRetiredRegion is QueueLocationPark's lifecycle check. A whole |
| 538 | // location is parked only where the region has stopped being authoritative: |
| 539 | // - the source's region (any region but the move's target) once the op is |
| 540 | // landing/committed; |
| 541 | // - the target's region once the op is restoring and the target has |
| 542 | // released (released, or claimed by the source). |
| 543 | // |
| 544 | // The source before the commit and the target after it are live, and so is |
| 545 | // every other state, which is ErrWrongState. A box, when named, is the retiring |
| 546 | // side's box. The claim records the target region and both boxes at the start |
| 547 | // of the move (Intent), not the source's region, so a region that is neither |
| 548 | // is not told apart from the source here. |
| 549 | func requireRetiredRegion(c *Claim, region, box string) error { |
| 550 | to := c.Op.Intent["to"] |
| 551 | switch { |
| 552 | case c.Op.Kind == KindLanding && c.Op.Phase == PhaseCommitted: |
| 553 | if to != "" && region != to && (box == "" || box == c.Op.Intent["source"]) { |
| 554 | return nil |
| 555 | } |
| 556 | case c.Op.Kind == KindRestoring && (c.Op.Phase == PhaseReleased || c.Op.Phase == PhaseClaimed): |
| 557 | if to != "" && region == to && (box == "" || box == c.Op.Intent["target"]) { |
| 558 | return nil |
| 559 | } |
| 560 | } |
| 561 | return fmt.Errorf("%w: a location park in %s is not legal for %s/%q (the source's after the commit, a rolled-back target's after its release)", ErrWrongState, region, c.Op.Kind, c.Op.Phase) |
| 562 | } |
| 563 | |
| 564 | // QueueLocationPark queues the park of a whole location in a region where it |
| 565 | // is no longer authoritative: the source's after the commit, a rolled-back |
| 566 | // target's at cleaning (requireRetiredRegion; any other state is |
| 567 | // ErrWrongState with nothing queued). The scope is holder.location, exactly. |
| 568 | // With a box, the box's park-local is queued in the same write. Idempotent |
| 569 | // per (op, region) and (op, box). |
| 570 | func QueueLocationPark(opID, region, box string, scope ParkScope, newID func() string) func(*Claim) error { |
| 571 | return func(c *Claim) error { |
| 572 | var err error |
| 573 | if scope, err = parkQueueGuard(c, opID, region, scope, newID); err != nil { |
| 574 | return err |
| 575 | } |
| 576 | if err := requireHolderLocation(c, scope); err != nil { |
| 577 | return err |
| 578 | } |
| 579 | if err := requireRetiredRegion(c, region, box); err != nil { |
| 580 | return err |
| 581 | } |
| 582 | if _, err := queueReplicaPark(c, opID, region, scope, newID); err != nil { |
| 583 | return err |
| 584 | } |
| 585 | if box != "" { |
| 586 | queueLocalPark(c, opID, box, newID) |
| 587 | } |
| 588 | return nil |
| 589 | } |
| 590 | } |
| 591 | |
| 592 | // QueueStagingPark queues the park of this move's own staging lane in a |
| 593 | // region. The scope's location must be exactly moves/<host>/<op>: not the |
| 594 | // live location, not the host-wide staging prefix, not another move's lane. |
| 595 | // A lane park a revert already queued under the op is the same chore. |
| 596 | // Idempotent per (op, region). EvidenceAt is the recorded writers' bound |
| 597 | // (G7); the caller computes it. |
| 598 | func QueueStagingPark(opID, region string, scope ParkScope, newID func() string) func(*Claim) error { |
| 599 | return func(c *Claim) error { |
| 600 | var err error |
| 601 | if scope, err = parkQueueGuard(c, opID, region, scope, newID); err != nil { |
| 602 | return err |
| 603 | } |
| 604 | // Restates moves.ScopeLocation(host, op); moves imports hostclaim, so |
| 605 | // it cannot be called from here. Keep the two together. |
| 606 | if scope.Location != "moves/"+c.Host+"/"+opID { |
| 607 | return fmt.Errorf("%w: a staging park is this move's own lane, moves/%s/%s", ErrMalformed, c.Host, opID) |
| 608 | } |
| 609 | _, err = queueReplicaPark(c, opID, region, scope, newID) |
| 610 | return err |
| 611 | } |
| 612 | } |
| 613 | |
| 614 | // QueueLocalPark queues a park-local for the box alone, for the landing's |
| 615 | // local phase: park.Local runs only against a persisted park-local chore of |
| 616 | // the claim and treats an unknown id as already completed, so the chore must |
| 617 | // be in the claim before the call. Idempotent per (op, box). Queue it only |
| 618 | // when InventoryHasStamp says the box has paths of the earlier stay (G5). |
| 619 | func QueueLocalPark(opID, box string, newID func() string) func(*Claim) error { |
| 620 | return func(c *Claim) error { |
| 621 | if err := RequireOp(opID)(c); err != nil { |
| 622 | return err |
| 623 | } |
| 624 | if box == "" || newID == nil { |
| 625 | return fmt.Errorf("%w: a local park names its box and has a way to mint ids", ErrMalformed) |
| 626 | } |
| 627 | queueLocalPark(c, opID, box, newID) |
| 628 | return nil |
| 629 | } |
| 630 | } |
| 631 | |
| 632 | // InventoryHasStamp reports whether the box's inventory lists any path |
| 633 | // tagged with the stamp: the landing's earlier stay S' (G5). A first landing |
| 634 | // has none, and then no park-local is queued and none is retired. |
| 635 | func InventoryHasStamp(c *Claim, box string, stamp uint64) bool { |
| 636 | if c == nil || stamp == 0 { |
| 637 | return false |
| 638 | } |
| 639 | for _, s := range c.Inventory[box].Paths { |
| 640 | if s == stamp { |
| 641 | return true |
| 642 | } |
| 643 | } |
| 644 | return false |
| 645 | } |
| 646 | |
| 647 | // CompleteLocalPark is the replay of a local phase that finds the box's |
| 648 | // park-local chore and no paths of the earlier stamp left: there is nothing |
| 649 | // to retire, so the chore is completed under the op (CompleteObligation). |
| 650 | // With paths of that stamp still listed it refuses (ErrWrongState): the |
| 651 | // retirement has to run. No chore is a no-op. |
| 652 | func CompleteLocalPark(opID, box string, earlierStamp uint64) func(*Claim) error { |
| 653 | return func(c *Claim) error { |
| 654 | if err := RequireOp(opID)(c); err != nil { |
| 655 | return err |
| 656 | } |
| 657 | if box == "" { |
| 658 | return fmt.Errorf("%w: a local park names its box", ErrMalformed) |
| 659 | } |
| 660 | for _, o := range c.Obligations { |
| 661 | if o.Op == opID && o.Kind == "park-local" && o.Box == box { |
| 662 | if InventoryHasStamp(c, box, earlierStamp) { |
| 663 | return fmt.Errorf("%w: %s still lists paths of stamp %d", ErrWrongState, box, earlierStamp) |
| 664 | } |
| 665 | return CompleteObligation(opID, o.ID)(c) |
| 666 | } |
| 667 | } |
| 668 | return nil |
| 669 | } |
| 670 | } |
| 671 | |
| 672 | // Commit is the claim half of the record's commit (§7.5, §7.9 step 1): the |
| 673 | // landing goes to committed and, in the same write, the parks the commit owes |
| 674 | // are queued through the Plan 2 helpers: the source region's whole location |
| 675 | // (QueueLocationPark, which holds the lifecycle and exact-location checks), |
| 676 | // the source box's park-local, and the target region's staging lane |
| 677 | // (QueueStagingPark). The source's park-local is queued only when the source |
| 678 | // box lists paths of the journalled source stamp S (InventoryHasStamp, H5b); |
| 679 | // otherwise none is. CoverRegionPredecessors then covers the source region's |
| 680 | // admission records. holder.stamp, session and generation are not touched |
| 681 | // (R10-3, H3). |
| 682 | // |
| 683 | // It follows probed only (platform-1e m-2). Idempotent under the op: a |
| 684 | // landing already committed returns nil and queues nothing. A refused commit |
| 685 | // leaves the claim as it found it. |
| 686 | func Commit(opID, sourceRegion, sourceBox, targetRegion string, sourceScope, stagingScope ParkScope, newID func() string) func(*Claim) error { |
| 687 | return func(c *Claim) error { |
| 688 | if err := RequireOp(opID)(c); err != nil { |
| 689 | return err |
| 690 | } |
| 691 | if c.Op.Kind != KindLanding { |
| 692 | return fmt.Errorf("%w: kind %q", ErrWrongState, c.Op.Kind) |
| 693 | } |
| 694 | if c.Op.Phase == PhaseCommitted { |
| 695 | return nil |
| 696 | } |
| 697 | // The commit follows probed only (platform-1e m-2): with Group D's |
| 698 | // adjacent-step rule (probing, probed, committed) the admitted |
| 699 | // session survives it. |
| 700 | if c.Op.Phase != PhaseLandingProbed { |
| 701 | return fmt.Errorf("%w: phase %q", ErrWrongState, c.Op.Phase) |
| 702 | } |
| 703 | if len(c.Op.Participants) != 2 || c.Op.Exec != c.Op.Participants[1] { |
| 704 | return fmt.Errorf("%w: exec %q is not the target", ErrWrongState, c.Op.Exec) |
| 705 | } |
| 706 | if sourceBox != c.Op.Participants[0] { |
| 707 | return fmt.Errorf("%w: %q is not the move's source", ErrWrongState, sourceBox) |
| 708 | } |
| 709 | parkBox := "" |
| 710 | if stamp, err := strconv.ParseUint(c.Op.Journal[JournalSourceStamp], 10, 64); err == nil && InventoryHasStamp(c, sourceBox, stamp) { |
| 711 | parkBox = sourceBox |
| 712 | } |
| 713 | phase, obligations := c.Op.Phase, len(c.Obligations) |
| 714 | c.Op.Phase = PhaseCommitted |
| 715 | for _, step := range []func(*Claim) error{ |
| 716 | QueueLocationPark(opID, sourceRegion, parkBox, sourceScope, newID), |
| 717 | QueueStagingPark(opID, targetRegion, stagingScope, newID), |
| 718 | CoverRegionPredecessors(opID, PhaseCommitted, sourceRegion), |
| 719 | } { |
| 720 | if err := step(c); err != nil { |
| 721 | c.Op.Phase, c.Obligations = phase, c.Obligations[:obligations] |
| 722 | return err |
| 723 | } |
| 724 | } |
| 725 | return nil |
| 726 | } |
| 727 | } |
| 728 | |
| 729 | // QueueResidueParks queues one park-generation per listed generation for a |
| 730 | // landing's earlier-stay residue in the target region: RootGeneration for |
| 731 | // the origin, the generation name for the rest, all scoped to the same |
| 732 | // location. Deduplicated by (op, region, generation); a generation already |
| 733 | // queued with another scope is refused, and nothing is queued then. |
| 734 | func QueueResidueParks(opID, region string, location ParkScope, generations []string, newID func() string) func(*Claim) error { |
| 735 | return func(c *Claim) error { |
| 736 | var err error |
| 737 | if location, err = parkQueueGuard(c, opID, region, location, newID); err != nil { |
| 738 | return err |
| 739 | } |
| 740 | if err := requireHolderLocation(c, location); err != nil { |
| 741 | return err |
| 742 | } |
| 743 | for _, g := range generations { |
| 744 | if g == "" { |
| 745 | return fmt.Errorf("%w: a residue generation has a name", ErrMalformed) |
| 746 | } |
| 747 | } |
| 748 | queued := map[string]bool{} |
| 749 | for _, o := range c.Obligations { |
| 750 | if o.Op == opID && o.Kind == "park-generation" && regionname.SameRegion(c.Op.Intent["home_region"], o.Region, region) { |
| 751 | if o.Park == nil || !sameParkScope(*o.Park, location) { |
| 752 | for _, g := range generations { |
| 753 | if g == o.Generation { |
| 754 | return fmt.Errorf("%w: %s already owes generation %q in %s with another scope", ErrWrongState, opID, g, region) |
| 755 | } |
| 756 | } |
| 757 | } |
| 758 | queued[o.Generation] = true |
| 759 | } |
| 760 | } |
| 761 | for _, g := range generations { |
| 762 | if queued[g] { |
| 763 | continue |
| 764 | } |
| 765 | queued[g] = true |
| 766 | s := location |
| 767 | c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-generation", Region: region, Generation: g, Retirement: newID(), Owner: "region:" + region, Park: &s}) |
| 768 | } |
| 769 | return nil |
| 770 | } |
| 771 | } |
| 772 | |
| 773 | // Recovery phases on the claim's op (spec §7.4, §7.8, §7.14; J1). The record |
| 774 | // carries the detailed recovery journal; the claim mirrors the phases the |
| 775 | // wake policy and the release key on, as the landing mirrors its own. |
| 776 | // |
| 777 | // restoring (a rollback, after the flip): the target's releasing and |
| 778 | // released, the source's claimed, then probing (written before the source's |
| 779 | // first recovery attempt), probed and opened. |
| 780 | // |
| 781 | // reverting (before the flip): op.phase stays the moving phase the revert |
| 782 | // began from ("" or "sealed") through rev_phase evidence and verify, then |
| 783 | // mirrors probing, probed and opened in the CAS that writes rev_phase. |
| 784 | const ( |
| 785 | PhaseReleasing = "releasing" // restoring: the target owes its stop evidence |
| 786 | PhaseReleased = "released" // restoring: the target gave execution up |
| 787 | PhaseClaimed = "claimed" // restoring: the source took execution back |
| 788 | PhaseProbing = "probing" // restoring, reverting: the recovery probe |
| 789 | PhaseProbed = "probed" // restoring, reverting: the probe passed |
| 790 | PhaseOpened = "opened" // restoring, reverting: the source serves again |
| 791 | |
| 792 | RevPhaseEvidence = "evidence" // rev_phase: stop evidence before anything starts |
| 793 | RevPhaseVerify = "verify" // rev_phase: files and timers against the stopped digests |
| 794 | ) |
| 795 | |
| 796 | // JournalTargetStamp is the target installation's stamp F, journalled at the |
| 797 | // rollback for its release (J15): the target's park-local is queued only |
| 798 | // when its inventory lists paths of F. |
| 799 | const JournalTargetStamp = "target_stamp" |
| 800 | |
| 801 | // journalReturnLane marks the return lane's park as queued, so a replay after |
| 802 | // the park engine has completed it does not queue it again. |
| 803 | const journalReturnLane = "return_lane" |
| 804 | |
| 805 | // RecoveryPhaseOrder is the op.phase order a recovery kind writes, or nil for |
| 806 | // any other kind. A revert's phases before probing are the moving phase it |
| 807 | // began from and are not listed. |
| 808 | func RecoveryPhaseOrder(kind string) []string { |
| 809 | switch kind { |
| 810 | case KindRestoring: |
| 811 | return []string{PhaseReleasing, PhaseReleased, PhaseClaimed, PhaseProbing, PhaseProbed, PhaseOpened} |
| 812 | case KindReverting: |
| 813 | return []string{PhaseProbing, PhaseProbed, PhaseOpened} |
| 814 | } |
| 815 | return nil |
| 816 | } |
| 817 | |
| 818 | // rollbackIDShape restates moves' rb_ op id shape (moves imports hostclaim, so |
| 819 | // it cannot be called from here). TestRollBackTakesMintedRollbackIDs holds |
| 820 | // the two together. |
| 821 | var rollbackIDShape = regexp.MustCompile(`^rb_[a-z2-7]{26}$`) |
| 822 | |
| 823 | // inOneWrite runs steps in order as one transition: if any refuses, the claim |
| 824 | // is put back exactly as it was found. |
| 825 | func inOneWrite(c *Claim, steps ...func(*Claim) error) error { |
| 826 | saved, err := cloneClaim(c) |
| 827 | if err != nil { |
| 828 | return err |
| 829 | } |
| 830 | for _, step := range steps { |
| 831 | if err := step(c); err != nil { |
| 832 | *c = *saved |
| 833 | return err |
| 834 | } |
| 835 | } |
| 836 | return nil |
| 837 | } |
| 838 | |
| 839 | // RollBack is the claim half of the record's rollback CAS (§7.5 rolling_back, |
| 840 | // §7.8): landing, at any phase before the commit, to restoring/releasing |
| 841 | // under the same op id, with the rollback id in the intent. Execution stays |
| 842 | // with the target, which still owes its stop evidence; the holder is not |
| 843 | // touched. The target installation's stamp F (holder.stamp now) is |
| 844 | // journalled for TargetReleased (J15). |
| 845 | // |
| 846 | // The commit and the rollback race on the claim as on the record: a committed |
| 847 | // landing refuses the rollback, and Commit refuses a restoring op. A replay |
| 848 | // that finds this rollback already written (any restoring phase) is a no-op; |
| 849 | // another rollback id is refused. |
| 850 | func RollBack(opID, rbID string) func(*Claim) error { |
| 851 | return func(c *Claim) error { |
| 852 | if err := RequireOp(opID)(c); err != nil { |
| 853 | return err |
| 854 | } |
| 855 | if !rollbackIDShape.MatchString(rbID) { |
| 856 | return fmt.Errorf("%w: %q is not a rollback id", ErrMalformed, rbID) |
| 857 | } |
| 858 | if c.Op.Kind == KindRestoring { |
| 859 | if c.Op.Intent["rollback"] == rbID { |
| 860 | return nil |
| 861 | } |
| 862 | return fmt.Errorf("%w: the op is restoring under rollback %q", ErrWrongState, c.Op.Intent["rollback"]) |
| 863 | } |
| 864 | if c.Op.Kind != KindLanding || c.Op.Phase == PhaseCommitted || c.Holder == nil { |
| 865 | return fmt.Errorf("%w: op is %s/%q, a rollback needs a landing before the commit", ErrWrongState, c.Op.Kind, c.Op.Phase) |
| 866 | } |
| 867 | intent := make(map[string]string, len(c.Op.Intent)+1) |
| 868 | for k, v := range c.Op.Intent { |
| 869 | intent[k] = v |
| 870 | } |
| 871 | intent["rollback"] = rbID |
| 872 | c.Op.Kind, c.Op.Phase, c.Op.Intent = KindRestoring, PhaseReleasing, intent |
| 873 | setJournal(c, JournalTargetStamp, strconv.FormatUint(c.Holder.Stamp, 10)) |
| 874 | return nil |
| 875 | } |
| 876 | } |
| 877 | |
| 878 | // RollBackFlipped is the rollback's claim writer for a record already at |
| 879 | // landing (J16): when the claim is still moving/sealed (the flip's claim half |
| 880 | // not yet written, G12) it writes Flip first, then RollBack, in one write; |
| 881 | // over any other kind it is RollBack. A refused flip leaves the claim as found. |
| 882 | func RollBackFlipped(opID, rbID, targetBox string, sourceStamp uint64, srcGen string) func(*Claim) error { |
| 883 | return func(c *Claim) error { |
| 884 | if err := RequireOp(opID)(c); err != nil { |
| 885 | return err |
| 886 | } |
| 887 | if !rollbackIDShape.MatchString(rbID) { |
| 888 | return fmt.Errorf("%w: %q is not a rollback id", ErrMalformed, rbID) |
| 889 | } |
| 890 | if c.Op.Kind != KindMoving { |
| 891 | return RollBack(opID, rbID)(c) |
| 892 | } |
| 893 | return inOneWrite(c, Flip(opID, targetBox, sourceStamp, srcGen), RollBack(opID, rbID)) |
| 894 | } |
| 895 | } |
| 896 | |
| 897 | // TargetReleased is the target's released CAS (§7.8): restoring/releasing, |
| 898 | // with the target executing, gives execution up citing its stop evidence |
| 899 | // (ExecToNone, evidence_target), moves to released and, in the same write, |
| 900 | // queues the park of the target region's whole location (QueueLocationPark, |
| 901 | // D6) and covers every admission record whose predecessor is in the target |
| 902 | // region (CoverRegionPredecessors, claim rev 6.14 item 6, R12-3). The |
| 903 | // target's park-local is queued only when its inventory lists paths of the |
| 904 | // stamp journalled at the rollback (J15); otherwise none is, and the |
| 905 | // target's cleaning parks its unregistered leftovers itself. |
| 906 | // |
| 907 | // A replay that finds the release written with the same evidence (released |
| 908 | // or any later phase) is a no-op and queues nothing, even after a chore has |
| 909 | // completed. A refused release leaves the claim as found. |
| 910 | const JournalTargetSession = "target_session" |
| 911 | const JournalTargetGeneration = "target_generation" |
| 912 | |
| 913 | func TargetReleased(opID, targetBox, targetRegion, evidenceKey string, scope ParkScope, newID func() string) func(*Claim) error { |
| 914 | return func(c *Claim) error { |
| 915 | if err := RequireOp(opID)(c); err != nil { |
| 916 | return err |
| 917 | } |
| 918 | if c.Op.Kind != KindRestoring { |
| 919 | return fmt.Errorf("%w: op is %s/%q, a release needs a rollback", ErrWrongState, c.Op.Kind, c.Op.Phase) |
| 920 | } |
| 921 | if targetBox == "" || targetRegion == "" || evidenceKey == "" || newID == nil { |
| 922 | return fmt.Errorf("%w: a release names the target, its region and its evidence", ErrMalformed) |
| 923 | } |
| 924 | if c.Op.Phase != PhaseReleasing { |
| 925 | switch c.Op.Phase { |
| 926 | case PhaseReleased, PhaseClaimed, PhaseProbing, PhaseProbed, PhaseOpened: |
| 927 | if c.Op.Journal[JournalEvidenceTarget] == evidenceKey { |
| 928 | return nil |
| 929 | } |
| 930 | } |
| 931 | return fmt.Errorf("%w: op is %s/%q, not releasing with this evidence", ErrWrongState, c.Op.Kind, c.Op.Phase) |
| 932 | } |
| 933 | if len(c.Op.Participants) != 2 || c.Op.Participants[1] != targetBox || c.Op.Intent["target"] != targetBox || c.Op.Exec != targetBox { |
| 934 | return fmt.Errorf("%w: %q is not the executing target", ErrWrongState, targetBox) |
| 935 | } |
| 936 | if c.Op.Intent["to"] != targetRegion { |
| 937 | return fmt.Errorf("%w: %q is not the move's target region", ErrWrongState, targetRegion) |
| 938 | } |
| 939 | parkBox := "" |
| 940 | if stamp, err := strconv.ParseUint(c.Op.Journal[JournalTargetStamp], 10, 64); err == nil && InventoryHasStamp(c, targetBox, stamp) { |
| 941 | parkBox = targetBox |
| 942 | } |
| 943 | return inOneWrite(c, |
| 944 | func(c *Claim) error { |
| 945 | if c.Holder != nil && c.Holder.WriterBox == targetBox { |
| 946 | setJournal(c, JournalTargetSession, strconv.FormatUint(c.Holder.Session, 10)) |
| 947 | setJournal(c, JournalTargetGeneration, c.Holder.Generation) |
| 948 | } |
| 949 | return nil |
| 950 | }, |
| 951 | ExecToNone(opID, targetBox, evidenceKey, JournalEvidenceTarget), |
| 952 | func(c *Claim) error { c.Op.Phase = PhaseReleased; return nil }, |
| 953 | QueueLocationPark(opID, targetRegion, parkBox, scope, newID), |
| 954 | CoverRegionPredecessors(opID, PhaseReleased, targetRegion), |
| 955 | ) |
| 956 | } |
| 957 | } |
| 958 | |
| 959 | // ReturnFence is a termination fence for a move's target that the caller |
| 960 | // verified before Update (VerifyReturnFence, J5): the op, the box and the |
| 961 | // exact fence.<box> journal entry it verified. The zero value vouches for |
| 962 | // nothing. |
| 963 | type ReturnFence struct{ op, box, ref, observedAt string } |
| 964 | |
| 965 | // VerifyReturnFence re-reads and verifies the fence the op journals for box |
| 966 | // (VerifyOperationFence) and returns what ReturnToSource checks inside its |
| 967 | // CAS. Run it on the observation that Update will write over. |
| 968 | func VerifyReturnFence(ctx context.Context, s release.Store, authority FenceAuthority, op Op, box string) (ReturnFence, error) { |
| 969 | if err := VerifyOperationFence(ctx, s, authority, op, box); err != nil { |
| 970 | return ReturnFence{}, err |
| 971 | } |
| 972 | return ReturnFence{op: op.ID, box: box, ref: op.Journal["fence."+box]}, nil |
| 973 | } |
| 974 | |
| 975 | // targetGaveUp is ReturnToSource's authority: the target's own stop evidence |
| 976 | // (evidence_target); or an operator's declaration under this op (a loss.<id> |
| 977 | // journal entry the claim records in loss_refs) that took the target out of |
| 978 | // the participants; or a verified termination fence whose journal entry the |
| 979 | // claim still carries. |
| 980 | func targetGaveUp(c *Claim, opID string, fence ReturnFence) bool { |
| 981 | target := c.Op.Intent["target"] |
| 982 | if target == "" { |
| 983 | return false |
| 984 | } |
| 985 | if c.Op.Journal[JournalEvidenceTarget] != "" { |
| 986 | return true |
| 987 | } |
| 988 | if !stringPresent(c.Op.Participants, target) { |
| 989 | if _, consumed := ConsumedMoveTargetFence(c); consumed { |
| 990 | return true |
| 991 | } |
| 992 | for _, id := range c.LossRefs { |
| 993 | if _, ok := c.Op.Journal["loss."+id]; ok { |
| 994 | return true |
| 995 | } |
| 996 | } |
| 997 | } |
| 998 | return fence.op == opID && fence.box == target && fence.ref != "" && c.Op.Journal["fence."+target] == fence.ref |
| 999 | } |
| 1000 | |
| 1001 | // ReturnToSource is the source's claimed CAS (§7.8, K4): restoring/released |
| 1002 | // with nobody executing ("" after the target's evidence, "none" after a |
| 1003 | // declaration) and the target's authority given up (targetGaveUp) makes the |
| 1004 | // source, the move's first participant, the executor again at claimed. |
| 1005 | // holder.stamp is the source installation's stamp S the flip journalled; a |
| 1006 | // clean return also names the source's generation (src_gen) again. The |
| 1007 | // admissions are not touched (R11-4): the source's re-admission names the |
| 1008 | // target's writer, their last record, as its predecessor. |
| 1009 | // |
| 1010 | // A declared-lost target is not released by TargetReleased: the resolve door |
| 1011 | // steps the claim to released and queues the target region's park itself |
| 1012 | // (J16), so this transition is the same for every form. A replay at claimed |
| 1013 | // with the source executing is a no-op. |
| 1014 | func ReturnToSource(opID, sourceBox string, clean bool, fence ReturnFence) func(*Claim) error { |
| 1015 | return func(c *Claim) error { |
| 1016 | if err := RequireOp(opID)(c); err != nil { |
| 1017 | return err |
| 1018 | } |
| 1019 | if sourceBox == "" { |
| 1020 | return fmt.Errorf("%w: a return names the source", ErrMalformed) |
| 1021 | } |
| 1022 | if c.Op.Kind != KindRestoring || c.Holder == nil { |
| 1023 | return fmt.Errorf("%w: op is %s/%q, a return needs a rollback", ErrWrongState, c.Op.Kind, c.Op.Phase) |
| 1024 | } |
| 1025 | if c.Op.Phase == PhaseClaimed && c.Op.Exec == sourceBox { |
| 1026 | return nil |
| 1027 | } |
| 1028 | if c.Op.Phase != PhaseReleased || (c.Op.Exec != "" && c.Op.Exec != "none") { |
| 1029 | return fmt.Errorf("%w: op is %s/%q with exec %q, a return needs the target released", ErrWrongState, c.Op.Kind, c.Op.Phase, c.Op.Exec) |
| 1030 | } |
| 1031 | if len(c.Op.Participants) == 0 || c.Op.Participants[0] != sourceBox || c.Op.Intent["source"] != sourceBox { |
| 1032 | return fmt.Errorf("%w: %q is not the move's source", ErrWrongState, sourceBox) |
| 1033 | } |
| 1034 | if !targetGaveUp(c, opID, fence) { |
| 1035 | return fmt.Errorf("%w: no evidence, declaration or verified fence for the target", ErrWrongState) |
| 1036 | } |
| 1037 | stamp, err := strconv.ParseUint(c.Op.Journal[JournalSourceStamp], 10, 64) |
| 1038 | srcGen, flipped := c.Op.Journal[JournalSrcGen] |
| 1039 | if err != nil || stamp == 0 || !flipped { |
| 1040 | return fmt.Errorf("%w: the flip's source stamp and generation are not journalled", ErrWrongState) |
| 1041 | } |
| 1042 | c.Op.Exec, c.Op.Phase = sourceBox, PhaseClaimed |
| 1043 | c.Holder.Stamp = stamp |
| 1044 | if clean { |
| 1045 | c.Holder.Generation = srcGen |
| 1046 | } |
| 1047 | return nil |
| 1048 | } |
| 1049 | } |
| 1050 | |
| 1051 | // SetRestoringPhase steps a rollback's source recovery on the claim (J1): |
| 1052 | // claimed, probing, probed, opened, one step at a time, with the source |
| 1053 | // executing. releasing to released is TargetReleased's and released to |
| 1054 | // claimed is ReturnToSource's. A replay finds the phase moved and is refused. |
| 1055 | func SetRestoringPhase(opID, from, to string) func(*Claim) error { |
| 1056 | return func(c *Claim) error { |
| 1057 | if err := RequireMoveOp(opID, KindRestoring, from)(c); err != nil { |
| 1058 | return err |
| 1059 | } |
| 1060 | step := from + ">" + to |
| 1061 | if step != PhaseClaimed+">"+PhaseProbing && step != PhaseProbing+">"+PhaseProbed && step != PhaseProbed+">"+PhaseOpened { |
| 1062 | return fmt.Errorf("%w: restoring does not step %q to %q", ErrWrongState, from, to) |
| 1063 | } |
| 1064 | if len(c.Op.Participants) == 0 || c.Op.Exec != c.Op.Participants[0] { |
| 1065 | return fmt.Errorf("%w: exec %q is not the source", ErrWrongState, c.Op.Exec) |
| 1066 | } |
| 1067 | c.Op.Phase = to |
| 1068 | return nil |
| 1069 | } |
| 1070 | } |
| 1071 | |
| 1072 | // SetRevPhase steps a revert's journalled rev_phase (§7.4): evidence, verify, |
| 1073 | // probing, probed, opened. verify needs the files baseline the record keeps |
| 1074 | // from stopped (sourceBaseline); a revert from stopping has none and goes from |
| 1075 | // evidence straight to probing (J8). From probing on, op.phase is written in |
| 1076 | // the same CAS (J1); before it, op.phase is the moving phase the revert began |
| 1077 | // from. The source must be executing. A replay finds the phase moved and is |
| 1078 | // refused. |
| 1079 | func SetRevPhase(opID, from, to string, sourceBaseline bool) func(*Claim) error { |
| 1080 | return func(c *Claim) error { |
| 1081 | if err := RequireOp(opID)(c); err != nil { |
| 1082 | return err |
| 1083 | } |
| 1084 | if c.Op.Kind != KindReverting || c.Op.Journal[JournalRevPhase] != from { |
| 1085 | return fmt.Errorf("%w: op is %s with rev_phase %q, the step expects reverting at %q", ErrWrongState, c.Op.Kind, c.Op.Journal[JournalRevPhase], from) |
| 1086 | } |
| 1087 | if len(c.Op.Participants) == 0 || c.Op.Exec != c.Op.Participants[0] { |
| 1088 | return fmt.Errorf("%w: exec %q is not the source", ErrWrongState, c.Op.Exec) |
| 1089 | } |
| 1090 | legal := false |
| 1091 | switch from + ">" + to { |
| 1092 | case RevPhaseEvidence + ">" + RevPhaseVerify: |
| 1093 | legal = sourceBaseline |
| 1094 | case RevPhaseEvidence + ">" + PhaseProbing: |
| 1095 | legal = !sourceBaseline |
| 1096 | case RevPhaseVerify + ">" + PhaseProbing, PhaseProbing + ">" + PhaseProbed, PhaseProbed + ">" + PhaseOpened: |
| 1097 | legal = true |
| 1098 | } |
| 1099 | if !legal { |
| 1100 | return fmt.Errorf("%w: rev_phase does not step %q to %q (baseline %v)", ErrWrongState, from, to, sourceBaseline) |
| 1101 | } |
| 1102 | mirrored := c.Op.Phase == from |
| 1103 | if from == RevPhaseEvidence || from == RevPhaseVerify { |
| 1104 | mirrored = c.Op.Phase == PhaseMoving || c.Op.Phase == PhaseSealed |
| 1105 | } |
| 1106 | if !mirrored { |
| 1107 | return fmt.Errorf("%w: op.phase %q disagrees with rev_phase %q", ErrWrongState, c.Op.Phase, from) |
| 1108 | } |
| 1109 | setJournal(c, JournalRevPhase, to) |
| 1110 | if to != RevPhaseVerify { |
| 1111 | c.Op.Phase = to |
| 1112 | } |
| 1113 | return nil |
| 1114 | } |
| 1115 | } |
| 1116 | |
| 1117 | // QueueReturnLanePark queues the park of a dirty rollback's return lane, |
| 1118 | // moves/<host>/<rollback id>, in the source's region (§7.8 probed): the |
| 1119 | // rollback's own lane, never the move's, the live location or the host-wide |
| 1120 | // staging prefix. It is legal at restoring/probed, composed into the probed |
| 1121 | // CAS. Idempotent: a chore already owed is not queued twice, and once the |
| 1122 | // park engine has completed it a replay does not queue it again. |
| 1123 | func QueueReturnLanePark(opID, sourceRegion string, scope ParkScope, newID func() string) func(*Claim) error { |
| 1124 | return func(c *Claim) error { |
| 1125 | var err error |
| 1126 | if scope, err = parkQueueGuard(c, opID, sourceRegion, scope, newID); err != nil { |
| 1127 | return err |
| 1128 | } |
| 1129 | rb := c.Op.Intent["rollback"] |
| 1130 | if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseProbed || !rollbackIDShape.MatchString(rb) { |
| 1131 | return fmt.Errorf("%w: op is %s/%q, the return lane is parked at a rollback's probed", ErrWrongState, c.Op.Kind, c.Op.Phase) |
| 1132 | } |
| 1133 | if sourceRegion == c.Op.Intent["to"] { |
| 1134 | return fmt.Errorf("%w: the return lane is in the source's region, not %s", ErrWrongState, sourceRegion) |
| 1135 | } |
| 1136 | if scope.Location != "moves/"+c.Host+"/"+rb { |
| 1137 | return fmt.Errorf("%w: the return lane is this rollback's own lane, moves/%s/%s", ErrMalformed, c.Host, rb) |
| 1138 | } |
| 1139 | owed := false |
| 1140 | for _, o := range c.Obligations { |
| 1141 | if o.Op == opID && o.Kind == "park-replica" && regionname.SameRegion(c.Op.Intent["home_region"], o.Region, sourceRegion) && o.Park != nil && isStagingLocation(o.Park.Location) { |
| 1142 | owed = true |
| 1143 | } |
| 1144 | } |
| 1145 | if !owed && c.Op.Journal[journalReturnLane] == scope.Location { |
| 1146 | return nil |
| 1147 | } |
| 1148 | if _, err := queueReplicaPark(c, opID, sourceRegion, scope, newID); err != nil { |
| 1149 | return err |
| 1150 | } |
| 1151 | setJournal(c, journalReturnLane, scope.Location) |
| 1152 | return nil |
| 1153 | } |
| 1154 | } |
| 1155 | |