| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "strconv" |
| 9 | "sync" |
| 10 | "sync/atomic" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostclaim" |
| 13 | "github.com/carlosframework/platform/internal/moves" |
| 14 | "github.com/carlosframework/platform/internal/park" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | ) |
| 18 | |
| 19 | // MoveRecoveryGrant is one callback's authority over an exact move and record. |
| 20 | // It exposes no store or arbitrary transition. Mutations consume the observation; |
| 21 | // another action requires another pass. The grant expires when the hook returns. |
| 22 | // Record-first owners close before releasing the claim; LastMove repairs a |
| 23 | // crash between those writes on the next pass. |
| 24 | type MoveRecoveryGrant struct { |
| 25 | state *moveRecoveryGrantState |
| 26 | w *ControlWriter |
| 27 | o ConsoleMoveObservation |
| 28 | authority hostclaim.FenceAuthority |
| 29 | scope *moveRecoveryPassScope |
| 30 | // wrote is what the grant's record write did (Transition), once it has |
| 31 | // made one. |
| 32 | wrote *moves.Transition |
| 33 | } |
| 34 | |
| 35 | type moveRecoveryGrantState struct { |
| 36 | mu sync.Mutex |
| 37 | active bool |
| 38 | } |
| 39 | |
| 40 | func moveRecoveryKind(kind string) bool { |
| 41 | switch kind { |
| 42 | case hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindRelocating: |
| 43 | return true |
| 44 | } |
| 45 | return false |
| 46 | } |
| 47 | |
| 48 | type moveRecoveryPassKey struct{} |
| 49 | type moveRecoveryPassScope struct{ live atomic.Bool } |
| 50 | |
| 51 | func recoveryWriter(ctx context.Context) *ControlWriter { |
| 52 | scope, _ := ctx.Value(moveRecoveryPassKey{}).(*moveRecoveryPassScope) |
| 53 | if scope == nil || !scope.live.Load() { |
| 54 | return nil |
| 55 | } |
| 56 | grants, _ := ctx.Value(lifecycleGrantKey{}).([]lifecycleGrant) |
| 57 | for _, g := range grants { |
| 58 | if g.kind == "recovery" && g.w != nil { |
| 59 | return g.w |
| 60 | } |
| 61 | } |
| 62 | return nil |
| 63 | } |
| 64 | |
| 65 | func runMoveRecovery(ctx context.Context, obs hostclaim.Observed, hook func(context.Context, *MoveRecoveryGrant) error, authorities ...hostclaim.FenceAuthority) error { |
| 66 | w := recoveryWriter(ctx) |
| 67 | if w == nil || hook == nil { |
| 68 | return ErrContract |
| 69 | } |
| 70 | if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || !moveRecoveryKind(obs.Claim.Op.Kind) { |
| 71 | return ErrIdentity |
| 72 | } |
| 73 | h, c := obs.Claim.Holder, obs.Claim |
| 74 | if !operationHomeMatches(w.homeRegion, c) { |
| 75 | return ErrIdentity |
| 76 | } |
| 77 | if err := moveAdmission(ctx, w.raw, obs.Host, c); err != nil { |
| 78 | return err |
| 79 | } |
| 80 | r, err := ReadRecord(ctx, w.raw, h.Key) |
| 81 | if err != nil { |
| 82 | return err |
| 83 | } |
| 84 | if !matches(r, h) { |
| 85 | return ErrIdentity |
| 86 | } |
| 87 | _, seq, err := recordSequence(r.Body) |
| 88 | if err != nil { |
| 89 | return err |
| 90 | } |
| 91 | scope, _ := ctx.Value(moveRecoveryPassKey{}).(*moveRecoveryPassScope) |
| 92 | var authority hostclaim.FenceAuthority |
| 93 | if len(authorities) == 1 { |
| 94 | authority = authorities[0] |
| 95 | } |
| 96 | g := &MoveRecoveryGrant{state: &moveRecoveryGrantState{active: true}, w: w, scope: scope, authority: authority, o: ConsoleMoveObservation{Account: h.Account, App: h.App, AppID: c.Op.Intent["app_id"], Host: obs.Host, Incarnation: h.Incarnation, OpID: c.Op.ID, ClaimETag: obs.ETag, RecordETag: r.ETag, RecordSeq: seq}} |
| 97 | state := g.state |
| 98 | defer func() { state.mu.Lock(); state.active = false; state.mu.Unlock() }() |
| 99 | // The owner receives only this capability, never the minting context. |
| 100 | ctx = context.WithValue(ctx, lifecycleGrantKey{}, []lifecycleGrant(nil)) |
| 101 | ctx = context.WithValue(ctx, moveRecoveryPassKey{}, (*moveRecoveryPassScope)(nil)) |
| 102 | return hook(ctx, g) |
| 103 | } |
| 104 | |
| 105 | func (g *MoveRecoveryGrant) current(ctx context.Context) (hostclaim.Observed, Record, provision.Record, error) { |
| 106 | if g == nil || g.state == nil || !g.state.active || g.w == nil || g.scope == nil || !g.scope.live.Load() { |
| 107 | return hostclaim.Observed{}, Record{}, provision.Record{}, ErrPending |
| 108 | } |
| 109 | o, err := g.w.consoleMoveClaim(ctx, g.o) |
| 110 | if err != nil { |
| 111 | return o, Record{}, provision.Record{}, err |
| 112 | } |
| 113 | r, err := ReadRecord(ctx, g.w.raw, o.Claim.Holder.Key) |
| 114 | if err != nil { |
| 115 | return o, r, provision.Record{}, err |
| 116 | } |
| 117 | _, seq, err := recordSequence(r.Body) |
| 118 | if err != nil { |
| 119 | return o, r, provision.Record{}, err |
| 120 | } |
| 121 | if r.ETag != g.o.RecordETag || seq != g.o.RecordSeq { |
| 122 | return o, r, provision.Record{}, release.ErrCASConflict |
| 123 | } |
| 124 | if !matches(r, o.Claim.Holder) { |
| 125 | return o, r, provision.Record{}, ErrIdentity |
| 126 | } |
| 127 | rec, err := provision.DecodeRecord(r.Body) |
| 128 | return o, r, rec, err |
| 129 | } |
| 130 | |
| 131 | // Transition is what the grant's record write did, named as the move log |
| 132 | // names it (moves.TransitionOf over the record before and after the write), |
| 133 | // once Record has written. It is a value, never authority: the caller's log |
| 134 | // entry is built from the write's own evidence, not from a later read. |
| 135 | func (g *MoveRecoveryGrant) Transition() (moves.Transition, bool) { |
| 136 | if g == nil || g.wrote == nil { |
| 137 | return moves.Transition{}, false |
| 138 | } |
| 139 | return *g.wrote, true |
| 140 | } |
| 141 | |
| 142 | // Observation contains values, never authority. |
| 143 | func (g *MoveRecoveryGrant) Observation() ConsoleMoveObservation { return g.o } |
| 144 | |
| 145 | type MoveRecoveryRecordAction string |
| 146 | |
| 147 | const ( |
| 148 | RecoveryMoveClose MoveRecoveryRecordAction = "close" |
| 149 | RecoveryMoveRolledBackClose MoveRecoveryRecordAction = "rolled-back-close" |
| 150 | RecoveryMoveRevertClose MoveRecoveryRecordAction = "revert-close" |
| 151 | RecoveryMoveDeadlineRollback MoveRecoveryRecordAction = "deadline-rollback" |
| 152 | ) |
| 153 | |
| 154 | func (g *MoveRecoveryGrant) Record(ctx context.Context, action MoveRecoveryRecordAction) error { |
| 155 | if g == nil || g.state == nil { |
| 156 | return ErrPending |
| 157 | } |
| 158 | g.state.mu.Lock() |
| 159 | defer g.state.mu.Unlock() |
| 160 | o, r, rec, err := g.current(ctx) |
| 161 | if err != nil { |
| 162 | return err |
| 163 | } |
| 164 | guard := provision.MoveGuard{ID: g.o.OpID, Incarnation: g.o.Incarnation} |
| 165 | if rec.Move == nil || rec.Move.ID != guard.ID || rec.Move.Incarnation != guard.Incarnation { |
| 166 | return ErrIdentity |
| 167 | } |
| 168 | switch action { |
| 169 | case RecoveryMoveClose, RecoveryMoveRolledBackClose, RecoveryMoveRevertClose: |
| 170 | for _, chore := range o.Claim.Obligations { |
| 171 | if chore.Op == g.o.OpID { |
| 172 | return ErrPending |
| 173 | } |
| 174 | } |
| 175 | } |
| 176 | now := g.w.now().UTC() |
| 177 | var mutate func(*provision.Record) error |
| 178 | switch action { |
| 179 | case RecoveryMoveClose: |
| 180 | if o.Claim.Op.Kind != hostclaim.KindLanding || o.Claim.Op.Phase != hostclaim.PhaseCommitted { |
| 181 | return ErrPending |
| 182 | } |
| 183 | mutate = provision.MoveClose(guard, now) |
| 184 | case RecoveryMoveRolledBackClose: |
| 185 | if o.Claim.Op.Kind != hostclaim.KindRestoring || o.Claim.Op.Phase != hostclaim.PhaseOpened { |
| 186 | return ErrPending |
| 187 | } |
| 188 | mutate = provision.MoveRolledBackClose(guard, now) |
| 189 | case RecoveryMoveRevertClose: |
| 190 | if o.Claim.Op.Kind != hostclaim.KindReverting || o.Claim.Op.Phase != hostclaim.PhaseOpened { |
| 191 | return ErrPending |
| 192 | } |
| 193 | mutate = provision.MoveRevertClose(guard, now) |
| 194 | case RecoveryMoveDeadlineRollback: |
| 195 | if o.Claim.Op.Kind != hostclaim.KindLanding && !(o.Claim.Op.Kind == hostclaim.KindMoving && o.Claim.Op.Phase == hostclaim.PhaseSealed) { |
| 196 | return ErrPending |
| 197 | } |
| 198 | deadline := rec.Move.CutDeadline |
| 199 | if d := rec.Move.LandingDeadline; d != nil && (deadline == nil || d.Before(*deadline)) { |
| 200 | deadline = d |
| 201 | } |
| 202 | if deadline == nil || !now.After(*deadline) { |
| 203 | return ErrPending |
| 204 | } |
| 205 | id, err := moves.NewRollbackID(rand.Reader) |
| 206 | if err != nil { |
| 207 | return err |
| 208 | } |
| 209 | mutate = provision.MoveRollBack(guard, id, provision.RollbackLandingDeadline, now) |
| 210 | default: |
| 211 | return ErrIdentity |
| 212 | } |
| 213 | if err := mutate(&rec); err != nil { |
| 214 | return err |
| 215 | } |
| 216 | body, err := g.prepareRecord(ctx, rec) |
| 217 | if err != nil { |
| 218 | return err |
| 219 | } |
| 220 | if _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, r.Holder.Key, r.Body, body, r.ETag); err != nil { |
| 221 | return err |
| 222 | } |
| 223 | if before, err := provision.DecodeRecord(r.Body); err == nil { |
| 224 | if t, ok := moves.TransitionOf(before, rec); ok { |
| 225 | g.wrote = &t |
| 226 | } |
| 227 | } |
| 228 | return nil |
| 229 | } |
| 230 | |
| 231 | func (g *MoveRecoveryGrant) prepareRecord(ctx context.Context, rec provision.Record) ([]byte, error) { |
| 232 | if rec.Move != nil { |
| 233 | rec.Move.UpdatedAt = g.w.now().UTC() |
| 234 | } |
| 235 | body, err := json.Marshal(rec) |
| 236 | if err != nil { |
| 237 | return nil, err |
| 238 | } |
| 239 | if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { |
| 240 | return nil, err |
| 241 | } |
| 242 | if _, _, _, err := g.current(ctx); err != nil { |
| 243 | return nil, err |
| 244 | } |
| 245 | g.state.active = false |
| 246 | return body, nil |
| 247 | } |
| 248 | |
| 249 | // captureMoveStartBase records the highest attempted CAS base before forwarding |
| 250 | // that Put. A lost journal answer never permits the Put. Only the issuing effect |
| 251 | // can extend this proof; an uncertain effect can never acquire another base. |
| 252 | func (h *StartHandle) captureMoveStartBase(ctx context.Context, seq uint64) error { |
| 253 | _, err := h.w.retryAdmission(ctx, h.account, h.app, h.appID, AdmissionObserved{}, func(a *AppAdmission) error { |
| 254 | e, ok := a.Effects[h.id] |
| 255 | if !ok || e.State != "issued" || e.Action != moveStartAction || e.Owner != h.owner || e.OperationID != h.id || e.AppID != h.appID { |
| 256 | return ErrPending |
| 257 | } |
| 258 | if old, ok := e.Inputs["record_base_seq"]; ok { |
| 259 | n, err := strconv.ParseUint(old, 10, 64) |
| 260 | if err != nil { |
| 261 | return ErrIdentity |
| 262 | } |
| 263 | if n >= seq { |
| 264 | return nil |
| 265 | } |
| 266 | } |
| 267 | if e.Inputs == nil { |
| 268 | e.Inputs = map[string]string{} |
| 269 | } |
| 270 | e.Inputs["record_base_seq"] = strconv.FormatUint(seq, 10) |
| 271 | a.Effects[h.id] = e |
| 272 | return nil |
| 273 | }) |
| 274 | return err |
| 275 | } |
| 276 | |
| 277 | // ResolveStart settles only an uncertain start. Unlike the live-handle path it |
| 278 | // never manufactures a fence: absence proves failure only above the durable base. |
| 279 | func (g *MoveRecoveryGrant) ResolveStart(ctx context.Context) (Outcome, error) { |
| 280 | if g == nil || g.state == nil { |
| 281 | return OutcomeUncertain, ErrPending |
| 282 | } |
| 283 | g.state.mu.Lock() |
| 284 | defer g.state.mu.Unlock() |
| 285 | o, _, rec, err := g.current(ctx) |
| 286 | if err != nil { |
| 287 | return OutcomeUncertain, err |
| 288 | } |
| 289 | if o.Claim.Op.Kind != hostclaim.KindMoving { |
| 290 | return OutcomeUncertain, ErrIdentity |
| 291 | } |
| 292 | a, err := g.w.read(ctx, g.o.Account, g.o.App, g.o.AppID) |
| 293 | if err != nil { |
| 294 | return OutcomeUncertain, err |
| 295 | } |
| 296 | e, ok := a.Value.Effects[g.o.OpID] |
| 297 | if !ok || e.ID != g.o.OpID || e.Action != moveStartAction || e.OperationID != g.o.OpID || e.AppID != g.o.AppID || e.Resource != o.Claim.Holder.Key || e.Inputs["move_id"] != g.o.OpID || e.Inputs["incarnation"] != g.o.Incarnation { |
| 298 | return OutcomeUncertain, ErrIdentity |
| 299 | } |
| 300 | if e.State == "complete" { |
| 301 | return settledMoveStart(e, g.o.OpID) |
| 302 | } |
| 303 | if e.State != "uncertain" { |
| 304 | return OutcomeUncertain, ErrPending |
| 305 | } |
| 306 | result := "" |
| 307 | if rec.Move != nil && rec.Move.ID == g.o.OpID && rec.Move.Incarnation == g.o.Incarnation || rec.LastMove != nil && rec.LastMove.ID == g.o.OpID || e.ResultReference == g.o.OpID { |
| 308 | result = g.o.OpID |
| 309 | } |
| 310 | if result == "" { |
| 311 | base, err := strconv.ParseUint(e.Inputs["record_base_seq"], 10, 64) |
| 312 | if err != nil || g.o.RecordSeq <= base { |
| 313 | return OutcomeUncertain, ErrMoveStartFenceUnproven |
| 314 | } |
| 315 | if e.ResultReference != "" && e.ResultReference != "not-applied" { |
| 316 | return OutcomeUncertain, ErrIdentity |
| 317 | } |
| 318 | result = "not-applied" |
| 319 | } |
| 320 | if _, _, _, err := g.current(ctx); err != nil { |
| 321 | return OutcomeUncertain, err |
| 322 | } |
| 323 | runner := EffectRunner{w: g.w, account: g.o.Account, app: g.o.App, appID: g.o.AppID, operation: g.o.OpID, owner: e.Owner, now: g.w.now} |
| 324 | g.state.active = false |
| 325 | if err := runner.resolveUncertain(ctx, e, result); err != nil { |
| 326 | if errors.Is(err, errSettledElsewhere) { |
| 327 | a, rerr := g.w.read(ctx, g.o.Account, g.o.App, g.o.AppID) |
| 328 | if rerr == nil { |
| 329 | return settledMoveStart(a.Value.Effects[g.o.OpID], g.o.OpID) |
| 330 | } |
| 331 | } |
| 332 | return OutcomeUncertain, err |
| 333 | } |
| 334 | if result == g.o.OpID { |
| 335 | return OutcomeComplete, nil |
| 336 | } |
| 337 | return OutcomeNotApplied, nil |
| 338 | } |
| 339 | |
| 340 | type MoveRecoveryClaimAction string |
| 341 | |
| 342 | const ( |
| 343 | RecoveryMoveFlip MoveRecoveryClaimAction = "flip" |
| 344 | RecoveryMoveCommit MoveRecoveryClaimAction = "commit" |
| 345 | RecoveryMoveRollback MoveRecoveryClaimAction = "rollback" |
| 346 | RecoveryMoveRelease MoveRecoveryClaimAction = "release" |
| 347 | RecoveryMoveReleaseNotApplied MoveRecoveryClaimAction = "release-not-applied" |
| 348 | ) |
| 349 | |
| 350 | // Claim catches up a claim half from its authoritative record. No caller |
| 351 | // supplies executor identity, scopes, generation, or an arbitrary transition. |
| 352 | func (g *MoveRecoveryGrant) Claim(ctx context.Context, action MoveRecoveryClaimAction) error { |
| 353 | if g == nil || g.state == nil { |
| 354 | return ErrPending |
| 355 | } |
| 356 | g.state.mu.Lock() |
| 357 | defer g.state.mu.Unlock() |
| 358 | o, _, rec, err := g.current(ctx) |
| 359 | if err != nil { |
| 360 | return err |
| 361 | } |
| 362 | c, h, m := o.Claim, o.Claim.Holder, rec.Move |
| 363 | var transition func(*hostclaim.Claim) error |
| 364 | if action == RecoveryMoveReleaseNotApplied { |
| 365 | if m != nil || rec.LastMove != nil && rec.LastMove.ID == g.o.OpID || c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseMoving { |
| 366 | return ErrPending |
| 367 | } |
| 368 | a, err := g.w.read(ctx, g.o.Account, g.o.App, g.o.AppID) |
| 369 | if err != nil { |
| 370 | return err |
| 371 | } |
| 372 | e, ok := a.Value.Effects[g.o.OpID] |
| 373 | if !ok || e.Action != moveStartAction || e.OperationID != g.o.OpID || e.AppID != g.o.AppID || e.Resource != h.Key || e.State != "complete" || e.ResultReference != "not-applied" { |
| 374 | return ErrPending |
| 375 | } |
| 376 | transition = hostclaim.ReleaseMove(g.o.OpID, c.Op.Intent["source"]) |
| 377 | } else if action == RecoveryMoveRelease { |
| 378 | // Close first, then release. A delayed close cannot strand a serving |
| 379 | // claim with a live move; a crash here replays from LastMove. |
| 380 | if m != nil || rec.LastMove == nil || rec.LastMove.ID != g.o.OpID { |
| 381 | return ErrPending |
| 382 | } |
| 383 | switch { |
| 384 | case rec.LastMove.Outcome == "done" && c.Op.Kind == hostclaim.KindLanding && c.Op.Phase == hostclaim.PhaseCommitted: |
| 385 | transition = hostclaim.ReleaseMove(g.o.OpID, c.Op.Intent["target"]) |
| 386 | case rec.LastMove.Outcome == "rolled_back" && (c.Op.Kind == hostclaim.KindRestoring || c.Op.Kind == hostclaim.KindReverting) && c.Op.Phase == hostclaim.PhaseOpened: |
| 387 | transition = hostclaim.ReleaseMove(g.o.OpID, c.Op.Intent["source"]) |
| 388 | case rec.LastMove.Outcome == "aborted" && c.Op.Kind == hostclaim.KindMoving && c.Op.Phase == hostclaim.PhaseAborting: |
| 389 | transition = hostclaim.ReleaseAbort(g.o.OpID, c.Op.Intent["source"]) |
| 390 | default: |
| 391 | return ErrPending |
| 392 | } |
| 393 | } else { |
| 394 | if m == nil || m.ID != g.o.OpID || m.Incarnation != g.o.Incarnation { |
| 395 | return ErrIdentity |
| 396 | } |
| 397 | switch action { |
| 398 | case RecoveryMoveFlip: |
| 399 | if m.State != provision.MoveLanding || m.FlippedAt == nil || rec.Region != m.To { |
| 400 | return ErrPending |
| 401 | } |
| 402 | transition = hostclaim.Flip(g.o.OpID, c.Op.Intent["target"], h.Stamp, h.Generation) |
| 403 | case RecoveryMoveRollback: |
| 404 | if m.State != provision.MoveRollingBack || m.Rollback == nil || rec.Region != m.From { |
| 405 | return ErrPending |
| 406 | } |
| 407 | if c.Op.Kind == hostclaim.KindMoving { |
| 408 | transition = hostclaim.RollBackFlipped(g.o.OpID, m.Rollback.ID, c.Op.Intent["target"], h.Stamp, h.Generation) |
| 409 | } else { |
| 410 | transition = hostclaim.RollBack(g.o.OpID, m.Rollback.ID) |
| 411 | } |
| 412 | case RecoveryMoveCommit: |
| 413 | if m.State != provision.MoveCommitted || m.CommittedAt == nil || m.FlippedAt == nil || rec.Region != m.To { |
| 414 | return ErrPending |
| 415 | } |
| 416 | source, staging, err := moves.CommitScopes(c, m.FlippedAt) |
| 417 | if err != nil { |
| 418 | return ErrIdentity |
| 419 | } |
| 420 | id, err := operationToken() |
| 421 | if err != nil { |
| 422 | return err |
| 423 | } |
| 424 | n := 0 |
| 425 | // The record's codes become engine regions as the box's commit |
| 426 | // makes them (park.PlacementRegion; K9): a move out of home queues |
| 427 | // its location park in park.HomeRegion, never "". |
| 428 | transition = hostclaim.Commit(g.o.OpID, park.PlacementRegion(m.From), c.Op.Intent["source"], park.PlacementRegion(m.To), source, staging, func() string { n++; return id + strconv.Itoa(n) }) |
| 429 | default: |
| 430 | return ErrIdentity |
| 431 | } |
| 432 | } |
| 433 | if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { |
| 434 | return err |
| 435 | } |
| 436 | g.state.active = false |
| 437 | _, err = hostclaim.Update(ctx, g.w.raw, o, func(c *hostclaim.Claim) error { |
| 438 | // Re-read the exact record immediately before the claim CAS. |
| 439 | r, err := ReadRecord(ctx, g.w.raw, h.Key) |
| 440 | if err != nil { |
| 441 | return err |
| 442 | } |
| 443 | if r.ETag != g.o.RecordETag { |
| 444 | return release.ErrCASConflict |
| 445 | } |
| 446 | if err := moveAdmission(ctx, g.w.raw, o.Host, c); err != nil { |
| 447 | return err |
| 448 | } |
| 449 | return transition(c) |
| 450 | }) |
| 451 | return err |
| 452 | } |
| 453 | |
| 454 | // Convert binds C7's existing conversion to the grant and a verified fence. |
| 455 | // The first pass writes the terminal record decision; a replay verifies it |
| 456 | // and performs the one claim CAS. It never authorizes relocation execution. |
| 457 | func (g *MoveRecoveryGrant) Convert(ctx context.Context, ref hostclaim.FenceRef) error { |
| 458 | if g == nil || g.state == nil { |
| 459 | return ErrPending |
| 460 | } |
| 461 | authority := g.authority |
| 462 | if authority == nil { |
| 463 | return ErrContract |
| 464 | } |
| 465 | g.state.mu.Lock() |
| 466 | defer g.state.mu.Unlock() |
| 467 | o, r, rec, err := g.current(ctx) |
| 468 | if err != nil { |
| 469 | return err |
| 470 | } |
| 471 | fence, err := hostclaim.VerifyFence(ctx, g.w.raw, authority, ref) |
| 472 | if err != nil { |
| 473 | return err |
| 474 | } |
| 475 | placement, ok := authority.(oneBoxRegionAuthority) |
| 476 | if !ok { |
| 477 | return ErrContract |
| 478 | } |
| 479 | guard := recoveryMovingDecision{movingRecordDecision: movingRecordDecision{store: g.w.raw, placement: placement, homeRegion: g.w.homeRegion}, observation: g.o} |
| 480 | c, h := o.Claim, o.Claim.Holder |
| 481 | if c.Op.Kind == hostclaim.KindRelocating { |
| 482 | if _, err := guard.CheckMovingBeforeFlip(ctx, o, c.Op.ID, ref); err != nil { |
| 483 | return err |
| 484 | } |
| 485 | return hostclaim.VerifyOperationFence(ctx, g.w.raw, authority, c.Op, c.Op.Intent["source_box"]) |
| 486 | } |
| 487 | if c.Op.Kind != hostclaim.KindMoving || c.Op.Intent["source_enrollment"] != ref.Enrollment || fence.Box() != c.Op.Intent["source"] || !park.PlacementMatches(c.Op.Intent["home_region"], rec.Region, fence.Region()) { |
| 488 | return hostclaim.ErrFence |
| 489 | } |
| 490 | if err := placement.CheckOneBoxRegion(ctx, fence.Box(), fence.Region()); err != nil { |
| 491 | return err |
| 492 | } |
| 493 | if rec.Move == nil || rec.Move.ID != g.o.OpID || rec.Move.Incarnation != h.Incarnation { |
| 494 | return ErrIdentity |
| 495 | } |
| 496 | if rec.Move.State != provision.MoveRelocating { |
| 497 | raw, err := json.Marshal(ref) |
| 498 | if err != nil { |
| 499 | return err |
| 500 | } |
| 501 | moving, holder, err := movingIdentities(c) |
| 502 | if err != nil { |
| 503 | return err |
| 504 | } |
| 505 | if err := provision.MoveRelocatingDecision(provision.MoveGuard{ID: g.o.OpID, Incarnation: h.Incarnation}, fence.Box(), ref.Enrollment, rec.Region, string(raw), c.Gen, o.ETag, moving, holder, g.w.now())(&rec); err != nil { |
| 506 | return err |
| 507 | } |
| 508 | body, err := g.prepareRecord(ctx, rec) |
| 509 | if err != nil { |
| 510 | return err |
| 511 | } |
| 512 | _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, r.Holder.Key, r.Body, body, r.ETag) |
| 513 | return err |
| 514 | } |
| 515 | if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { |
| 516 | return err |
| 517 | } |
| 518 | g.state.active = false |
| 519 | _, err = hostclaim.ConvertMovingToRelocating(ctx, g.w.raw, o, g.o.OpID, ref, authority, guard) |
| 520 | return err |
| 521 | } |
| 522 | |
| 523 | // recoveryMovingDecision retains the recovery grant's exact record binding |
| 524 | // through the final pre-CAS guard, including after fence authority reads. |
| 525 | type recoveryMovingDecision struct { |
| 526 | movingRecordDecision |
| 527 | observation ConsoleMoveObservation |
| 528 | } |
| 529 | |
| 530 | func (g recoveryMovingDecision) CheckMovingBeforeFlip(ctx context.Context, o hostclaim.Observed, id string, ref hostclaim.FenceRef) (string, error) { |
| 531 | region, err := g.movingRecordDecision.CheckMovingBeforeFlip(ctx, o, id, ref) |
| 532 | if err != nil { |
| 533 | return "", err |
| 534 | } |
| 535 | r, err := ReadRecord(ctx, g.store, o.Claim.Holder.Key) |
| 536 | if err != nil { |
| 537 | return "", err |
| 538 | } |
| 539 | _, seq, err := recordSequence(r.Body) |
| 540 | if err != nil { |
| 541 | return "", err |
| 542 | } |
| 543 | if r.ETag != g.observation.RecordETag || seq != g.observation.RecordSeq { |
| 544 | return "", release.ErrCASConflict |
| 545 | } |
| 546 | return region, nil |
| 547 | } |
| 548 | |
amadan