| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "encoding/json" |
| 8 | "github.com/carlosframework/platform/internal/park" |
| 9 | "strconv" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostclaim" |
| 13 | "github.com/carlosframework/platform/internal/provision" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | ) |
| 16 | |
| 17 | // movingIdentities digests the claim's operation and holder. The record |
| 18 | // decision binds both, so a changed operation can never reuse it, while a |
| 19 | // chore acknowledgement (which changes neither) can. |
| 20 | func movingIdentities(c *hostclaim.Claim) (string, string, error) { |
| 21 | if c == nil || c.Holder == nil { |
| 22 | return "", "", ErrIdentity |
| 23 | } |
| 24 | h, err := json.Marshal(c.Holder) |
| 25 | if err != nil { |
| 26 | return "", "", err |
| 27 | } |
| 28 | op, err := json.Marshal(c.Op) |
| 29 | if err != nil { |
| 30 | return "", "", err |
| 31 | } |
| 32 | hs, ops := sha256.Sum256(h), sha256.Sum256(op) |
| 33 | return hex.EncodeToString(ops[:]), hex.EncodeToString(hs[:]), nil |
| 34 | } |
| 35 | |
| 36 | // moveAdmission rereads the app admission the move started under (Task 7). |
| 37 | // Accepted work drains under closed admission and expiry does not revoke it, |
| 38 | // so only identity is compared: the exact "move" registration, named by the |
| 39 | // move's own op id, for this host and incarnation, at the admission's epoch, |
| 40 | // under the claim's app id. |
| 41 | func moveAdmission(ctx context.Context, s release.Store, host string, c *hostclaim.Claim) error { |
| 42 | if c == nil || c.Holder == nil { |
| 43 | return ErrIdentity |
| 44 | } |
| 45 | appID, id := c.Op.Intent["app_id"], c.Op.Intent["admission_id"] |
| 46 | if appID == "" || id == "" || id != c.Op.ID { |
| 47 | return ErrIdentity |
| 48 | } |
| 49 | a, err := ReadAdmission(ctx, s, c.Holder.Account, c.Holder.App, appID) |
| 50 | if err != nil { |
| 51 | return err |
| 52 | } |
| 53 | r, ok := a.Value.Registrations[id] |
| 54 | if !ok || r.Kind != RegistrationMove || r.AppID != appID || r.Epoch != a.Value.Epoch || r.Host != host || r.Incarnation != c.Holder.Incarnation { |
| 55 | return ErrPending |
| 56 | } |
| 57 | return nil |
| 58 | } |
| 59 | |
| 60 | // movingRecordDecision is the PreFlipGuard over the authoritative record's |
| 61 | // durable decision (provision.MoveRelocatingDecision). |
| 62 | type oneBoxRegionAuthority interface { |
| 63 | CheckOneBoxRegion(context.Context, string, string) error |
| 64 | } |
| 65 | |
| 66 | type movingRecordDecision struct { |
| 67 | homeRegion string |
| 68 | placement oneBoxRegionAuthority |
| 69 | store release.Store |
| 70 | } |
| 71 | |
| 72 | var _ hostclaim.PreFlipGuard = movingRecordDecision{} |
| 73 | |
| 74 | // CheckMovingBeforeFlip reads the move's own intent while the claim is |
| 75 | // moving ("source", "to") and the conversion's once relocating |
| 76 | // ("source_box", "region", "move_to_region"). The region it returns is the |
| 77 | // verified roster identity for the record placement; the home code stays |
| 78 | // empty in the decision. ConvertMovingToRelocating compares the returned |
| 79 | // identity with the unmodified fence binding. |
| 80 | // ConvertMovingToRelocating calls it again inside its transition, so the |
| 81 | // admission reread here is the last read before the one claim CAS. |
| 82 | func (g movingRecordDecision) CheckMovingBeforeFlip(ctx context.Context, o hostclaim.Observed, opID string, ref hostclaim.FenceRef) (string, error) { |
| 83 | if o.Claim == nil || o.Claim.Holder == nil || o.Absent || o.ETag == "" || opID == "" || o.Claim.Op.ID != opID { |
| 84 | return "", ErrIdentity |
| 85 | } |
| 86 | c, h := o.Claim, o.Claim.Holder |
| 87 | if !operationHomeMatches(g.homeRegion, c) { |
| 88 | return "", ErrIdentity |
| 89 | } |
| 90 | if err := moveAdmission(ctx, g.store, o.Host, c); err != nil { |
| 91 | return "", err |
| 92 | } |
| 93 | rec, etag, err := provision.GetRecordETag(ctx, g.store, h.Account, h.App, o.Host) |
| 94 | if err != nil { |
| 95 | return "", err |
| 96 | } |
| 97 | inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) |
| 98 | if err != nil || etag == "" || inc != h.Incarnation || rec.Move == nil || rec.Move.ID != c.Op.ID || rec.Move.Incarnation != h.Incarnation || rec.Move.State != provision.MoveRelocating || rec.Move.Relocation == nil { |
| 99 | return "", ErrIdentity |
| 100 | } |
| 101 | d := rec.Move.Relocation |
| 102 | moving, holder, err := movingIdentities(c) |
| 103 | if err != nil { |
| 104 | return "", err |
| 105 | } |
| 106 | if holder != d.HolderIdentity { |
| 107 | return "", ErrIdentity |
| 108 | } |
| 109 | var source, region, to string |
| 110 | switch c.Op.Kind { |
| 111 | case hostclaim.KindMoving: |
| 112 | if d.MovingIdentity != moving || c.Gen < d.ClaimGen { |
| 113 | return "", hostclaim.ErrConflict |
| 114 | } |
| 115 | region = park.PlacementRegion(rec.Region) |
| 116 | if rec.Region == "" && c.Op.Intent["home_region"] != "" { |
| 117 | region = c.Op.Intent["home_region"] |
| 118 | } |
| 119 | source, to = c.Op.Intent["source"], c.Op.Intent["to"] |
| 120 | case hostclaim.KindRelocating: |
| 121 | convertedGen, err := strconv.ParseUint(c.Op.Journal["conversion_claim_gen"], 10, 64) |
| 122 | raw, e := json.Marshal(d) |
| 123 | if e != nil { |
| 124 | return "", e |
| 125 | } |
| 126 | sum := sha256.Sum256(raw) |
| 127 | if err != nil || convertedGen < d.ClaimGen || c.Op.Journal["conversion_claim_etag"] == "" || c.Op.Journal["conversion_decision_sha256"] != hex.EncodeToString(sum[:]) { |
| 128 | return "", hostclaim.ErrConflict |
| 129 | } |
| 130 | source, region, to = c.Op.Intent["source_box"], c.Op.Intent["region"], c.Op.Intent["move_to_region"] |
| 131 | default: |
| 132 | return "", hostclaim.ErrWrongState |
| 133 | } |
| 134 | raw, err := json.Marshal(ref) |
| 135 | if err != nil { |
| 136 | return "", err |
| 137 | } |
| 138 | if source == "" || region == "" || d.DecidedAt.IsZero() || d.Fence != string(raw) || d.Enrollment != ref.Enrollment || d.Enrollment != c.Op.Intent["source_enrollment"] || d.SourceBox != source || |
| 139 | !park.PlacementMatches(c.Op.Intent["home_region"], d.Region, region) || rec.Region != d.Region || rec.Move.From != d.Region || !park.PlacementMatches(c.Op.Intent["home_region"], rec.Move.To, to) || rec.Move.FlippedAt != nil || rec.Move.CommittedAt != nil || rec.Move.Rollback != nil { |
| 140 | return "", ErrIdentity |
| 141 | } |
| 142 | switch d.PreviousState { |
| 143 | case provision.MovePreparing, provision.MoveReady, provision.MoveCutting: |
| 144 | default: |
| 145 | return "", ErrIdentity |
| 146 | } |
| 147 | if g.placement == nil { |
| 148 | return "", ErrContract |
| 149 | } |
| 150 | if err := g.placement.CheckOneBoxRegion(ctx, source, region); err != nil { |
| 151 | return "", err |
| 152 | } |
| 153 | return region, nil |
| 154 | } |
| 155 | |
| 156 | // ConversionDecisionDigest binds the converted claim to the exact record |
| 157 | // decision it was converted under. |
| 158 | func (g movingRecordDecision) ConversionDecisionDigest(ctx context.Context, o hostclaim.Observed, id string, ref hostclaim.FenceRef) (string, error) { |
| 159 | if _, err := g.CheckMovingBeforeFlip(ctx, o, id, ref); err != nil { |
| 160 | return "", err |
| 161 | } |
| 162 | h := o.Claim.Holder |
| 163 | rec, err := provision.GetRecord(ctx, g.store, h.Account, h.App, o.Host) |
| 164 | if err != nil { |
| 165 | return "", err |
| 166 | } |
| 167 | if rec.Move == nil || rec.Move.Relocation == nil { |
| 168 | return "", ErrIdentity |
| 169 | } |
| 170 | raw, err := json.Marshal(rec.Move.Relocation) |
| 171 | if err != nil { |
| 172 | return "", err |
| 173 | } |
| 174 | digest := sha256.Sum256(raw) |
| 175 | return hex.EncodeToString(digest[:]), nil |
| 176 | } |
| 177 | |
| 178 | // heldMoveRegistration: mh is the writer-minted handle of the move's own |
| 179 | // registration (its id is the op id and admission_id, its app id the |
| 180 | // claim's), still owned by mh at mh's epoch and live by the writer clock. |
| 181 | // The data boundary checks the same again at each write (the issued effect |
| 182 | // at the record Put, the lifecycle grant at the claim CAS); this check makes |
| 183 | // an unbound call refuse before any of them. Admission may be closed: |
| 184 | // accepted work drains under closed admission. |
| 185 | func heldMoveRegistration(ctx context.Context, s release.Store, mh *StartHandle, host string, c *hostclaim.Claim) error { |
| 186 | if mh == nil || mh.w == nil || c == nil || c.Holder == nil || mh.account != c.Holder.Account || mh.app != c.Holder.App || mh.id != c.Op.ID || mh.appID != c.Op.Intent["app_id"] { |
| 187 | return ErrIdentity |
| 188 | } |
| 189 | if err := moveAdmission(ctx, s, host, c); err != nil { |
| 190 | return err |
| 191 | } |
| 192 | a, err := ReadAdmission(ctx, s, mh.account, mh.app, mh.appID) |
| 193 | if err != nil { |
| 194 | return err |
| 195 | } |
| 196 | r := a.Value.Registrations[mh.id] |
| 197 | if r.Owner != mh.owner || r.Epoch != mh.epoch || a.Value.Epoch != mh.epoch || !RegistrationLive(r, mh.w.now()) { |
| 198 | return ErrPending |
| 199 | } |
| 200 | return nil |
| 201 | } |
| 202 | |
| 203 | // RecoverTerminatedMovingHost is called only with claim mode enabled, by the |
| 204 | // Moves coordinator holding mh, the handle RegisterMove minted for this move |
| 205 | // and kept live (Renew) since (no caller is wired yet). It persists the |
| 206 | // record decision as one issued effect of that registration, then converts |
| 207 | // the claim in one CAS under its lifecycle grant. No relocation |
| 208 | // installation or execution is activated, including on replay. |
| 209 | // |
| 210 | // While moving, the source is the claim's "source"; once relocating, its |
| 211 | // "source_box". The region is never claim input: it is the authoritative |
| 212 | // record's, and must equal the fence's roster binding. |
| 213 | func RecoverTerminatedMovingHost(ctx context.Context, s release.Store, mh *StartHandle, authority hostclaim.FenceAuthority, host string, ref hostclaim.FenceRef, now time.Time) error { |
| 214 | o, err := hostclaim.Read(ctx, s, host) |
| 215 | if err != nil { |
| 216 | return err |
| 217 | } |
| 218 | if o.Absent || o.Claim.Holder == nil { |
| 219 | return ErrIdentity |
| 220 | } |
| 221 | c, h := o.Claim, o.Claim.Holder |
| 222 | if err := heldMoveRegistration(ctx, s, mh, host, c); err != nil { |
| 223 | return err |
| 224 | } |
| 225 | fence, err := hostclaim.VerifyFence(ctx, s, authority, ref) |
| 226 | if err != nil { |
| 227 | return err |
| 228 | } |
| 229 | placement, ok := authority.(oneBoxRegionAuthority) |
| 230 | if !ok { |
| 231 | return ErrContract |
| 232 | } |
| 233 | if !operationHomeMatches(mh.w.homeRegion, c) { |
| 234 | return ErrIdentity |
| 235 | } |
| 236 | guard := movingRecordDecision{store: s, placement: placement, homeRegion: mh.w.homeRegion} |
| 237 | if c.Op.Kind == hostclaim.KindRelocating { |
| 238 | source := c.Op.Intent["source_box"] |
| 239 | if fence.Box() != source || fence.Region() != c.Op.Intent["region"] { |
| 240 | return hostclaim.ErrFence |
| 241 | } |
| 242 | if _, err := guard.CheckMovingBeforeFlip(ctx, o, c.Op.ID, ref); err != nil { |
| 243 | return err |
| 244 | } |
| 245 | return hostclaim.VerifyOperationFence(ctx, s, authority, c.Op, source) |
| 246 | } |
| 247 | if c.Op.Kind != hostclaim.KindMoving || c.Op.Intent["source_enrollment"] != ref.Enrollment { |
| 248 | return hostclaim.ErrWrongState |
| 249 | } |
| 250 | source := c.Op.Intent["source"] |
| 251 | if fence.Box() != source { |
| 252 | return hostclaim.ErrFence |
| 253 | } |
| 254 | rec, err := provision.GetRecord(ctx, s, h.Account, h.App, host) |
| 255 | if err != nil { |
| 256 | return err |
| 257 | } |
| 258 | region := rec.Region |
| 259 | if !park.PlacementMatches(c.Op.Intent["home_region"], region, fence.Region()) || rec.Move == nil || rec.Move.From != region { |
| 260 | return hostclaim.ErrFence |
| 261 | } |
| 262 | if err := placement.CheckOneBoxRegion(ctx, source, fence.Region()); err != nil { |
| 263 | return err |
| 264 | } |
| 265 | raw, err := json.Marshal(ref) |
| 266 | if err != nil { |
| 267 | return err |
| 268 | } |
| 269 | movingIdentity, holderIdentity, err := movingIdentities(c) |
| 270 | if err != nil { |
| 271 | return err |
| 272 | } |
| 273 | // A replay whose decision is already durable for this exact operation |
| 274 | // and holder writes nothing more (a crash after the record Put leaves |
| 275 | // that effect uncertain, and it would block a second one). |
| 276 | if _, undecided := guard.CheckMovingBeforeFlip(ctx, o, c.Op.ID, ref); undecided != nil { |
| 277 | moveGuard := provision.MoveGuard{ID: c.Op.ID, Incarnation: h.Incarnation} |
| 278 | effect, err := operationToken() |
| 279 | if err != nil { |
| 280 | return err |
| 281 | } |
| 282 | if _, err = runRecordEffect(ctx, mh, Effect{ID: effect, Resource: h.Key, Action: "move-relocating-decision", Inputs: map[string]string{"move_id": c.Op.ID, "claim_etag": o.ETag, "fence_sha256": ref.SHA256}}, func(ctx context.Context) (string, error) { |
| 283 | _, err := setLifecycleRecordMove(ctx, s, o, func(rec *provision.Record) error { |
| 284 | // Compare the claim before deciding the record. MoveRelocatingDecision |
| 285 | // rechecks the fresh record's region against the bound region. |
| 286 | fresh, err := hostclaim.Read(ctx, s, host) |
| 287 | if err != nil { |
| 288 | return err |
| 289 | } |
| 290 | if fresh.Absent || fresh.ETag != o.ETag { |
| 291 | return hostclaim.ErrConflict |
| 292 | } |
| 293 | return provision.MoveRelocatingDecision(moveGuard, source, ref.Enrollment, region, string(raw), o.Claim.Gen, o.ETag, movingIdentity, holderIdentity, now)(rec) |
| 294 | }) |
| 295 | if err != nil { |
| 296 | return "", err |
| 297 | } |
| 298 | return c.Op.ID, nil |
| 299 | }); err != nil { |
| 300 | return err |
| 301 | } |
| 302 | } |
| 303 | fresh, err := hostclaim.Read(ctx, s, host) |
| 304 | if err != nil { |
| 305 | return err |
| 306 | } |
| 307 | if fresh.Absent || fresh.Claim.Op.Kind != hostclaim.KindMoving { |
| 308 | return hostclaim.ErrWrongState |
| 309 | } |
| 310 | freshMoving, freshHolder, err := movingIdentities(fresh.Claim) |
| 311 | if err != nil { |
| 312 | return err |
| 313 | } |
| 314 | if freshMoving != movingIdentity || freshHolder != holderIdentity { |
| 315 | return hostclaim.ErrConflict |
| 316 | } |
| 317 | _, err = hostclaim.ConvertMovingToRelocating(mh.Lifecycle(ctx), s, fresh, c.Op.ID, ref, authority, guard) |
| 318 | return err |
| 319 | } |
| 320 | |
amadan