| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "encoding/json" |
| 7 | "github.com/carlosframework/platform/internal/hostclaim" |
| 8 | "github.com/carlosframework/platform/internal/hostops/internal/fencerelease" |
| 9 | "github.com/carlosframework/platform/internal/moves" |
| 10 | "github.com/carlosframework/platform/internal/park" |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | // FencedTargetOutcome is what one ReleaseFencedTarget step found or wrote, |
| 16 | // as values, never authority. VerifiedDirty is set only with ErrPending, and |
| 17 | // only after the fence verified: the target released dirty, which the grant |
| 18 | // leaves for J4. Every other ErrPending (a withdrawn registration, a spent |
| 19 | // grant) leaves it unset. Transition is the step's record write, named as the |
| 20 | // move log names it (moves.TransitionOf over the record before and after the |
| 21 | // write), and ClaimRolledBack that the step's claim write rolled a landing |
| 22 | // claim back: the caller logs the write from this evidence, never from a |
| 23 | // later read (Transition's rule). |
| 24 | type FencedTargetOutcome struct { |
| 25 | VerifiedDirty bool |
| 26 | Transition *moves.Transition |
| 27 | ClaimRolledBack bool |
| 28 | } |
| 29 | |
| 30 | // ReleaseFencedTarget advances one record-first recovery step per grant: |
| 31 | // record decision, claim release with parks, then record release. Each pass |
| 32 | // verifies unconsumed authority; completion uses the durable claim evidence. |
| 33 | func (g *MoveRecoveryGrant) ReleaseFencedTarget(ctx context.Context, ref hostclaim.FenceRef) (FencedTargetOutcome, error) { |
| 34 | var out FencedTargetOutcome |
| 35 | err := g.releaseFencedTarget(ctx, ref, &out) |
| 36 | return out, err |
| 37 | } |
| 38 | |
| 39 | func (g *MoveRecoveryGrant) releaseFencedTarget(ctx context.Context, ref hostclaim.FenceRef, out *FencedTargetOutcome) error { |
| 40 | if g == nil || g.state == nil { |
| 41 | return ErrPending |
| 42 | } |
| 43 | g.state.mu.Lock() |
| 44 | defer g.state.mu.Unlock() |
| 45 | o, record, rec, err := g.current(ctx) |
| 46 | if err != nil { |
| 47 | return err |
| 48 | } |
| 49 | |
| 50 | m := rec.Move |
| 51 | if m == nil || m.ID != g.o.OpID || m.Incarnation != g.o.Incarnation { |
| 52 | return ErrIdentity |
| 53 | } |
| 54 | guard := provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation} |
| 55 | writeRecord := func() error { |
| 56 | body, err := g.prepareRecord(ctx, rec) |
| 57 | if err != nil { |
| 58 | return err |
| 59 | } |
| 60 | if _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, record.Holder.Key, record.Body, body, record.ETag); err != nil { |
| 61 | return err |
| 62 | } |
| 63 | if before, err := provision.DecodeRecord(record.Body); err == nil { |
| 64 | if t, ok := moves.TransitionOf(before, rec); ok { |
| 65 | out.Transition = &t |
| 66 | } |
| 67 | } |
| 68 | return nil |
| 69 | } |
| 70 | if consumed, ok := hostclaim.ConsumedMoveTargetFence(o.Claim); ok { |
| 71 | if consumed != ref || m.Rollback == nil || m.State != provision.MoveRollingBack || rec.Region != m.From { |
| 72 | return ErrIdentity |
| 73 | } |
| 74 | rb := m.Rollback |
| 75 | if (rb.TargetPhase == provision.RbTargetReleased || rb.TargetPhase == provision.RbTargetCleaned && rb.TargetParkedAt != nil) && !rb.Dirty { |
| 76 | if rb.LossRef == ref.Key && rb.Evidence == ref.Key { |
| 77 | return nil |
| 78 | } |
| 79 | return g.verifyReleasedTarget(ctx, o, rec) |
| 80 | } |
| 81 | if rb.LossRef != ref.Key { |
| 82 | return ErrIdentity |
| 83 | } |
| 84 | if err := provision.MoveRollbackLossCommitted(guard)(&rec); err != nil { |
| 85 | return err |
| 86 | } |
| 87 | return writeRecord() |
| 88 | } |
| 89 | fence, err := hostclaim.VerifyMoveTargetFence(ctx, g.w.raw, o, ref, MoveFenceRegions{Authority: g.authority, HomeRegion: o.Claim.Op.Intent["home_region"]}) |
| 90 | if err != nil { |
| 91 | return err |
| 92 | } |
| 93 | if m.State == provision.MoveLanding { |
| 94 | if rec.Region != m.To { |
| 95 | return ErrIdentity |
| 96 | } |
| 97 | id, err := moves.NewRollbackID(rand.Reader) |
| 98 | if err != nil { |
| 99 | return err |
| 100 | } |
| 101 | if err := provision.MoveRollBack(guard, id, provision.RollbackTargetFenced, g.w.now())(&rec); err != nil { |
| 102 | return err |
| 103 | } |
| 104 | } else if m.State != provision.MoveRollingBack || m.Rollback == nil || rec.Region != m.From { |
| 105 | return ErrIdentity |
| 106 | } |
| 107 | rb := m.Rollback |
| 108 | if rb.Dirty { |
| 109 | out.VerifiedDirty = true |
| 110 | return ErrPending |
| 111 | } // J4: a dead dirty target cannot build a return lane. |
| 112 | released := rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty |
| 113 | if released { |
| 114 | if err := g.verifyReleasedTarget(ctx, o, rec); err != nil { |
| 115 | return err |
| 116 | } |
| 117 | if o.Claim.Op.Kind != hostclaim.KindRestoring || o.Claim.Op.Phase != hostclaim.PhaseReleasing || o.Claim.Op.Exec != o.Claim.Op.Intent["target"] { |
| 118 | return ErrPending |
| 119 | } |
| 120 | } else if rb.TargetPhase != provision.RbTargetLost || rb.LossRef != ref.Key { |
| 121 | if err := provision.MoveRollbackTargetLost(guard, ref.Key, g.w.now())(&rec); err != nil { |
| 122 | return err |
| 123 | } |
| 124 | return writeRecord() |
| 125 | } |
| 126 | h := o.Claim.Holder |
| 127 | // The record's codes become engine regions (park.PlacementRegion): a |
| 128 | // move out of home names its source park.HomeRegion, never "", which |
| 129 | // the claim refuses (home-region parks, part 1). |
| 130 | r := hostclaim.FencedTargetRelease{OpID: m.ID, RollbackID: rb.ID, Target: o.Claim.Op.Intent["target"], TargetRegion: park.PlacementRegion(m.To), SourceRegion: park.PlacementRegion(m.From), |
| 131 | Location: moves.TargetLocationScope(*h, fence.ObservedAt()), Lane: moves.StagingScope(o.Host, m.ID, h.Incarnation, fence.ObservedAt()), NewID: moveLossNewID(ref.SHA256), Fence: fence} |
| 132 | if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { |
| 133 | return err |
| 134 | } |
| 135 | // Fence and roster reads may have raced a new record. Bind both ETag and sequence again. |
| 136 | if _, _, _, err := g.current(ctx); err != nil { |
| 137 | return err |
| 138 | } |
| 139 | g.state.active = false |
| 140 | landing := false |
| 141 | _, err = hostclaim.Update(ctx, g.w.raw, o, func(c *hostclaim.Claim) error { |
| 142 | record, err := ReadRecord(ctx, g.w.raw, h.Key) |
| 143 | if err != nil { |
| 144 | return err |
| 145 | } |
| 146 | _, seq, err := recordSequence(record.Body) |
| 147 | if err != nil { |
| 148 | return err |
| 149 | } |
| 150 | if record.ETag != g.o.RecordETag || seq != g.o.RecordSeq { |
| 151 | return release.ErrCASConflict |
| 152 | } |
| 153 | if err := moveAdmission(ctx, g.w.raw, o.Host, c); err != nil { |
| 154 | return err |
| 155 | } |
| 156 | r.Token = fencerelease.New(c.Host, c.Holder.Incarnation, c.Gen, r.OpID, r.RollbackID, r.Target) |
| 157 | defer r.Token.Revoke() |
| 158 | landing = c.Op.Kind == hostclaim.KindLanding |
| 159 | return hostclaim.ReleaseFencedTarget(r)(c) |
| 160 | }) |
| 161 | out.ClaimRolledBack = err == nil && landing |
| 162 | return err |
| 163 | } |
| 164 | |
| 165 | // The record-first release can outlive its target. Require the canonical complete report. |
| 166 | func (g *MoveRecoveryGrant) verifyReleasedTarget(ctx context.Context, o hostclaim.Observed, rec provision.Record) error { |
| 167 | m, h := rec.Move, o.Claim.Holder |
| 168 | if m == nil || m.Rollback == nil || m.Rollback.Dirty || (m.Rollback.TargetPhase != provision.RbTargetReleased && (m.Rollback.TargetPhase != provision.RbTargetCleaned || m.Rollback.TargetParkedAt == nil)) { |
| 169 | return ErrIdentity |
| 170 | } |
| 171 | key := moves.StoppedKey(moves.ReportPrefix(h.Account, h.App, o.Host, m.ID), "target") |
| 172 | if m.Rollback.Evidence != key { |
| 173 | return ErrIdentity |
| 174 | } |
| 175 | raw, _, err := g.w.raw.Get(ctx, key) |
| 176 | if err != nil { |
| 177 | return err |
| 178 | } |
| 179 | var report moves.StoppedReport |
| 180 | if json.Unmarshal(raw, &report) != nil || !report.Complete() || report.Role != "target" || report.Box != o.Claim.Op.Intent["target"] || report.Host != o.Host || report.OpID != m.ID || report.Incarnation != h.Incarnation { |
| 181 | return ErrIdentity |
| 182 | } |
| 183 | return nil |
| 184 | } |
| 185 | |
| 186 | // StampLostMoveTargetParked uses the same expiring, record-sequenced grant as closure. |
| 187 | func (g *MoveRecoveryGrant) StampLostMoveTargetParked(ctx context.Context) error { |
| 188 | if g == nil || g.state == nil { |
| 189 | return ErrPending |
| 190 | } |
| 191 | g.state.mu.Lock() |
| 192 | defer g.state.mu.Unlock() |
| 193 | o, r, rec, err := g.current(ctx) |
| 194 | if err != nil { |
| 195 | return err |
| 196 | } |
| 197 | m := rec.Move |
| 198 | if m == nil || m.ID != g.o.OpID || m.Incarnation != g.o.Incarnation || m.Rollback == nil || !hostclaim.LostMoveTargetParked(o.Claim, park.PlacementRegion(m.To)) { |
| 199 | return ErrPending |
| 200 | } |
| 201 | if m.Rollback.TargetParkedAt != nil { |
| 202 | return nil |
| 203 | } |
| 204 | if err := provision.MoveLostTargetParked(provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation}, m.Rollback.ID, g.w.now())(&rec); err != nil { |
| 205 | return err |
| 206 | } |
| 207 | body, err := g.prepareRecord(ctx, rec) |
| 208 | if err != nil { |
| 209 | return err |
| 210 | } |
| 211 | _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, r.Holder.Key, r.Body, body, r.ETag) |
| 212 | return err |
| 213 | } |
| 214 | |
amadan