| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "log" |
| 10 | "os" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/moves" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | ) |
| 18 | |
| 19 | // heldStageReadTimeout bounds the bucket reads one authorization makes. |
| 20 | const heldStageReadTimeout = 30 * time.Second |
| 21 | |
| 22 | // heldStageFrozenReadTimeout bounds the one claim read made while the |
| 23 | // registry's write lock is held: well under the registry's 5 s busy_timeout, |
| 24 | // so a SetHold or ReleaseHold waiting on the freeze is delayed, never failed. |
| 25 | const heldStageFrozenReadTimeout = 2 * time.Second |
| 26 | |
| 27 | // heldStageAuthority is the authorize closure the root adopt pass hands |
| 28 | // release.ServeHeldStageRequests (moves design §7.3, T7). A held-stage |
| 29 | // request is honoured only for the move this box is the target of, right |
| 30 | // now, and only for the release the source runs: |
| 31 | // |
| 32 | // - the op is a well-formed move or rollback id (it becomes part of a |
| 33 | // bucket key below); |
| 34 | // - the request names this box; |
| 35 | // - the registry row for the host, when one exists, is held by the |
| 36 | // request's op and names the request's account and app (during |
| 37 | // preparing the target has no row yet, so an absent row is accepted, and |
| 38 | // a row held by another op refuses); |
| 39 | // - the host claim names the op, is moving or landing, targets this box, |
| 40 | // and is held by the request's account and app; |
| 41 | // - the source's report for the op publishes a Release equal to the |
| 42 | // requested version. |
| 43 | // |
| 44 | // Any mismatch is release.ErrStaleHeldStage, so a request queued for an |
| 45 | // operation that was aborted or replaced never repoints live/<host>. A |
| 46 | // read that fails is returned as itself. |
| 47 | // |
| 48 | // With a non-nil then (the publication), the whole check runs again, and |
| 49 | // then, with the registry's holds frozen (WithHoldsFrozen, so no SetHold or |
| 50 | // ReleaseHold can land in between), the hold is checked from the frozen |
| 51 | // rows and the claim is re-read; then runs last, inside the freeze. Staging |
| 52 | // takes minutes, so this is what keeps a move aborted or replaced meanwhile |
| 53 | // from having its release published. The registry allows one connection, |
| 54 | // so inside the freeze only the held rows it hands over are read: a row the |
| 55 | // first check found held must still be held by the op there, and a row |
| 56 | // that appeared since must be held by the op too. An unheld row appearing |
| 57 | // where there was none would mean the move finished, which the claim |
| 58 | // re-read refuses. |
| 59 | // |
| 60 | // Inside the freeze, then also holds the host's held-stage lock under |
| 61 | // dataDir (withHeldStageLock), so T7's reading of the live link waits out a |
| 62 | // publication and its rollback. |
| 63 | func heldStageAuthority(reg *registry.Registry, store release.Store, box, dataDir string) func(release.HeldStageRequest, func() error) error { |
| 64 | return func(req release.HeldStageRequest, then func() error) error { |
| 65 | rowHeld, err := heldStageCheck(reg, store, box, req) |
| 66 | if err != nil || then == nil { |
| 67 | return err |
| 68 | } |
| 69 | heldStageBeforeFreeze() |
| 70 | return reg.WithHoldsFrozen(func(held []registry.Route) error { |
| 71 | var row *registry.Route |
| 72 | for i := range held { |
| 73 | if held[i].Host == req.Host { |
| 74 | row = &held[i] |
| 75 | } |
| 76 | } |
| 77 | switch { |
| 78 | case row != nil: |
| 79 | if err := heldStageRow(*row, req); err != nil { |
| 80 | return err |
| 81 | } |
| 82 | case rowHeld: |
| 83 | return staleHeldStage("the registry row is no longer held by %s", req.Op) |
| 84 | } |
| 85 | ctx, cancel := context.WithTimeout(context.Background(), heldStageFrozenReadTimeout) |
| 86 | defer cancel() |
| 87 | obs, err := hostclaim.Read(ctx, store, req.Host) |
| 88 | if err != nil { |
| 89 | return fmt.Errorf("host claim for %s: %w", req.Host, err) |
| 90 | } |
| 91 | if err := heldStageClaim(obs, req); err != nil { |
| 92 | return err |
| 93 | } |
| 94 | // The publication (link read, bind, swap, re-hash, rollback) |
| 95 | // holds the host's held-stage lock, which T7's reading takes |
| 96 | // too (final review finding 1). Taken last, inside the freeze: |
| 97 | // it is a leaf of the lock order. |
| 98 | return withHeldStageLock(dataDir, req.Host, then) |
| 99 | }) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // heldStageBeforeFreeze runs between the full check and the freeze. It is |
| 104 | // a test seam: the window a hold or claim change can land in. |
| 105 | var heldStageBeforeFreeze = func() {} |
| 106 | |
| 107 | func staleHeldStage(format string, args ...any) error { |
| 108 | return fmt.Errorf("%w: "+format, append([]any{release.ErrStaleHeldStage}, args...)...) |
| 109 | } |
| 110 | |
| 111 | // heldStageCheck is the authority's full check, outside any freeze. It |
| 112 | // reports whether the host has a registry row (held by the op, or the check |
| 113 | // would have refused). |
| 114 | func heldStageCheck(reg *registry.Registry, store release.Store, box string, req release.HeldStageRequest) (bool, error) { |
| 115 | if !moves.ValidOpID(req.Op) { |
| 116 | return false, staleHeldStage("malformed op id") |
| 117 | } |
| 118 | if box == "" || req.Box != box { |
| 119 | return false, staleHeldStage("it names box %q and this is %q", req.Box, box) |
| 120 | } |
| 121 | rt, err := reg.Get(req.Host) |
| 122 | if err != nil { |
| 123 | return false, fmt.Errorf("registry row for %s: %w", req.Host, err) |
| 124 | } |
| 125 | if rt != nil { |
| 126 | if err := heldStageRow(*rt, req); err != nil { |
| 127 | return false, err |
| 128 | } |
| 129 | } |
| 130 | ctx, cancel := context.WithTimeout(context.Background(), heldStageReadTimeout) |
| 131 | defer cancel() |
| 132 | obs, err := hostclaim.Read(ctx, store, req.Host) |
| 133 | if err != nil { |
| 134 | return false, fmt.Errorf("host claim for %s: %w", req.Host, err) |
| 135 | } |
| 136 | if err := heldStageClaim(obs, req); err != nil { |
| 137 | return false, err |
| 138 | } |
| 139 | raw, _, err := store.Get(ctx, moves.SourceReportKey(moves.ReportPrefix(req.Account, req.App, req.Host, req.Op))) |
| 140 | if errors.Is(err, os.ErrNotExist) { |
| 141 | return false, staleHeldStage("the source has not reported its release") |
| 142 | } |
| 143 | if err != nil { |
| 144 | return false, fmt.Errorf("source report for %s: %w", req.Host, err) |
| 145 | } |
| 146 | var sr moves.SourceReport |
| 147 | if err := json.Unmarshal(raw, &sr); err != nil { |
| 148 | return false, fmt.Errorf("source report for %s: %w", req.Host, err) |
| 149 | } |
| 150 | if sr.Release == "" || sr.Release != req.Version { |
| 151 | return false, staleHeldStage("the source runs %q", sr.Release) |
| 152 | } |
| 153 | return rt != nil, nil |
| 154 | } |
| 155 | |
| 156 | // heldStageRow admits a registry row: held by the request's op, of the |
| 157 | // request's account and app. |
| 158 | func heldStageRow(rt registry.Route, req release.HeldStageRequest) error { |
| 159 | if rt.HoldOp != req.Op { |
| 160 | return staleHeldStage("the registry row is held by %q", rt.HoldOp) |
| 161 | } |
| 162 | if release.RouteAccount(rt) != req.Account || rt.App != req.App { |
| 163 | return staleHeldStage("the registry row is %s/%s", release.RouteAccount(rt), rt.App) |
| 164 | } |
| 165 | return nil |
| 166 | } |
| 167 | |
| 168 | // heldStageClaim admits a host claim: it names the op, is moving or |
| 169 | // landing, targets the request's box, and is held by its account and app. |
| 170 | func heldStageClaim(obs hostclaim.Observed, req release.HeldStageRequest) error { |
| 171 | if obs.Absent || obs.Claim == nil { |
| 172 | return staleHeldStage("the host has no claim") |
| 173 | } |
| 174 | op := obs.Claim.Op |
| 175 | if op.ID != req.Op { |
| 176 | return staleHeldStage("the claim names op %q", op.ID) |
| 177 | } |
| 178 | if op.Kind != hostclaim.KindMoving && op.Kind != hostclaim.KindLanding { |
| 179 | return staleHeldStage("the claim's op is %s", op.Kind) |
| 180 | } |
| 181 | if op.Intent["target"] != req.Box { |
| 182 | return staleHeldStage("the claim's target is %q", op.Intent["target"]) |
| 183 | } |
| 184 | if h := obs.Claim.Holder; h == nil || h.Account != req.Account || h.App != req.App { |
| 185 | return staleHeldStage("the claim is not held by %s/%s", req.Account, req.App) |
| 186 | } |
| 187 | return nil |
| 188 | } |
| 189 | |
| 190 | // heldStagePass serves the held-stage requests the sandboxed agent left |
| 191 | // for this root pass. A no-op that reads nothing unless a request exists, |
| 192 | // and none exist while CARLOS_MOVES_PHASE1 is off: only a move's target |
| 193 | // writes one. Never fatal. |
| 194 | // |
| 195 | // D3-held: each request passes D3 as adoption would, through a fresh gate |
| 196 | // (this is a fresh root process, as for the adoption pass before it) whose |
| 197 | // bake directory is the healthy history that pass reads. |
| 198 | // |
| 199 | // Settings spec rev 11 §4.3a: each request stages only a build §4.1 |
| 200 | // authorizes, a signed promotion record or a pointer of the app verified |
| 201 | // under pub and admitted by r, root's pointer digest ratchet (the one |
| 202 | // adoption uses). Anything else is dropped before a byte is downloaded. |
| 203 | func heldStagePass(ctx context.Context, dataDir, optDir string, pub ed25519.PublicKey, r *release.DigestRatchet, reg *registry.Registry, store release.Store, box string) { |
| 204 | proof := release.NewRootProofGate(release.ProofPolicyPath, dataDir) |
| 205 | if err := release.ServeHeldStageRequestsWithPins(ctx, store, pub, r, optDir, dataDir, proof, heldStageAuthority(reg, store, box, dataDir), productionRootGenerationPins()); err != nil { |
| 206 | log.Printf("adopt: held stage requests: %v", err) |
| 207 | } |
| 208 | retireEndedHeldRefusals(ctx, store, dataDir, box) |
| 209 | } |
| 210 | |
| 211 | // retireEndedHeldRefusals is root's half of a held refusal's lifecycle |
| 212 | // (D3-held): every refusal record a move's held stage left (the held |
| 213 | // namespace, release.HeldRefusalBase) whose move the host claim no longer |
| 214 | // names as moving or landing into this box is removed, so an aborted or |
| 215 | // abandoned move leaves nothing behind. |
| 216 | // The agent retires the ones it can (movesWorker.retireHeldRefusals), but |
| 217 | // not a record root wrote. A claim that cannot be read keeps the record for |
| 218 | // the next pass. Adoption refusals are never touched, and no claim is read |
| 219 | // unless a held refusal exists. |
| 220 | func retireEndedHeldRefusals(ctx context.Context, store release.Store, dataDir, box string) { |
| 221 | retired, err := release.RetireHeldRefusals(dataDir, func(r release.Refusal) bool { |
| 222 | rctx, cancel := context.WithTimeout(ctx, heldStageReadTimeout) |
| 223 | defer cancel() |
| 224 | obs, err := hostclaim.Read(rctx, store, r.Host) |
| 225 | if err != nil { |
| 226 | return true |
| 227 | } |
| 228 | if obs.Absent || obs.Claim == nil { |
| 229 | return false |
| 230 | } |
| 231 | op := obs.Claim.Op |
| 232 | return op.ID == r.MoveOp && (op.Kind == hostclaim.KindMoving || op.Kind == hostclaim.KindLanding) && op.Intent["target"] == box |
| 233 | }) |
| 234 | for _, host := range retired { |
| 235 | log.Printf("adopt: %s: retired the held refusal of a move that is over", host) |
| 236 | } |
| 237 | if err != nil { |
| 238 | log.Printf("adopt: held refusals: %v", err) |
| 239 | } |
| 240 | } |
| 241 | |
amadan