| 1 | // Package provision owns instance provisioning as a control-plane data |
| 2 | // write: the bucket record and app manifest shapes, their validation, the |
| 3 | // store operations the console's instance verbs use, and the route |
| 4 | // conventions shared with `carlos add`. The reconcile pass that turns |
| 5 | // records into routes lives in reconcile.go (Task 3). Spec: |
| 6 | // docs/superpowers/specs/2026-08-04-instance-provisioning-design.md. |
| 7 | package provision |
| 8 | |
| 9 | import ( |
| 10 | "context" |
| 11 | "crypto/sha256" |
| 12 | "encoding/hex" |
| 13 | "encoding/json" |
| 14 | "errors" |
| 15 | "fmt" |
| 16 | "log" |
| 17 | "net" |
| 18 | "os" |
| 19 | "path/filepath" |
| 20 | "regexp" |
| 21 | "sort" |
| 22 | "strconv" |
| 23 | "strings" |
| 24 | "time" |
| 25 | |
| 26 | "github.com/carlosframework/platform/internal/controlname" |
| 27 | "github.com/carlosframework/platform/internal/controlsock" |
| 28 | "github.com/carlosframework/platform/internal/registry" |
| 29 | "github.com/carlosframework/platform/internal/release" |
| 30 | "golang.org/x/crypto/bcrypt" |
| 31 | ) |
| 32 | |
| 33 | // Record is the wire shape at apps/<sqid>/<app>/instances/<host>.json — one |
| 34 | // per provisioned instance, create-only so a lost race is a 409 and never a |
| 35 | // silent overwrite. The body must agree with its own key path (Host/App/ |
| 36 | // Account vs the key's segments); GetRecord and ListRecords enforce that, a |
| 37 | // record that lies about identity is never guessed at (design §3.2). |
| 38 | type Record struct { |
| 39 | // RecordSeq is advanced by every record writer once present; absent is legacy zero. |
| 40 | RecordSeq uint64 `json:"record_seq,omitempty"` |
| 41 | Acquisition string `json:"acq,omitempty"` |
| 42 | // Maintenance is preserved by ordinary writers; only a future guarded |
| 43 | // maintenance transition may install, replace, or clear it. |
| 44 | Maintenance MaintenanceProjection `json:"maintenance,omitempty"` |
| 45 | Schema int `json:"schema"` // 1 |
| 46 | Host string `json:"host"` |
| 47 | Slug string `json:"slug"` |
| 48 | App string `json:"app"` |
| 49 | Account string `json:"account"` |
| 50 | Pool string `json:"pool,omitempty"` |
| 51 | Region string `json:"region,omitempty"` |
| 52 | Channel string `json:"channel"` |
| 53 | CreatedAt time.Time `json:"created_at"` |
| 54 | CreatedBy string `json:"created_by,omitempty"` |
| 55 | |
| 56 | // Move is an in-flight residence move (issue #331, instance-move spec): |
| 57 | // present only between move-prepare and the cut's terminal flip, absent |
| 58 | // on every record at rest. Additive + omitempty — pre-move records parse |
| 59 | // unchanged. Written ONLY through SetRecordMove's guarded transitions. |
| 60 | Move *MoveState `json:"move,omitempty"` |
| 61 | |
| 62 | // LastMove is the terminal outcome the close CAS leaves as it clears |
| 63 | // Move (index D11), so a CLI that missed the intermediate states still |
| 64 | // reads how the move it requested ended. Written by the guarded close |
| 65 | // transitions only (MoveGuardedAbortComplete, and the done and |
| 66 | // rolled_back closes the later plans add). Additive and omitempty: |
| 67 | // absent on every record at rest on main. |
| 68 | LastMove *MoveOutcome `json:"last_move,omitempty"` |
| 69 | |
| 70 | // ActiveRun is the instance's `carlos run`, if any (run spec §5.1): |
| 71 | // queued, picked or cancelling while open, then done, cancelled or |
| 72 | // expired until the console clears it. Runs, moves and imports all |
| 73 | // change this one object, so they serialize on its ETag. Ordinary |
| 74 | // writers carry it through unchanged; only the Run* transitions |
| 75 | // (run.go) write it. Additive and omitempty. |
| 76 | ActiveRun *RunState `json:"run,omitempty"` |
| 77 | |
| 78 | // RunKeys is the instance's recent keyed run admissions (service runs |
| 79 | // spec § 3): each Idempotency-Key, scoped to its caller and hashed, |
| 80 | // with the run it admitted. RunAdmit writes it in the same |
| 81 | // compare-and-swap as the admission, so admission and dedupe are one |
| 82 | // write; every other writer carries it through unchanged. Additive and |
| 83 | // omitempty. A writer built before it drops it, which is why keyed |
| 84 | // creates wait for CARLOS_RUN_KEYS=on. |
| 85 | RunKeys []RunKey `json:"run_keys,omitempty"` |
| 86 | |
| 87 | // Seed is where this instance's database started (seeded-instances spec |
| 88 | // §3.2): a sibling's replica (--from) or an uploaded snapshot (import). |
| 89 | // Additive and omitempty. Like Maintenance, it is preserved by ordinary |
| 90 | // writers and changed only through the named seed transitions (seed.go); |
| 91 | // it stays on the record after placement as provenance. |
| 92 | Seed *SeedSource `json:"seed,omitempty"` |
| 93 | |
| 94 | // --- shape fields (2026-08-10 route-records design §3) -------------- |
| 95 | // Every one is omitempty with a zero value that reproduces the mint |
| 96 | // reconcile hardcoded before they existed: a record written by an older |
| 97 | // console still materializes a byte-identical route. |
| 98 | |
| 99 | // Kind is the route class: "instance" | "service" | "static". Empty |
| 100 | // means "instance", which is what reconcile minted unconditionally |
| 101 | // before this field existed. |
| 102 | Kind string `json:"kind,omitempty"` |
| 103 | // Addr is a TCP upstream as host:port, for whole-app tenants whose |
| 104 | // process the platform does not spawn (kass, keymail, titogo, the |
| 105 | // consoles). Set means the route proxies to it and derives no socket. |
| 106 | Addr string `json:"addr,omitempty"` |
| 107 | // Unit is the systemd unit that owns this route's process, so adoption |
| 108 | // can cycle it (console.service — issue #110). See registry.Route.Unit |
| 109 | // for why a typo here is a route whose adoption silently restarts |
| 110 | // nothing; ValidateShape is the door that refuses one. |
| 111 | Unit string `json:"unit,omitempty"` |
| 112 | // Hibernate is a POINTER because absent and false are different facts: |
| 113 | // every record already in a production bucket has no such key, and a |
| 114 | // plain bool would read those as false and un-hibernate the fleet on |
| 115 | // the first pass after deploy. nil = HibernateOrDefault's derivation. |
| 116 | Hibernate *bool `json:"hibernate,omitempty"` |
| 117 | // Environment is the named config bundle this route binds to ("" = the |
| 118 | // app's default bundle alone). The second member-facing axis of |
| 119 | // `carlos route`, and owned by this record for the same reason Channel |
| 120 | // is (design §5). |
| 121 | Environment string `json:"environment,omitempty"` |
| 122 | // StatusNamed opts this app into being named on the public status page. |
| 123 | // A POINTER for the same reason Hibernate is: this column is CONVERGED, |
| 124 | // so absent must mean "this record does not speak to the status page" — |
| 125 | // a plain bool would read every legacy record (and every record whose |
| 126 | // writer left the field out) as an explicit false and silently opt out a |
| 127 | // route an operator opted in with `carlos add -status-named`. nil = |
| 128 | // StatusNamedOrDefault's answer. |
| 129 | StatusNamed *bool `json:"status_named,omitempty"` |
| 130 | |
| 131 | // --- multi-upstream fields (2026-08-12, HA design, issue #152) -------- |
| 132 | |
| 133 | // Addrs is a route's upstream set, host:port each, for a tenant fronting |
| 134 | // more than one process. Mutually exclusive with Addr — a record picks |
| 135 | // one shape and ValidateShape refuses both set. Empty means "no |
| 136 | // upstreams of its own" (AddrsOrDefault folds Addr in for the legacy |
| 137 | // single-upstream case). |
| 138 | Addrs []string `json:"addrs,omitempty"` |
| 139 | // Health is the route's HTTP probe config. nil means unset; |
| 140 | // HealthOrDefault derives the answer (probed once Addrs names more than |
| 141 | // one upstream, unprobed otherwise — today's behavior for every record |
| 142 | // that predates this field). Wire key is "probe", not "health": the |
| 143 | // console's instance-list API already ships "health" for a computed |
| 144 | // one-word status string (internal/console/apiinstances.go's |
| 145 | // instanceListRecord.Health) that a deployed CLI parses — that surface |
| 146 | // is shipped and keeps the name, this field is new and yields it. |
| 147 | Health *Health `json:"probe,omitempty"` |
| 148 | |
| 149 | // --- steering fields (2026-08-23, multi-region ingress design Part 4) -- |
| 150 | |
| 151 | // Steering is this route's DNS steering opt-in: "" (default — today's |
| 152 | // static per-host answer, and what every record written before this |
| 153 | // field existed reads as) or "latency" (a Route53 latency record per |
| 154 | // armed edge, converged by the steering housekeeper duty — Tasks 4-6 of |
| 155 | // the multi-region-steering plan). A STRING, deliberately, not a bool: |
| 156 | // Part 4 names other steering policies as future work, and a string |
| 157 | // leaves room for a new value to land without another wire-shape break. |
| 158 | // ValidSteering is the door. |
| 159 | // |
| 160 | // SINGLE-BOX INERT as of this field's introduction (plan Task 2): |
| 161 | // nothing reads it yet. It reaches the registry row (routeFor, |
| 162 | // convergeOwned) and nothing further — the console door (Task 3) is |
| 163 | // what lets a member set it, and the desired-state builder and Route53 |
| 164 | // converger (Tasks 4-6) are what give a non-empty value any effect, and |
| 165 | // even then only once a deployment arms CARLOS_STEERING_ZONE_ID. |
| 166 | Steering string `json:"steering,omitempty"` |
| 167 | |
| 168 | // Replica is NOT a separate record field: Role carries the role |
| 169 | // vocabulary alone (ValidRole's door), and every reader derives the |
| 170 | // boolean from it. A second spelling of the same fact is two chances |
| 171 | // to disagree. |
| 172 | // |
| 173 | // Gate is the bcrypt hash of the host's shared password ("" = open, |
| 174 | // the value every record written before this field existed decodes |
| 175 | // to). The ONLY place the secret material lives in the bucket; the |
| 176 | // plaintext crosses the console once, at the door that hashes it |
| 177 | // (internal/gate.Hash), and is never stored, logged or echoed — the |
| 178 | // console's record views shadow this field (apiinstances.go). A |
| 179 | // string, not a pointer: absent and "" are the same fact, an open |
| 180 | // host. Converged into routes.gate by Reconcile's owned-converge arm |
| 181 | // (reconcile.go) and read by the edge on every request. Spec docs/superpowers/specs/2026-09-07-password-gate-design.md. |
| 182 | Gate string `json:"gate,omitempty"` |
| 183 | |
| 184 | // --- replica fields (carlos/platform discussion #162, edge checkout) --- |
| 185 | |
| 186 | // Role is what this instance IS within its app: "" (the default every |
| 187 | // record written before this field existed decodes to) is the serving |
| 188 | // home — the instance the account's hosts resolve to, the one whose |
| 189 | // data is authoritative. "replica" is a SECOND instance of the same |
| 190 | // account, holding a one-way mirror of it, placed near the buyers: an |
| 191 | // edge-checkout shard selling against a capacity lease from home and |
| 192 | // draining orders back in batches (titogo edge-checkout design |
| 193 | // record, 2026-09-21). The role is NOT a routing fact by itself — it |
| 194 | // takes effect through Replicas (below) and the reconciler's mint |
| 195 | // gate — and ValidRole is the door. |
| 196 | // |
| 197 | // Written only by the create door that also names the placement; no |
| 198 | // PATCH axis flips a record into or out of the replica role in place |
| 199 | // (delete and re-create is the supported transition), because a role |
| 200 | // flip is a data-plane event — which database is authoritative — not |
| 201 | // a shape edit. |
| 202 | Role string `json:"role,omitempty"` |
| 203 | |
| 204 | // Replicas names the placement codes an account's replicas live in: |
| 205 | // the home record's own opt-in, "ie,uk" style, deduplicated, |
| 206 | // canonically sorted and validated against ValidRegion. Empty on |
| 207 | // every record that predates the field = no replicas anywhere = |
| 208 | // today's routing, byte for byte. It is CONVERGED (Reconcile writes |
| 209 | // the record's value over the row's on every pass, through |
| 210 | // SetReplicas — the one deliberate exception to that arm's |
| 211 | // never-widen rule, because replica placement is the bucket's |
| 212 | // decision to own: a stale placement left behind after the set |
| 213 | // shrank would keep a retired region's replica on the account's |
| 214 | // hosts forever), published by the edges file, honoured by the |
| 215 | // steering converger — and never read by the edge's own dispatch, |
| 216 | // which never hairpins a replica. |
| 217 | Replicas []string `json:"replicas,omitempty"` |
| 218 | |
| 219 | // --- update settings (instance update settings spec §3.1, §4.2) ----- |
| 220 | // Every one is omitempty with "never set" as its zero value, so a record |
| 221 | // written before them decodes with none and reconcile resolves it to |
| 222 | // auto: today's adoption. Set only through the console's /updates doors. |
| 223 | |
| 224 | // Updates is "auto", "manual", or "" (never set: the app default). |
| 225 | Updates string `json:"updates,omitempty"` |
| 226 | // Major holds the instance on one semver major of stable. nil: never set. |
| 227 | Major *int `json:"major,omitempty"` |
| 228 | // Pin is the exact build to run. It must have a promotion record. |
| 229 | Pin string `json:"pin,omitempty"` |
| 230 | // InstallRequest is "Update now": an id that only ever increases. Root |
| 231 | // resolves and completes it (plan 4); the console never chooses a build. |
| 232 | InstallRequest *InstallRequest `json:"install_request,omitempty"` |
| 233 | } |
| 234 | |
| 235 | // Health is the per-route probe config (HA spec Part II). Zero value = |
| 236 | // unset; WithDefaults fills in the platform's defaults for whatever the |
| 237 | // caller left at zero. |
| 238 | type Health struct { |
| 239 | Path string `json:"path,omitempty"` // default "/" |
| 240 | Status int `json:"status,omitempty"` // default 200 |
| 241 | HostHeader string `json:"host_header,omitempty"` // default: the route host |
| 242 | IntervalMS int `json:"interval_ms,omitempty"` // default 2000 |
| 243 | FailAfter int `json:"fail_after,omitempty"` // default 3 |
| 244 | RiseAfter int `json:"rise_after,omitempty"` // default 2 |
| 245 | } |
| 246 | |
| 247 | // WithDefaults fills the five zero-valued fields above with the platform's |
| 248 | // defaults. HostHeader is deliberately left as-is (including empty) — the |
| 249 | // edge substitutes the route's own host for an unset HostHeader, and that |
| 250 | // substitution needs to see "unset", not a value this method invented. |
| 251 | func (h Health) WithDefaults() Health { |
| 252 | if h.Path == "" { |
| 253 | h.Path = "/" |
| 254 | } |
| 255 | if h.Status == 0 { |
| 256 | h.Status = 200 |
| 257 | } |
| 258 | if h.IntervalMS == 0 { |
| 259 | h.IntervalMS = 2000 |
| 260 | } |
| 261 | if h.FailAfter == 0 { |
| 262 | h.FailAfter = 3 |
| 263 | } |
| 264 | if h.RiseAfter == 0 { |
| 265 | h.RiseAfter = 2 |
| 266 | } |
| 267 | return h |
| 268 | } |
| 269 | |
| 270 | // Manifest is the wire shape at apps/<sqid>/<app>/instances.json — the app's |
| 271 | // opt-in to instance provisioning and the policy every record is checked |
| 272 | // against (host suffix, region allow-list, seed argv). Absence means the |
| 273 | // feature does not exist for that app, the same convention litestream's own |
| 274 | // manifest uses (design §3.1). |
| 275 | type Manifest struct { |
| 276 | Schema int `json:"schema"` // 1 |
| 277 | App string `json:"app"` |
| 278 | Domain string `json:"domain"` |
| 279 | Pool string `json:"pool,omitempty"` |
| 280 | Channel string `json:"channel,omitempty"` |
| 281 | Regions []string `json:"regions,omitempty"` |
| 282 | Seed []string `json:"seed,omitempty"` |
| 283 | |
| 284 | // PrivateStorage is the app's standing policy that every instance |
| 285 | // created under this manifest gets a config environment of its own and |
| 286 | // an object store bound to that environment (issue #412, spec |
| 287 | // 2026-09-22-instance-private-storage-design.md §5). Declared once, by |
| 288 | // a signed-in person, at `carlos instances enable --private-storage`; |
| 289 | // spent by the create door, which an `instance`-role service credential |
| 290 | // can already reach. That split is the point: the policy widens no |
| 291 | // role, so a signup path provisions a tenant's bucket without ever |
| 292 | // holding the power to create one. |
| 293 | // |
| 294 | // A plain bool, not a pointer, and the difference from Record's |
| 295 | // Hibernate and StatusNamed is worth stating because their reason does |
| 296 | // not apply here. Those are CONVERGED columns — a box re-reads them |
| 297 | // every pass, so "absent" has to be distinguishable from an explicit |
| 298 | // false or the first pass after deploy acts on a value nobody wrote. |
| 299 | // This is read at one door, at create time, and false IS the behaviour |
| 300 | // every manifest in a production bucket already has. There is nothing |
| 301 | // for absence to mean that false does not already mean. |
| 302 | PrivateStorage bool `json:"private_storage,omitempty"` |
| 303 | |
| 304 | // Moves is the app's move declaration (move-any-app design §7.3 S1): |
| 305 | // the readiness path a probe requests. Absent means "/". |
| 306 | Moves *MovesManifest `json:"moves,omitempty"` |
| 307 | |
| 308 | // Updates is the app's instance update policy (spec §3.2). nil: no |
| 309 | // instance may change anything itself, and the defaults are auto, this |
| 310 | // manifest's Channel, and no major hold. |
| 311 | Updates *UpdatePolicy `json:"updates,omitempty"` |
| 312 | } |
| 313 | |
| 314 | type MovesManifest struct { |
| 315 | ReadyPath string `json:"ready_path,omitempty"` |
| 316 | } |
| 317 | |
| 318 | // ReadyPathOrDefault is the probe's path (§7.7). |
| 319 | func (m Manifest) ReadyPathOrDefault() string { |
| 320 | if m.Moves != nil && m.Moves.ReadyPath != "" { |
| 321 | return m.Moves.ReadyPath |
| 322 | } |
| 323 | return "/" |
| 324 | } |
| 325 | |
| 326 | // ManifestKey, RecordsPrefix and RecordKey are the bucket key conventions |
| 327 | // for a provisioned app — beside release.AppPrefix's channels/ and |
| 328 | // releases/, matching the ledgers.json precedent of a per-app manifest |
| 329 | // sitting beside the collection it governs. |
| 330 | func ManifestKey(acct, app string) string { return release.AppPrefix(acct, app) + "instances.json" } |
| 331 | func RecordsPrefix(acct, app string) string { return release.AppPrefix(acct, app) + "instances/" } |
| 332 | func RecordKey(acct, app, host string) string { |
| 333 | return RecordsPrefix(acct, app) + host + ".json" |
| 334 | } |
| 335 | |
| 336 | // ParseRecordKey reports whether key is an account-qualified instance |
| 337 | // record (apps/<sqid>/<app>/instances/<host>.json) known to belong to a |
| 338 | // materialized account, and if so its account, app and host. Mirrors |
| 339 | // domains.appKeyOwner in miniature: known guards against a key whose |
| 340 | // account has not materialized yet, and app is refused when it collides |
| 341 | // with the release plane's own words ("channels", "releases") — the same |
| 342 | // two-namespace collision appKeyOwner documents. A key with anything but |
| 343 | // exactly one non-empty segment after "instances/" is not a record: the |
| 344 | // manifest (instances.json, no trailing segment) and any nested path both |
| 345 | // refuse. |
| 346 | func ParseRecordKey(key string, known map[string]bool) (acct, app, host string, ok bool) { |
| 347 | rest, ok := strings.CutPrefix(key, "apps/") |
| 348 | if !ok { |
| 349 | return "", "", "", false |
| 350 | } |
| 351 | acct, rest, ok = strings.Cut(rest, "/") |
| 352 | if !ok || !known[acct] { |
| 353 | return "", "", "", false |
| 354 | } |
| 355 | app, rest, ok = strings.Cut(rest, "/") |
| 356 | if !ok || app == "" || app == "channels" || app == "releases" { |
| 357 | return "", "", "", false |
| 358 | } |
| 359 | seg, rest, ok := strings.Cut(rest, "/") |
| 360 | if !ok || seg != "instances" { |
| 361 | return "", "", "", false |
| 362 | } |
| 363 | if rest == "" || strings.Contains(rest, "/") { |
| 364 | return "", "", "", false |
| 365 | } |
| 366 | host, ok = strings.CutSuffix(rest, ".json") |
| 367 | if !ok || host == "" { |
| 368 | return "", "", "", false |
| 369 | } |
| 370 | return acct, app, host, true |
| 371 | } |
| 372 | |
| 373 | // ValidHost reports whether host is exactly one label followed by |
| 374 | // "."+domain. This is path-traversal defense stated as such: the host |
| 375 | // becomes a bucket key segment (RecordKey) and, on the box, the |
| 376 | // <host>.sock / <host>.db filesystem path segments — the label regex is |
| 377 | // what keeps a hostile host from ever reaching either as anything but an |
| 378 | // opaque, safe path component. |
| 379 | func ValidHost(host, domain string) bool { |
| 380 | if domain == "" || len(host) > 253 { |
| 381 | return false |
| 382 | } |
| 383 | label, ok := strings.CutSuffix(host, "."+domain) |
| 384 | if !ok || label == "" { |
| 385 | return false |
| 386 | } |
| 387 | return controlname.ValidLabel(label) |
| 388 | } |
| 389 | |
| 390 | // ValidHostShape checks host's shape alone, with no domain to validate a |
| 391 | // suffix against — for doors that must still refuse a hostile host after |
| 392 | // its app's manifest (and so its domain) is gone: a delete, for instance, |
| 393 | // which must work even if the app disabled instances since the record was |
| 394 | // created. |
| 395 | func ValidHostShape(host string) bool { return controlname.ValidHostShape(host) } |
| 396 | |
| 397 | // KindOrDefault resolves the record's route class. "" means "instance" — |
| 398 | // the class reconcile minted unconditionally before Kind existed, and so |
| 399 | // the only value that keeps a legacy record materializing unchanged. |
| 400 | func (r Record) KindOrDefault() string { |
| 401 | if r.Kind == "" { |
| 402 | return "instance" |
| 403 | } |
| 404 | return r.Kind |
| 405 | } |
| 406 | |
| 407 | // DerivesPaths reports whether this record's route builds filesystem paths |
| 408 | // out of its host — the <host>.sock and <host>.db segments. It is the hinge |
| 409 | // two separate rules turn on: which host validator applies (ValidRecordHost: |
| 410 | // a host that becomes a path segment stays suffix-pinned, one that does not |
| 411 | // needs only a valid shape), and what hibernation defaults to (only a route |
| 412 | // with a socket has a process for the activator to wake). |
| 413 | func (r Record) DerivesPaths() bool { |
| 414 | return r.KindOrDefault() == "instance" && len(r.AddrsOrDefault()) == 0 |
| 415 | } |
| 416 | |
| 417 | // AddrsOrDefault resolves the record's upstream set: Addrs if set, else the |
| 418 | // single-element fold of the legacy Addr field, else nil. Every caller that |
| 419 | // used to branch on Addr != "" reads this instead, so a record minted |
| 420 | // before Addrs existed proxies to exactly the upstream it always did. |
| 421 | func (r Record) AddrsOrDefault() []string { |
| 422 | if len(r.Addrs) > 0 { |
| 423 | return r.Addrs |
| 424 | } |
| 425 | if r.Addr != "" { |
| 426 | return []string{r.Addr} |
| 427 | } |
| 428 | return nil |
| 429 | } |
| 430 | |
| 431 | // HealthOrDefault resolves the record's probe config: Health if set, else |
| 432 | // the platform defaults once AddrsOrDefault names more than one upstream |
| 433 | // (a route fronting several processes needs to know which ones are up), |
| 434 | // else nil. A single-upstream record — Addrs unset, Addr alone or one |
| 435 | // element of Addrs — stays unprobed, the behavior every record had before |
| 436 | // Health existed. |
| 437 | func (r Record) HealthOrDefault() *Health { |
| 438 | if r.Health != nil { |
| 439 | return r.Health |
| 440 | } |
| 441 | if len(r.AddrsOrDefault()) > 1 { |
| 442 | h := Health{}.WithDefaults() |
| 443 | return &h |
| 444 | } |
| 445 | return nil |
| 446 | } |
| 447 | |
| 448 | // HibernateOrDefault resolves the tri-state. An explicit value wins. Absent |
| 449 | // derives: a socket-backed instance hibernates (the activator model, and |
| 450 | // what reconcile minted unconditionally before this field existed, so every |
| 451 | // legacy record is unchanged), and everything else does not — a static |
| 452 | // route has no process to wake, and an addr-backed tenant is always on. |
| 453 | func (r Record) HibernateOrDefault() bool { |
| 454 | if r.Hibernate != nil { |
| 455 | return *r.Hibernate |
| 456 | } |
| 457 | return r.DerivesPaths() |
| 458 | } |
| 459 | |
| 460 | // StatusNamedOrDefault resolves the tri-state. An explicit value wins. |
| 461 | // Absent is false — the status page's naming is an opt-in, and false is what |
| 462 | // reconcile minted unconditionally before this field existed, so every legacy |
| 463 | // record is unchanged. Absent and false are still different facts, and the |
| 464 | // difference is load-bearing at the converge site: absent leaves the row's |
| 465 | // column alone (an operator's `carlos add -status-named` survives), while an |
| 466 | // explicit false clears it. |
| 467 | func (r Record) StatusNamedOrDefault() bool { |
| 468 | if r.StatusNamed != nil { |
| 469 | return *r.StatusNamed |
| 470 | } |
| 471 | return false |
| 472 | } |
| 473 | |
| 474 | // ValidKind pins the record's route-class vocabulary to the set `carlos |
| 475 | // add` documents. "status" is deliberately absent: the fleet status page's |
| 476 | // route is the platform's own, not an app's, and nothing should be able to |
| 477 | // mint one from an app-scoped record. |
| 478 | func ValidKind(kind string) bool { |
| 479 | switch kind { |
| 480 | case "", "instance", "service", "static": |
| 481 | return true |
| 482 | } |
| 483 | return false |
| 484 | } |
| 485 | |
| 486 | // ValidSteering pins the record's DNS-steering vocabulary (multi-region |
| 487 | // ingress design Part 4): "" (no opinion — a route's host resolves exactly |
| 488 | // as it does today) or "latency" (opt into a Route53 latency record per |
| 489 | // armed edge). Case-sensitive and closed, the same discipline ValidKind |
| 490 | // applies to its own vocabulary — "Latency" or "geo" are refused at this |
| 491 | // door rather than discovered as a steering record the Route53 converger |
| 492 | // (Tasks 4-6) silently ignores. |
| 493 | func ValidSteering(s string) bool { |
| 494 | switch s { |
| 495 | case "", "latency": |
| 496 | return true |
| 497 | } |
| 498 | return false |
| 499 | } |
| 500 | |
| 501 | // ValidRole pins the record's role vocabulary (#162): "" (the serving |
| 502 | // home — every record written before the field existed decodes to it, |
| 503 | // which is what makes the field additive) or "replica" (a one-way-mirror |
| 504 | // instance of the same account, selling near the buyer). Case-sensitive |
| 505 | // and closed, ValidSteering's discipline: a value a future reader has |
| 506 | // never heard of is refused where it is typed rather than discovered as a |
| 507 | // row nothing routes for. |
| 508 | func ValidRole(s string) bool { |
| 509 | switch s { |
| 510 | case "", "replica": |
| 511 | return true |
| 512 | } |
| 513 | return false |
| 514 | } |
| 515 | |
| 516 | // MaxReplicas caps the placement set one record may name. See validReplicas. |
| 517 | const MaxReplicas = 8 |
| 518 | |
| 519 | // ValidReplicas normalizes and validates a replica placement set: trim, |
| 520 | // drop empties, dedupe, sort (the canonical order every comma-joined |
| 521 | // column here keeps), and check each element against ValidRegion — the |
| 522 | // same vocabulary a record's own Region speaks, so "is this placement |
| 523 | // armed" stays one question with one answer. Exported because the console's |
| 524 | // create door canonicalizes the request body through the SAME derivation |
| 525 | // the reconciler re-checks at (neither side trusts the other about what is |
| 526 | // in the bucket). The count cap is the residence rule made concrete |
| 527 | // (design record § Residence): replicas are per-account opt-in constrained |
| 528 | // by the account's residence policy, not "sprinkle the fleet" — a number a |
| 529 | // deployment can raise deliberately beats an unbounded set nobody sized. |
| 530 | func ValidReplicas(regions []string) ([]string, error) { |
| 531 | if len(regions) > MaxReplicas { |
| 532 | return nil, fmt.Errorf("replicas: %d placements, at most %d — replicas hold order data in every region named until a drain confirms, so each one is a residence decision, not a cache hint", len(regions), MaxReplicas) |
| 533 | } |
| 534 | seen := make(map[string]bool, len(regions)) |
| 535 | out := make([]string, 0, len(regions)) |
| 536 | for _, r := range regions { |
| 537 | r = strings.TrimSpace(r) |
| 538 | if r == "" { |
| 539 | continue |
| 540 | } |
| 541 | if !ValidRegion(r) { |
| 542 | return nil, fmt.Errorf("replica placement %q: want a placement code (e.g. ie, uk)", r) |
| 543 | } |
| 544 | if seen[r] { |
| 545 | continue |
| 546 | } |
| 547 | seen[r] = true |
| 548 | out = append(out, r) |
| 549 | } |
| 550 | sort.Strings(out) |
| 551 | return out, nil |
| 552 | } |
| 553 | |
| 554 | // MaxUpstreams caps the Addrs set one record may name. The prober probes a |
| 555 | // host's upstreams SEQUENTIALLY once per interval (internal/edge/prober.go), |
| 556 | // and every scale figure the multi-upstream design rests on — probe traffic, |
| 557 | // per-tick wall time, the dashboard's per-upstream table — was drawn for a |
| 558 | // handful of backends behind one route, not hundreds. The cap is stated at |
| 559 | // the door so an oversized set is refused where it is typed, rather than |
| 560 | // discovered as a prober tick that no longer fits inside its own interval. |
| 561 | const MaxUpstreams = 16 |
| 562 | |
| 563 | // ValidateShape checks the §3 fields at the door, before a record is written |
| 564 | // or materialized. The rails live at the door, exactly as they do for |
| 565 | // registry.SetCaps and SetChannel — Upsert validates no column at all, so a |
| 566 | // value that reaches the row unvalidated is discovered months later as an |
| 567 | // adopt pass that restarted nothing (the `-backing unti` incident, cli.go). |
| 568 | func ValidateShape(rec Record) error { |
| 569 | if !ValidKind(rec.Kind) { |
| 570 | return fmt.Errorf("kind %q: want instance, service or static (empty means instance)", rec.Kind) |
| 571 | } |
| 572 | if !ValidRole(rec.Role) { |
| 573 | return fmt.Errorf("role %q: want \"\" or \"replica\"", rec.Role) |
| 574 | } |
| 575 | if !ValidSteering(rec.Steering) { |
| 576 | return fmt.Errorf("steering %q: want \"\" or \"latency\"", rec.Steering) |
| 577 | } |
| 578 | // A replica is a process holding a mirror of live data — never a file |
| 579 | // tree, never a foreign upstream the platform did not spawn. Pinning |
| 580 | // the class here (not only at the reconciler's mint gate) keeps the rail |
| 581 | // holding no matter which writer produced the record — and the ADDR half |
| 582 | // matters as much as the kind half: a replica with a public addr would |
| 583 | // pass everywhereClass's dial test and mint on EVERY non-fleet box, the |
| 584 | // residence rule's second door. |
| 585 | if rec.Role == "replica" && rec.KindOrDefault() != "instance" { |
| 586 | return errors.New("role replica with kind " + rec.KindOrDefault() + ": a replica is an instance of the account — a process the activator spawns holding a mirror of its data — drop kind or drop role") |
| 587 | } |
| 588 | if rec.Role == "replica" && len(rec.AddrsOrDefault()) > 0 { |
| 589 | return errors.New("role replica with addr: a replica's process is the platform's own, spawned by the activator on the box its placement names — an upstream would make it an everywhere-class route and mint it on every box — drop the addr or drop role") |
| 590 | } |
| 591 | for _, r := range rec.Replicas { |
| 592 | r = strings.TrimSpace(r) |
| 593 | if r != "" && !ValidRegion(r) { |
| 594 | return fmt.Errorf("replica placement %q: want a placement code (e.g. ie, uk)", r) |
| 595 | } |
| 596 | } |
| 597 | if len(rec.Replicas) > MaxReplicas { |
| 598 | return fmt.Errorf("replicas: %d placements, at most %d", len(rec.Replicas), MaxReplicas) |
| 599 | } |
| 600 | if rec.Gate != "" { |
| 601 | if _, err := bcrypt.Cost([]byte(rec.Gate)); err != nil { |
| 602 | return errors.New("gate: want a bcrypt hash — the console hashes the password; a record never carries it in clear") |
| 603 | } |
| 604 | } |
| 605 | if rec.Addr != "" { |
| 606 | if err := validAddr(rec.Addr); err != nil { |
| 607 | return err |
| 608 | } |
| 609 | } |
| 610 | if rec.Addr != "" && len(rec.Addrs) > 0 { |
| 611 | return errors.New("addr and addrs both set: a record picks one upstream shape — the single legacy addr or the addrs set — drop one of the two") |
| 612 | } |
| 613 | if len(rec.Addrs) > MaxUpstreams { |
| 614 | return fmt.Errorf("addrs: %d upstreams, at most %d: the prober walks a host's upstreams one after another every tick, and the whole design is sized for a handful of backends per route, not hundreds — split the fleet across routes instead", len(rec.Addrs), MaxUpstreams) |
| 615 | } |
| 616 | if len(rec.Addrs) > 0 { |
| 617 | seen := make(map[string]bool, len(rec.Addrs)) |
| 618 | for _, a := range rec.Addrs { |
| 619 | if err := validAddrsElement(a); err != nil { |
| 620 | return err |
| 621 | } |
| 622 | if seen[a] { |
| 623 | return fmt.Errorf("addrs: duplicate upstream %q", a) |
| 624 | } |
| 625 | seen[a] = true |
| 626 | } |
| 627 | } |
| 628 | if rec.Unit != "" && !registry.ValidUnitName(rec.Unit) { |
| 629 | return fmt.Errorf("unit %q: want a systemd service unit name (e.g. console.service)", rec.Unit) |
| 630 | } |
| 631 | if rec.Environment != "" && !validEnvironmentName(rec.Environment) { |
| 632 | return fmt.Errorf("environment %q: want [a-z0-9-]{1,32}", rec.Environment) |
| 633 | } |
| 634 | if rec.KindOrDefault() == "static" { |
| 635 | if len(rec.AddrsOrDefault()) > 0 { |
| 636 | return errors.New("kind static with addr: a static route serves a file tree and proxies to nothing — drop one of the two") |
| 637 | } |
| 638 | if rec.Unit != "" { |
| 639 | return errors.New("kind static with unit: a static route has no process for a unit to own — drop one of the two") |
| 640 | } |
| 641 | if rec.Hibernate != nil && *rec.Hibernate { |
| 642 | return errors.New("kind static with hibernate: a static route has no process to wake, so arming the activator for it would wedge the host — drop hibernate") |
| 643 | } |
| 644 | } |
| 645 | if rec.KindOrDefault() == "service" && len(rec.AddrsOrDefault()) == 0 { |
| 646 | return errors.New("kind service with no addr: the platform only proxies to a service, so the record must name the upstream it proxies to") |
| 647 | } |
| 648 | // Same wedge as static+hibernate, arriving from the other side. Addr does |
| 649 | // NOT exclude a route from activator management: internal/edge's gate is |
| 650 | // `Kind == "instance" && Hibernate && !viaTunnel`, so an addr-backed |
| 651 | // instance flagged for hibernation is armed for an exec-child wake it has |
| 652 | // no socket for and whose process the platform never spawned. |
| 653 | // HibernateOrDefault's own derivation already states the invariant ("an |
| 654 | // addr-backed tenant is always on"); this is the door that enforces it. |
| 655 | if len(rec.AddrsOrDefault()) > 0 && rec.Hibernate != nil && *rec.Hibernate { |
| 656 | return errors.New("addr with hibernate: an addr-backed route proxies to a process the platform does not spawn and has no socket to wake, so arming the activator for it would wedge the host — drop hibernate") |
| 657 | } |
| 658 | return nil |
| 659 | } |
| 660 | |
| 661 | // validAddr checks one upstream's shape: host:port, with the port an |
| 662 | // integer in 1..65535. This is the LEGACY single Addr's rail, unchanged: |
| 663 | // records carrying it exist in production buckets today, some of them naming |
| 664 | // their backend by hostname, so nothing here may grow stricter. The new Addrs |
| 665 | // set gets the stricter rail instead — see validAddrsElement. |
| 666 | func validAddr(a string) error { |
| 667 | h, p, err := net.SplitHostPort(a) |
| 668 | if err != nil || h == "" || p == "" { |
| 669 | return fmt.Errorf("addr %q: want a TCP upstream as host:port (e.g. 127.0.0.1:8090)", a) |
| 670 | } |
| 671 | // Parse port as integer and validate range 1..65535. net.SplitHostPort |
| 672 | // accepts garbage ports (e.g. "127.0.0.1:abc") — we need to refuse them |
| 673 | // here at the door, not discover them months later as a route that never |
| 674 | // connects (the -backing unti class of incident). |
| 675 | portNum, err := strconv.ParseUint(p, 10, 16) |
| 676 | if err != nil || portNum == 0 { |
| 677 | return fmt.Errorf("addr %q: port must be an integer in the range 1..65535", a) |
| 678 | } |
| 679 | // 169.254.0.0/16 — IPv4 link-local, which on every cloud box the platform |
| 680 | // runs on is the instance metadata service (169.254.169.254) and its |
| 681 | // neighbours. An upstream is a URL the edge fetches on the caller's |
| 682 | // behalf, so a route pointed there is an SSRF into the box's own |
| 683 | // credential endpoint, dressed as an ordinary proxy. Refused at the same |
| 684 | // door the port range is, so the record can never reach a box. |
| 685 | // |
| 686 | // This rail is DELIBERATELY minimal (ruling 2026-08-12): loopback and |
| 687 | // RFC1918 stay allowed, because that is where every legitimate addr-backed |
| 688 | // route on the fleet points today (127.0.0.1:8090 consoles, private-subnet |
| 689 | // tenants). The wider question — what a MEMBER may state — was ruled on |
| 690 | // in discussion 50 (GitHub #170) and is answered at the console's doors |
| 691 | // (refuseMemberUpstreams), not here; do not widen this check into an |
| 692 | // answer for it. |
| 693 | if ip := net.ParseIP(h); ip != nil && ip.To4() != nil && ip.IsLinkLocalUnicast() { |
| 694 | return fmt.Errorf("addr %q: 169.254.0.0/16 is link-local (the cloud metadata service lives there) — an upstream must be a real backend address", a) |
| 695 | } |
| 696 | return nil |
| 697 | } |
| 698 | |
| 699 | // validAddrsElement is validAddr plus the two rails the NEW Addrs set carries |
| 700 | // and the legacy Addr cannot (wire-compat: no record in any bucket carries |
| 701 | // Addrs yet, so this field's rules can be written the way they should have |
| 702 | // been all along, while Addr keeps exactly the checks it shipped with). |
| 703 | // |
| 704 | // 1. The host must be an IP LITERAL. validAddr's link-local refusal only |
| 705 | // looks at a parsed IP, so a NAME skips it entirely and |
| 706 | // "md.attacker.example:80" resolving to 169.254.169.254 walks straight |
| 707 | // through the very rail that exists to stop it. A name cannot be |
| 708 | // validated at the door at all, in fact: the platform dials these |
| 709 | // upstreams itself, and whatever a name resolved to when the record was |
| 710 | // written is not what it must resolve to when the edge connects — the |
| 711 | // target is free to move after validation. An address that is checked |
| 712 | // here and dialled there has to be the same address. |
| 713 | // 2. Link-local is refused in BOTH families, unicast and multicast: |
| 714 | // 169.254.0.0/16 AND fe80::/10 (plus ff02::/16), because a box reaches |
| 715 | // its metadata service over IPv6 as readily as IPv4 and "[fe80::1]:80" |
| 716 | // is the same SSRF wearing the other family's clothes. |
| 717 | // |
| 718 | // Everything else validAddr allows stays allowed here — loopback and RFC1918 |
| 719 | // are where every legitimate addr-backed route on the fleet points today |
| 720 | // (127.0.0.1:8090 consoles, private-subnet tenants). This is the metadata |
| 721 | // rail closing its own holes, not a widening of it. Whether a MEMBER may |
| 722 | // state an upstream at all is the console's call: refuseMemberUpstreams. |
| 723 | func validAddrsElement(a string) error { |
| 724 | if err := validAddr(a); err != nil { |
| 725 | return err |
| 726 | } |
| 727 | h, _, err := net.SplitHostPort(a) |
| 728 | if err != nil { |
| 729 | return fmt.Errorf("addr %q: want a TCP upstream as host:port (e.g. 127.0.0.1:8090)", a) |
| 730 | } |
| 731 | ip := net.ParseIP(h) |
| 732 | if ip == nil { |
| 733 | return fmt.Errorf("addrs: upstream %q names a host, not an IP: the platform dials these addresses itself, and a name can resolve somewhere else after this check passes — state the backend's IP address as ip:port (e.g. 10.0.0.1:3000)", a) |
| 734 | } |
| 735 | if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { |
| 736 | return fmt.Errorf("addrs: upstream %q is link-local (169.254.0.0/16 and fe80::/10, where the cloud metadata service lives) — an upstream must be a real backend address", a) |
| 737 | } |
| 738 | return nil |
| 739 | } |
| 740 | |
| 741 | // ValidRecordHost applies the host rail this record actually needs (design |
| 742 | // §3.2). ValidHost is path-traversal defence: it is what keeps a hostile |
| 743 | // host from reaching the <host>.sock and <host>.db path segments as anything |
| 744 | // but an opaque, safe component, and it stays exactly where that danger is. |
| 745 | // A record that derives no filesystem path has no such exposure — its host |
| 746 | // reaches a bucket key and the edge's routing table only — so it needs the |
| 747 | // shape check alone, which is what lets an apex (carloku.com) or a console's |
| 748 | // own hostname be expressed at all. |
| 749 | func ValidRecordHost(rec Record, domain string) bool { |
| 750 | if rec.DerivesPaths() { |
| 751 | return ValidHost(rec.Host, domain) |
| 752 | } |
| 753 | return ValidHostShape(rec.Host) |
| 754 | } |
| 755 | |
| 756 | // regionPoolRe is the shared charset for a region or pool code: lowercase |
| 757 | // a-z, 0-9, hyphen, up to 16 characters. Empty is deliberately valid — the |
| 758 | // unconstrained region and the shared (default) pool are both "". |
| 759 | var regionPoolRe = regexp.MustCompile(`^[a-z0-9-]{0,16}$`) |
| 760 | |
| 761 | func ValidRegion(region string) bool { return regionPoolRe.MatchString(region) } |
| 762 | |
| 763 | // replicaSet renders a record's replica placement set for the wire column |
| 764 | // that carries it (registry.Route.Replicas): comma-joined placement codes in |
| 765 | // canonical (sorted, deduplicated, ValidRegion-checked at the door) order, |
| 766 | // "" = none. routeFor and convergeOwned both call this — one derivation, |
| 767 | // two writers, the upstreamCols rule: a second, independent rendering is how |
| 768 | // a route minted correctly comes to be "converged" onto a different value |
| 769 | // every pass, forever. |
| 770 | func replicaSet(regions []string) string { |
| 771 | norm, err := ValidReplicas(regions) |
| 772 | if err != nil { |
| 773 | // The door has already checked the set; a record that slipped past |
| 774 | // it (a hand-written bucket body — the reconcile pass reads those) |
| 775 | // still renders DETERMINISTICALLY, never as two different columns |
| 776 | // on two boxes. Sorted, deduped, best-effort — and LOGGED, so a |
| 777 | // hand-edited body is visible rather than silently normalized. |
| 778 | log.Printf("provision: replicas set failed validation (%v) — rendering the sorted, deduplicated remainder", err) |
| 779 | seen := map[string]bool{} |
| 780 | var keep []string |
| 781 | for _, r := range regions { |
| 782 | if r = strings.TrimSpace(r); r != "" && !seen[r] { |
| 783 | seen[r] = true |
| 784 | keep = append(keep, r) |
| 785 | } |
| 786 | } |
| 787 | norm = keep |
| 788 | sort.Strings(norm) |
| 789 | } |
| 790 | return strings.Join(norm, ",") |
| 791 | } |
| 792 | func ValidPool(pool string) bool { return regionPoolRe.MatchString(pool) } |
| 793 | |
| 794 | // envNameRe is appconfig.ValidEnvironmentName's own shape (pipelines design |
| 795 | // §4a), spelled out again here rather than imported: appconfig already |
| 796 | // imports this package (LoadMerged reads a route's environment column), |
| 797 | // so the reverse import would be a cycle. A record's Environment column |
| 798 | // and a bundle's environment name are the SAME vocabulary; if the shape |
| 799 | // in appconfig/environments.go ever changes, this copy must change with |
| 800 | // it. It exists only because of the import direction, not because the |
| 801 | // rule is provision's to define. |
| 802 | var envNameRe = regexp.MustCompile(`^[a-z0-9-]{1,32}$`) |
| 803 | |
| 804 | // validEnvironmentName is appconfig.ValidEnvironmentName's check, duplicated |
| 805 | // for the reason envNameRe's comment gives: "default" names the implicit |
| 806 | // base bundle, never a named overlay, and "" is not a record's problem to |
| 807 | // validate here (the field is optional; the caller above already gates on |
| 808 | // rec.Environment != ""). |
| 809 | func validEnvironmentName(name string) bool { |
| 810 | return name != "default" && envNameRe.MatchString(name) |
| 811 | } |
| 812 | |
| 813 | // GetManifest reads app's instances manifest. Not-found stays |
| 814 | // os.ErrNotExist-wrapped (absence IS the "instances disabled" answer, not |
| 815 | // an error condition the caller must special-case beyond errors.Is). A |
| 816 | // schema this binary doesn't understand is refused rather than silently |
| 817 | // truncated by json.Unmarshal (appconfig/store.go's precedent) — acting on |
| 818 | // a newer manifest would silently discard whatever the unknown fields meant. |
| 819 | func GetManifest(ctx context.Context, s release.Store, acct, app string) (*Manifest, error) { |
| 820 | m, _, err := GetManifestETag(ctx, s, acct, app) |
| 821 | return m, err |
| 822 | } |
| 823 | |
| 824 | // GetManifestETag is GetManifest carrying the etag its bytes were read |
| 825 | // under — for a caller that must write back GUARDED against what it read |
| 826 | // (PutManifestIfMatch), not against whatever is there at write time. |
| 827 | func GetManifestETag(ctx context.Context, s release.Store, acct, app string) (*Manifest, string, error) { |
| 828 | raw, etag, err := s.Get(ctx, ManifestKey(acct, app)) |
| 829 | if err != nil { |
| 830 | return nil, "", err |
| 831 | } |
| 832 | var m Manifest |
| 833 | if err := json.Unmarshal(raw, &m); err != nil { |
| 834 | return nil, "", fmt.Errorf("provision: manifest %s/%s: %w", acct, app, err) |
| 835 | } |
| 836 | if m.Schema > 1 { |
| 837 | return nil, "", fmt.Errorf("provision: manifest %s/%s: schema %d is newer than this binary understands — upgrade carlos", acct, app, m.Schema) |
| 838 | } |
| 839 | return &m, etag, nil |
| 840 | } |
| 841 | |
| 842 | // PutManifest publishes app's instances manifest — a CAS upsert (read |
| 843 | // etag, put, one retry on a lost race), the WriteAccountManifest shape, |
| 844 | // because unlike a record this object is meant to be overwritten as an |
| 845 | // operator adjusts policy, not created once. |
| 846 | func claimRawPutManifest(ctx context.Context, s release.Store, acct, app string, m Manifest) error { |
| 847 | key := ManifestKey(acct, app) |
| 848 | body, err := json.Marshal(m) |
| 849 | if err != nil { |
| 850 | return err |
| 851 | } |
| 852 | put := func() error { |
| 853 | etag := "" |
| 854 | if _, cur, err := s.Get(ctx, key); err == nil { |
| 855 | etag = cur |
| 856 | } else if !errors.Is(err, os.ErrNotExist) { |
| 857 | return err |
| 858 | } |
| 859 | _, err := s.Put(ctx, key, body, etag) |
| 860 | return err |
| 861 | } |
| 862 | if err := put(); errors.Is(err, release.ErrCASConflict) { |
| 863 | return put() // one retry on a lost race; a second loss surfaces |
| 864 | } else if err != nil { |
| 865 | return err |
| 866 | } |
| 867 | return nil |
| 868 | } |
| 869 | |
| 870 | // PutManifestIfMatch writes app's instances manifest guarded by the etag |
| 871 | // the caller READ it under — "" meaning it read nothing, which makes the |
| 872 | // write create-only. Unlike PutManifest's upsert (which re-reads inside and |
| 873 | // retries, so the last writer always wins) a lost race here surfaces as |
| 874 | // release.ErrCASConflict for the caller to answer: the console's |
| 875 | // member-facing enable door uses this because its domain-conflict check is |
| 876 | // only honest if nothing can slip in between the check and the write |
| 877 | // (review 2026-08-06 — an operator's store-mode manifest landing in that |
| 878 | // window must never be silently re-homed). |
| 879 | func claimRawPutManifestIfMatch(ctx context.Context, s release.Store, acct, app string, m Manifest, etag string) error { |
| 880 | body, err := json.Marshal(m) |
| 881 | if err != nil { |
| 882 | return err |
| 883 | } |
| 884 | _, err = s.Put(ctx, ManifestKey(acct, app), body, etag) |
| 885 | return err |
| 886 | } |
| 887 | |
| 888 | // CreateRecord writes a new instance record, create-only: a lost race |
| 889 | // against a second signup for the same host comes back wrapping |
| 890 | // release.ErrCASConflict, never a silent overwrite of the winner's record. |
| 891 | func CreateRecord(ctx context.Context, s release.Store, rec Record) error { |
| 892 | if len(rec.Maintenance) != 0 { |
| 893 | return ErrMaintenanceProjection |
| 894 | } |
| 895 | body, err := json.Marshal(rec) |
| 896 | if err != nil { |
| 897 | return err |
| 898 | } |
| 899 | _, _, err = WriteRecord(ctx, s, RecordKey(rec.Account, rec.App, rec.Host), []byte(`{}`), body, "") |
| 900 | return err |
| 901 | } |
| 902 | |
| 903 | // SetRecordRegion repoints an existing instance record's Region field — the |
| 904 | // one field a record is expected to change AFTER creation. Unlike |
| 905 | // CreateRecord (create-only: a lost race is a permanent 409) this is a |
| 906 | // narrow read-modify-write CAS — etag from Get, Put, one retry on a lost |
| 907 | // race — mirroring PutManifest's own retry shape, because unlike a record's |
| 908 | // identity (Host/App/Account, checked at CREATE and never touched again) |
| 909 | // Region is meant to be repointed in place. |
| 910 | // |
| 911 | // This is placement's (console/fleets.go, customer-fleets plan) write onto |
| 912 | // instances that already exist: "place" walks an app's already-provisioned |
| 913 | // records and calls this for each, so a fleet or platform placement change |
| 914 | // governs the NEXT reconcile pass for every resident instance, not only the |
| 915 | // next one created. Without it, `carlos apps place` would only ever change |
| 916 | // where FUTURE instances land — the app's existing, already-parked ones |
| 917 | // would keep the region they were created under forever, silently exempt |
| 918 | // from the placement a caller was just told had taken effect. |
| 919 | // |
| 920 | // Not-found is returned verbatim (os.ErrNotExist-wrapped): a caller walking |
| 921 | // a key list that has gone stale between listing and this call should see |
| 922 | // the record is gone, not a manufactured success for a write that touched |
| 923 | // nothing. |
| 924 | func SetRecordRegion(ctx context.Context, s release.Store, acct, app, host, region string) error { |
| 925 | _, err := updateRecord(ctx, s, acct, app, host, func(rec *Record) error { |
| 926 | if rec.Region != region { |
| 927 | if err := seedMoveRefusal(rec); err != nil { |
| 928 | return err |
| 929 | } |
| 930 | } |
| 931 | rec.Region = region |
| 932 | return nil |
| 933 | }) |
| 934 | return err |
| 935 | } |
| 936 | |
| 937 | // updateRecord is the one read-CAS-put-one-retry scaffold under both |
| 938 | // in-place record writers (SetRecordRegion, SetRecordMove): re-reading and |
| 939 | // re-applying a narrow mutation on a lost race changes nothing about what |
| 940 | // it meant. Returns the record as written. |
| 941 | func updateRecord(ctx context.Context, s release.Store, acct, app, host string, mut func(*Record) error) (*Record, error) { |
| 942 | return updateRecordSeeding(ctx, s, acct, app, host, mut, false) |
| 943 | } |
| 944 | |
| 945 | // updateRecordSeeding is updateRecord's scaffold. seedWrite is true only for |
| 946 | // the seed transitions: every other caller is refused a change to Seed. |
| 947 | func updateRecordSeeding(ctx context.Context, s release.Store, acct, app, host string, mut func(*Record) error, seedWrite bool) (*Record, error) { |
| 948 | key := RecordKey(acct, app, host) |
| 949 | update := func() (*Record, error) { |
| 950 | raw, etag, err := s.Get(ctx, key) |
| 951 | if err != nil { |
| 952 | return nil, err |
| 953 | } |
| 954 | rec, err := DecodeRecord(raw) |
| 955 | if err != nil { |
| 956 | return nil, fmt.Errorf("provision: record %s/%s/%s: %w", acct, app, host, err) |
| 957 | } |
| 958 | if rec.Account != acct || rec.App != app || rec.Host != host { |
| 959 | return nil, ErrMaintenanceProjection |
| 960 | } |
| 961 | before := rec |
| 962 | before.Maintenance = append(MaintenanceProjection(nil), rec.Maintenance...) |
| 963 | before.Seed = cloneSeed(rec.Seed) // deep: mut edits rec.Seed through its pointer |
| 964 | if err := mut(&rec); err != nil { |
| 965 | if errors.Is(err, errRecordUnchanged) { |
| 966 | return &rec, nil // the record already says it: no write, no sequence advance |
| 967 | } |
| 968 | return nil, err |
| 969 | } |
| 970 | if !protectedFieldsEqual(before, rec) { |
| 971 | return nil, ErrMaintenanceProjection |
| 972 | } |
| 973 | if !seedWrite && !seedEqual(before.Seed, rec.Seed) { |
| 974 | return nil, ErrSeedProtected |
| 975 | } |
| 976 | body, err := json.Marshal(rec) |
| 977 | if err != nil { |
| 978 | return nil, err |
| 979 | } |
| 980 | _, seq, err := WriteRecord(ctx, s, key, raw, body, etag) |
| 981 | if err != nil { |
| 982 | return nil, err |
| 983 | } |
| 984 | rec.RecordSeq = seq |
| 985 | return &rec, nil |
| 986 | } |
| 987 | rec, err := update() |
| 988 | if errors.Is(err, release.ErrCASConflict) { |
| 989 | return update() // one retry on a lost race; a second loss surfaces |
| 990 | } |
| 991 | return rec, err |
| 992 | } |
| 993 | |
| 994 | // MoveState is the record's move journal (move-any-app design §7.2). The |
| 995 | // claim carries the cross-operation serialization; this field carries one |
| 996 | // move's own phases, because only a move reads them. Every field after |
| 997 | // UpdatedAt is additive and omitempty: a v0.22 console reads the ones it |
| 998 | // knows and refuses to abort any state it does not (G16). |
| 999 | type MoveState struct { |
| 1000 | // ID is the move id (mv_…), Incarnation the holder incarnation the |
| 1001 | // claim named when the move began. Both guard every box CAS (§7.2). |
| 1002 | ID string `json:"id,omitempty"` |
| 1003 | Incarnation string `json:"incarnation,omitempty"` |
| 1004 | From string `json:"from,omitempty"` |
| 1005 | To string `json:"to"` |
| 1006 | // Cut is "auto" or "manual" (§7.1). |
| 1007 | Cut string `json:"cut,omitempty"` |
| 1008 | State string `json:"state"` |
| 1009 | RequestedBy string `json:"requested_by,omitempty"` |
| 1010 | RequestedAt time.Time `json:"requested_at"` |
| 1011 | ReadyAt *time.Time `json:"ready_at,omitempty"` |
| 1012 | // UpdatedAt is when the state last changed — stamped by SetRecordMove |
| 1013 | // itself on every transition that leaves a move in place, so the |
| 1014 | // console's flat move view can carry one freshness stamp without each |
| 1015 | // transition re-implementing it. |
| 1016 | UpdatedAt time.Time `json:"updated_at"` |
| 1017 | |
| 1018 | // Inputs are the versions pinned at readiness and re-checked at |
| 1019 | // admission (§7.3 T3, §7.4 admitted, INV-6). |
| 1020 | Inputs *MoveInputs `json:"inputs,omitempty"` |
| 1021 | |
| 1022 | CutPhase string `json:"cut_phase,omitempty"` |
| 1023 | CutAt *time.Time `json:"cut_at,omitempty"` |
| 1024 | CutBudgetS int `json:"cut_budget_s,omitempty"` |
| 1025 | // CutAttempts is the number of probe attempts the admission's T11 |
| 1026 | // pricing admitted (the target report's AttemptsAdmitted, H10). The |
| 1027 | // landing's probe reads it; absent refuses the probe. |
| 1028 | CutAttempts int `json:"cut_attempts,omitempty"` |
| 1029 | CutDeadline *time.Time `json:"cut_deadline,omitempty"` |
| 1030 | Source *MoveSource `json:"source,omitempty"` |
| 1031 | |
| 1032 | FlippedAt *time.Time `json:"flipped_at,omitempty"` |
| 1033 | LandingDeadline *time.Time `json:"landing_deadline,omitempty"` |
| 1034 | LandingPhase string `json:"landing_phase,omitempty"` |
| 1035 | Landing *MoveLandingJournal `json:"landing,omitempty"` |
| 1036 | |
| 1037 | CommittedAt *time.Time `json:"committed_at,omitempty"` |
| 1038 | OpenedAt *time.Time `json:"opened_at,omitempty"` |
| 1039 | SourceParkedAt *time.Time `json:"source_parked_at,omitempty"` |
| 1040 | |
| 1041 | // ReopenedAt is when a revert's source answered its probe again |
| 1042 | // (rev_phase probed, ruling J26): a revert has no Rollback journal, and |
| 1043 | // MoveRevertClose counts the offline time from CutAt to this stamp. |
| 1044 | ReopenedAt *time.Time `json:"reopened_at,omitempty"` |
| 1045 | // RecoveryInputs are the inputs a revert's probe rendered (Plan 4 Task |
| 1046 | // 6; §7.7, J18), recorded before the render by MoveRecordRevertInputs; |
| 1047 | // never compared with Inputs. A rollback keeps its own in |
| 1048 | // Rollback.RecoveryInputs. |
| 1049 | RecoveryInputs *MoveInputs `json:"recovery_inputs,omitempty"` |
| 1050 | |
| 1051 | Rollback *MoveRollback `json:"rollback,omitempty"` |
| 1052 | Reason string `json:"reason,omitempty"` |
| 1053 | // StuckReason is why a stuck move waits for an operator (§7.8 stuck), |
| 1054 | // a recorded sentence that MoveMarkStuck writes and MoveRetry clears. |
| 1055 | // It is a field of its own so that Reason keeps the move's cause (a |
| 1056 | // revert's, a landing's mark) and Rollback.Reason the rollback's |
| 1057 | // through a stuck and a retry. |
| 1058 | StuckReason string `json:"stuck_reason,omitempty"` |
| 1059 | // RecoveryRetryEpoch advances only on an operator retry. Recovery |
| 1060 | // probe budgets use it so restarts and replays keep their spent starts. |
| 1061 | RecoveryRetryEpoch uint64 `json:"recovery_retry_epoch,omitempty"` |
| 1062 | |
| 1063 | // Relocation is the durable pre-flip conversion decision |
| 1064 | // (move_relocation.go). It never installs anything. |
| 1065 | Relocation *MoveRelocationDecision `json:"relocation,omitempty"` |
| 1066 | |
| 1067 | // Kind, Sealer and Boxes belong to the offline operator move |
| 1068 | // (offline_move.go). All three are additive and omitempty: a journal |
| 1069 | // written before them parses and re-encodes byte for byte. Kind is |
| 1070 | // "offline" for that move and empty for every other. |
| 1071 | Kind string `json:"kind,omitempty"` |
| 1072 | // Sealer is the one box that seals the move, named at seal. |
| 1073 | Sealer string `json:"sealer,omitempty"` |
| 1074 | // Boxes are the source-region boxes that had to confirm the stop, named |
| 1075 | // at seal. |
| 1076 | Boxes []string `json:"boxes,omitempty"` |
| 1077 | // StagedAt is when an offline move was staged (OfflineStage): created |
| 1078 | // in offline-staging, with the instance still serving. RequestedAt is |
| 1079 | // then moved to the start, when the instance went offline, so a staged |
| 1080 | // move's offline time counts from the start like any other's; and a |
| 1081 | // move aborted before its start was never offline (OfflineFinish). |
| 1082 | // Additive and omitempty like the three above. |
| 1083 | StagedAt *time.Time `json:"staged_at,omitempty"` |
| 1084 | // Stager is the one source box that pre-copies a staged move, named |
| 1085 | // by the stage door and never changed (Astra round 1, finding 2): no |
| 1086 | // other box ever writes the staging prefix for this move, and an abort |
| 1087 | // of a staged move finishes only once this box has removed the staged |
| 1088 | // copy. Additive and omitempty. |
| 1089 | Stager string `json:"stager,omitempty"` |
| 1090 | // StageBucket and StageRegion are the staged copy's destination, the |
| 1091 | // target's data bucket and its region as the stage door read them, |
| 1092 | // fixed before any copy (Astra round 2, finding 2). Every stage round, |
| 1093 | // the seal's hand-over and an abort's cleanup use them; a target that |
| 1094 | // later publishes other coordinates is refused, never followed. |
| 1095 | // Additive and omitempty. |
| 1096 | StageBucket string `json:"stage_bucket,omitempty"` |
| 1097 | StageRegion string `json:"stage_region,omitempty"` |
| 1098 | } |
| 1099 | |
| 1100 | // MoveInputs (what the target was checked against, §7.3 T3) is NOT |
| 1101 | // defined here: it is main's internal/provision/moveinputs_contract.go |
| 1102 | // (claim Plan 6, d649d8d4), the shared contract this journal references. |
| 1103 | |
| 1104 | // MoveSource is what the source recorded at `stopped` (§7.4): the |
| 1105 | // baselines every recovery compares against. |
| 1106 | type MoveSource struct { |
| 1107 | Replication string `json:"replication,omitempty"` |
| 1108 | NextDue int64 `json:"next_due,omitempty"` |
| 1109 | FinalTXID uint64 `json:"final_txid,omitempty"` |
| 1110 | DBSHA256 string `json:"db_sha256,omitempty"` |
| 1111 | Files *MoveFilesDigest `json:"files,omitempty"` |
| 1112 | TimersSHA256 string `json:"timers_sha256,omitempty"` |
| 1113 | Evidence string `json:"evidence,omitempty"` |
| 1114 | // Release and EnvDigest are the offline move's: the build the source |
| 1115 | // ran and the portable digest of its launch environment, recorded at |
| 1116 | // seal. Additive and omitempty. |
| 1117 | Release string `json:"release,omitempty"` |
| 1118 | EnvDigest string `json:"env_digest,omitempty"` |
| 1119 | } |
| 1120 | |
| 1121 | type MoveFilesDigest struct { |
| 1122 | Plain string `json:"plain,omitempty"` |
| 1123 | Sealed string `json:"sealed,omitempty"` |
| 1124 | Bytes int64 `json:"bytes,omitempty"` |
| 1125 | Entries int `json:"entries,omitempty"` |
| 1126 | } |
| 1127 | |
| 1128 | // MoveLandingJournal is the landing's own record (§7.2 "landing"). Named |
| 1129 | // so because MoveLanding is the state constant below. |
| 1130 | type MoveLandingJournal struct { |
| 1131 | InstalledDBSHA256 string `json:"installed_db_sha256,omitempty"` |
| 1132 | Attempts int `json:"attempts,omitempty"` |
| 1133 | } |
| 1134 | |
| 1135 | // MoveOutcome is the terminal result the close CAS leaves on the record |
| 1136 | // as it clears Move (index D11), so a CLI that missed the intermediate |
| 1137 | // states still reads the outcome of the move it requested. |
| 1138 | type MoveOutcome struct { |
| 1139 | ID string `json:"id"` |
| 1140 | Outcome string `json:"outcome"` // done | rolled_back | aborted |
| 1141 | At time.Time `json:"at"` |
| 1142 | OfflineS float64 `json:"offline_s,omitempty"` |
| 1143 | } |
| 1144 | |
| 1145 | // MoveRollback is §7.8's journal. |
| 1146 | type MoveRollback struct { |
| 1147 | ID string `json:"id,omitempty"` |
| 1148 | RequestedAt *time.Time `json:"requested_at,omitempty"` |
| 1149 | Reason string `json:"reason,omitempty"` |
| 1150 | TargetPhase string `json:"target_phase,omitempty"` |
| 1151 | Dirty bool `json:"dirty,omitempty"` |
| 1152 | Evidence string `json:"evidence,omitempty"` |
| 1153 | SourcePhase string `json:"source_phase,omitempty"` |
| 1154 | RecoveredAt *time.Time `json:"recovered_at,omitempty"` |
| 1155 | OpenedAt *time.Time `json:"opened_at,omitempty"` |
| 1156 | TargetParkedAt *time.Time `json:"target_parked_at,omitempty"` |
| 1157 | ReleaseDeadline *time.Time `json:"release_deadline,omitempty"` |
| 1158 | // StuckAt is the source phase a stuck rollback stopped in (§7.8 stuck); |
| 1159 | // MoveRetry returns the source to it, so a verification mismatch is |
| 1160 | // verified again and never bypassed (INV-4). Empty unless stuck. |
| 1161 | StuckAt string `json:"stuck_at,omitempty"` |
| 1162 | // LossRef is the declared loss's record key, or the termination fence |
| 1163 | // record key (§7.10, spec rev 7.19), and LostAt when the record took |
| 1164 | // it. Written once, by MoveRollbackTargetLost. |
| 1165 | LossRef string `json:"loss_ref,omitempty"` |
| 1166 | LostAt *time.Time `json:"lost_at,omitempty"` |
| 1167 | // RecoveryInputs are the inputs a recovery install rendered itself |
| 1168 | // (§7.7, J2-R2-5); never compared with Inputs. |
| 1169 | RecoveryInputs *MoveInputs `json:"recovery_inputs,omitempty"` |
| 1170 | } |
| 1171 | |
| 1172 | // Legacy reports a move journal written before ids existed (a v0.22 |
| 1173 | // console). The guarded transitions below refuse it: a legacy record is |
| 1174 | // driven only by the legacy transitions (MovePromoteReady, MoveAbort, |
| 1175 | // MoveAbortComplete and the rest). The new door routes a Legacy() record |
| 1176 | // to the legacy MoveAbort, and the new worker completes a legacy aborting |
| 1177 | // record through the legacy code path only. |
| 1178 | func (m *MoveState) Legacy() bool { return m != nil && m.ID == "" } |
| 1179 | |
| 1180 | // The move states. There is no "done" state: the close CAS clears Move, so |
| 1181 | // a record at rest never says anything about moves. |
| 1182 | const ( |
| 1183 | MovePreparing = "preparing" |
| 1184 | MoveReady = "ready" |
| 1185 | MoveCutting = "cutting" |
| 1186 | // MoveAborting is abort's in-flight half (review round 1): the console |
| 1187 | // marks it, the SOURCE box deletes the staged copy from the target |
| 1188 | // bucket and then clears Move — because a staged replica left behind by |
| 1189 | // a bare clear would win a LATER landing on that box (staging-wins, |
| 1190 | // backend_exec.go's promoteStagedReplica) with data from an abandoned |
| 1191 | // move. |
| 1192 | MoveAborting = "aborting" |
| 1193 | MoveReverting = "reverting" |
| 1194 | MoveLanding = "landing" |
| 1195 | MoveCommitted = "committed" |
| 1196 | MoveRollingBack = "rolling_back" |
| 1197 | MoveStuck = "stuck" |
| 1198 | ) |
| 1199 | |
| 1200 | // Cut phases (§7.4), landing phases (§7.6), reverting phases (§7.4) and |
| 1201 | // rollback phases (§7.8). Each is written before its side effects. |
| 1202 | const ( |
| 1203 | CutRequested = "requested" |
| 1204 | CutAdmitted = "admitted" |
| 1205 | CutStopping = "stopping" |
| 1206 | CutStopped = "stopped" |
| 1207 | CutStaged = "staged" |
| 1208 | CutArchived = "archived" |
| 1209 | CutFlipped = "flipped" |
| 1210 | |
| 1211 | LandChecking = "checking" |
| 1212 | LandParking = "parking" |
| 1213 | LandBuilding = "building" |
| 1214 | LandPublished = "published" |
| 1215 | LandLocal = "local" |
| 1216 | LandConfigured = "configured" |
| 1217 | LandRestored = "restored" |
| 1218 | LandProbing = "probing" |
| 1219 | LandProbed = "probed" |
| 1220 | |
| 1221 | RevEvidence = "evidence" |
| 1222 | RevVerify = "verify" |
| 1223 | RevProbing = "probing" |
| 1224 | RevProbed = "probed" |
| 1225 | RevOpened = "opened" |
| 1226 | |
| 1227 | RbTargetReleasing = "releasing" |
| 1228 | RbTargetReleased = "released" |
| 1229 | RbTargetReturning = "returning" |
| 1230 | RbTargetReturned = "returned" |
| 1231 | RbTargetCleaning = "cleaning" |
| 1232 | RbTargetCleaned = "cleaned" |
| 1233 | // RbTargetLost is the declared-loss decision (§7.10), a phase of its |
| 1234 | // own: no forward step leaves it and the source's waiting never takes |
| 1235 | // it. Only MoveRollbackLossCommitted moves it on, to released. |
| 1236 | RbTargetLost = "lost" |
| 1237 | |
| 1238 | RbSourceWaiting = "waiting" |
| 1239 | RbSourceClaimed = "claimed" |
| 1240 | RbSourceRestoring = "restoring" |
| 1241 | RbSourceVerified = "verified" |
| 1242 | RbSourceParking = "parking" |
| 1243 | RbSourceBuilding = "building" |
| 1244 | RbSourcePublished = "published" |
| 1245 | RbSourceLocal = "local" |
| 1246 | RbSourceConfigured = "configured" |
| 1247 | RbSourceRestored = "restored" |
| 1248 | RbSourceProbing = "probing" |
| 1249 | RbSourceProbed = "probed" |
| 1250 | RbSourceRecovered = "recovered" |
| 1251 | RbSourceOpened = "opened" |
| 1252 | ) |
| 1253 | |
| 1254 | // ErrMoveConflict is a transition refused because the record's current move |
| 1255 | // state does not admit it — a second prepare while one is in flight, a cut |
| 1256 | // before ready, an abort after the point of no return. The console maps it |
| 1257 | // to 409; the box treats it as "someone else already advanced this" and |
| 1258 | // re-reads. |
| 1259 | var ErrMoveConflict = errors.New("provision: move state conflict") |
| 1260 | |
| 1261 | // The move refusals that name no value (copy record MV-N12, MV-N14, MV-N15, |
| 1262 | // MV-N40, MV-N41 and, for the legacy abort, MV-N45). Each is a whole recorded |
| 1263 | // sentence, so the copy gate can read it; the ones that name a value (MV-N13, |
| 1264 | // MV-N42, MV-N43, MV-N44) are written in place as one format. |
| 1265 | const ( |
| 1266 | refusedNoMove = "no move in flight" |
| 1267 | refusedNoJournaledMove = "no journaled move in flight" |
| 1268 | refusedMalformedStart = "malformed move start" |
| 1269 | refusedPastNoReturn = "past the point of no return. the cut completes forward or moves back on its own" |
| 1270 | refusedCutNotReady = "cut is only legal from ready" |
| 1271 | refusedAlreadyAborting = "already aborting" |
| 1272 | ) |
| 1273 | |
| 1274 | // SetRecordMove applies one guarded move transition: the same narrow |
| 1275 | // read-CAS-write with one retry that SetRecordRegion uses (and for the same |
| 1276 | // reason — re-reading and re-applying a transition on a lost race changes |
| 1277 | // nothing about what it meant; an ILLEGAL transition after the re-read is |
| 1278 | // ErrMoveConflict, which is the answer, not a race). mut sees the freshly |
| 1279 | // read record and either mutates it or returns the error that names why it |
| 1280 | // can't. |
| 1281 | func SetRecordMove(ctx context.Context, s release.Store, acct, app, host string, mut func(*Record) error) (*Record, error) { |
| 1282 | return updateRecord(ctx, s, acct, app, host, func(rec *Record) error { |
| 1283 | if err := mut(rec); err != nil { |
| 1284 | return err |
| 1285 | } |
| 1286 | if rec.Move != nil { |
| 1287 | rec.Move.UpdatedAt = time.Now().UTC() |
| 1288 | } |
| 1289 | return nil |
| 1290 | }) |
| 1291 | } |
| 1292 | |
| 1293 | // The five transitions, each a mut for SetRecordMove. They are the ONLY |
| 1294 | // writers of Record.Move. |
| 1295 | |
| 1296 | // MoveRequest starts a move to region `to`. Refused when one is already in |
| 1297 | // flight — the caller aborts first or waits. |
| 1298 | func MoveRequest(to string, now time.Time) func(*Record) error { |
| 1299 | return func(rec *Record) error { |
| 1300 | if err := seedMoveRefusal(rec); err != nil { |
| 1301 | return err |
| 1302 | } |
| 1303 | if rec.ActiveRun.Open() { |
| 1304 | return &MoveRunOpenError{Run: *rec.ActiveRun} |
| 1305 | } |
| 1306 | if rec.Move != nil { |
| 1307 | return fmt.Errorf("%w: a move to %s is already %s", ErrMoveConflict, rec.Move.To, rec.Move.State) |
| 1308 | } |
| 1309 | if rec.Region == to { |
| 1310 | return fmt.Errorf("%w: the instance is already in %s", ErrMoveConflict, to) |
| 1311 | } |
| 1312 | // A replica never gets a route on a platform box (reconcile's mint |
| 1313 | // gate), so no box owns its move and it would wedge in preparing |
| 1314 | // (G14). Checked here as well as at the console's door because |
| 1315 | // records have other writers. |
| 1316 | if rec.Role != "" { |
| 1317 | return fmt.Errorf("%w: a %s does not move. delete it and create one in the new region", ErrMoveConflict, rec.Role) |
| 1318 | } |
| 1319 | rec.Move = &MoveState{To: to, State: MovePreparing, RequestedAt: now.UTC()} |
| 1320 | return nil |
| 1321 | } |
| 1322 | } |
| 1323 | |
| 1324 | // MoveStart is what the request door decided (§7.1, §7.2): the id it |
| 1325 | // minted, the incarnation the claim CAS named, the cut mode and who asked. |
| 1326 | type MoveStart struct { |
| 1327 | ID, Incarnation, From, To, Cut, By string |
| 1328 | Now time.Time |
| 1329 | } |
| 1330 | |
| 1331 | // MoveStartTransition starts a move. Refused when one is in flight, when |
| 1332 | // the target is the current region, for a replica (a replica never gets a |
| 1333 | // route on a platform box, G14), and for any malformed start: a move |
| 1334 | // without an id has no journal to run. |
| 1335 | func MoveStartTransition(p MoveStart) func(*Record) error { |
| 1336 | return func(rec *Record) error { |
| 1337 | if err := seedMoveRefusal(rec); err != nil { |
| 1338 | return err |
| 1339 | } |
| 1340 | if rec.ActiveRun.Open() { |
| 1341 | return &MoveRunOpenError{Run: *rec.ActiveRun} |
| 1342 | } |
| 1343 | if rec.Move != nil { |
| 1344 | return fmt.Errorf("%w: %s", ErrMoveConflict, fmt.Sprintf("a move to %s is already %s", rec.Move.To, rec.Move.State)) |
| 1345 | } |
| 1346 | if !moveIDShape.MatchString(p.ID) || len(p.Incarnation) != 64 || (p.Cut != "auto" && p.Cut != "manual") || p.Now.IsZero() { |
| 1347 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart) |
| 1348 | } |
| 1349 | if rec.Region == p.To || p.From != rec.Region { |
| 1350 | return fmt.Errorf("%w: the instance is already in %s", ErrMoveConflict, rec.Region) |
| 1351 | } |
| 1352 | if rec.Role != "" { |
| 1353 | return fmt.Errorf("%w: %s", ErrMoveConflict, fmt.Sprintf("a %s does not move. delete it and create one in the new region", rec.Role)) |
| 1354 | } |
| 1355 | rec.Move = &MoveState{ID: p.ID, Incarnation: p.Incarnation, From: p.From, To: p.To, Cut: p.Cut, State: MovePreparing, RequestedBy: p.By, RequestedAt: p.Now.UTC()} |
| 1356 | return nil |
| 1357 | } |
| 1358 | } |
| 1359 | |
| 1360 | // moveIDShape is a move id: "mv_" and 26 lowercase base32 characters. |
| 1361 | // |
| 1362 | // Twin of moves.opIDShape (internal/moves/ids.go), which also accepts the |
| 1363 | // rollback prefix: the copy is forced, because moves imports provision, so |
| 1364 | // provision cannot import moves. Keep the two in step. |
| 1365 | var moveIDShape = regexp.MustCompile(`^mv_[a-z2-7]{26}$`) |
| 1366 | |
| 1367 | // rollbackIDShape is a rollback id: "rb_" and 26 lowercase base32 |
| 1368 | // characters, the other half of moves.opIDShape (moves.NewRollbackID mints |
| 1369 | // it). Twin of moves.ValidOpID's rollback form, for the same reason as |
| 1370 | // moveIDShape; moves' tests hold the two to each other. |
| 1371 | var rollbackIDShape = regexp.MustCompile(`^rb_[a-z2-7]{26}$`) |
| 1372 | |
| 1373 | // ValidRollbackID reports a rollback id (rollbackIDShape). |
| 1374 | func ValidRollbackID(id string) bool { return rollbackIDShape.MatchString(id) } |
| 1375 | |
| 1376 | // MoveGuard is what every Phase 1 transition carries (§7.2): the move id |
| 1377 | // and the holder incarnation. A write whose guard does not match the |
| 1378 | // record's move is a late write for another move or another incarnation |
| 1379 | // and is refused, so a delayed round for move A can never advance a |
| 1380 | // replacement move B that happens to be in the same state. |
| 1381 | type MoveGuard struct{ ID, Incarnation string } |
| 1382 | |
| 1383 | func (g MoveGuard) check(rec *Record) error { |
| 1384 | if rec.Move == nil || rec.Move.Legacy() { |
| 1385 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedNoJournaledMove) |
| 1386 | } |
| 1387 | // An offline move has its own transitions and no incarnation. A guard |
| 1388 | // with an empty incarnation must not reach it: kinds never cross. |
| 1389 | if rec.Move.Offline() { |
| 1390 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedOfflineKind) |
| 1391 | } |
| 1392 | if rec.Move.ID != g.ID || rec.Move.Incarnation != g.Incarnation { |
| 1393 | return fmt.Errorf("%w: %s", ErrMoveConflict, fmt.Sprintf("the record's move is %s for incarnation %s", rec.Move.ID, rec.Move.Incarnation)) |
| 1394 | } |
| 1395 | return nil |
| 1396 | } |
| 1397 | |
| 1398 | // MoveGuardedPromoteReady is the new worker's preparing → ready. It pins |
| 1399 | // the inputs the target was checked against onto the journal at readiness |
| 1400 | // (§7.2, §7.4), so the admission re-check has something to compare with. |
| 1401 | func MoveGuardedPromoteReady(g MoveGuard, now time.Time, in MoveInputs) func(*Record) error { |
| 1402 | return func(rec *Record) error { |
| 1403 | if err := g.check(rec); err != nil { |
| 1404 | return err |
| 1405 | } |
| 1406 | if err := MovePromoteReady(now)(rec); err != nil { |
| 1407 | return err |
| 1408 | } |
| 1409 | rec.Move.Inputs = pinInputs(in) |
| 1410 | return nil |
| 1411 | } |
| 1412 | } |
| 1413 | |
| 1414 | // MoveGuardedCut is the cut's entry (the console's cut door, or the source |
| 1415 | // for cut == "auto"): ready → cutting/requested. |
| 1416 | func MoveGuardedCut(g MoveGuard) func(*Record) error { |
| 1417 | return func(rec *Record) error { |
| 1418 | if err := g.check(rec); err != nil { |
| 1419 | return err |
| 1420 | } |
| 1421 | if rec.Move.State != MoveReady { |
| 1422 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedCutNotReady) |
| 1423 | } |
| 1424 | // Spec §7.4 pins at readiness: a ready move with no pinned inputs |
| 1425 | // (after a back-to-ready, until the repin) has nothing for the |
| 1426 | // admission to compare with, so it is not cut. |
| 1427 | if rec.Move.Inputs == nil { |
| 1428 | return fmt.Errorf("%w: the move has no pinned inputs to cut against", ErrMoveConflict) |
| 1429 | } |
| 1430 | rec.Move.State, rec.Move.CutPhase = MoveCutting, CutRequested |
| 1431 | return nil |
| 1432 | } |
| 1433 | } |
| 1434 | |
| 1435 | // pinInputs copies in, BundleETags map included, so the journal never |
| 1436 | // aliases a caller's map. |
| 1437 | func pinInputs(in MoveInputs) *MoveInputs { |
| 1438 | pinned := in |
| 1439 | if in.BundleETags != nil { |
| 1440 | pinned.BundleETags = make(map[string]string, len(in.BundleETags)) |
| 1441 | for k, v := range in.BundleETags { |
| 1442 | pinned.BundleETags[k] = v |
| 1443 | } |
| 1444 | } |
| 1445 | return &pinned |
| 1446 | } |
| 1447 | |
| 1448 | // MoveGuardedRepin pins fresh inputs on a ready move that has none: the |
| 1449 | // state MoveBackToReady leaves after the admission found the pin stale. |
| 1450 | // Legal in ready with Inputs == nil only; a pin already there is never |
| 1451 | // overwritten (the pin is what the admission compares with). |
| 1452 | func MoveGuardedRepin(g MoveGuard, in MoveInputs) func(*Record) error { |
| 1453 | return func(rec *Record) error { |
| 1454 | if err := g.check(rec); err != nil { |
| 1455 | return err |
| 1456 | } |
| 1457 | if rec.Move.State != MoveReady || rec.Move.Inputs != nil { |
| 1458 | return fmt.Errorf("%w: inputs are re-pinned only on a ready move that has none", ErrMoveConflict) |
| 1459 | } |
| 1460 | rec.Move.Inputs = pinInputs(in) |
| 1461 | // A refusal MoveBackToReadyRefused recorded ends here (fix round |
| 1462 | // 1): a move is pinned again only once its cause is gone, and from |
| 1463 | // then on `carlos move status` says it is ready to cut. |
| 1464 | rec.Move.Reason = "" |
| 1465 | return nil |
| 1466 | } |
| 1467 | } |
| 1468 | |
| 1469 | // MoveGuardedAbort: legal in preparing, ready and cutting/requested (§7.11). |
| 1470 | func MoveGuardedAbort(g MoveGuard) func(*Record) error { |
| 1471 | return func(rec *Record) error { |
| 1472 | if err := g.check(rec); err != nil { |
| 1473 | return err |
| 1474 | } |
| 1475 | switch rec.Move.State { |
| 1476 | case MovePreparing, MoveReady: |
| 1477 | case MoveCutting: |
| 1478 | if rec.Move.CutPhase != CutRequested { |
| 1479 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedPastNoReturn) |
| 1480 | } |
| 1481 | case MoveAborting: |
| 1482 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedAlreadyAborting) |
| 1483 | default: |
| 1484 | return fmt.Errorf("%w: %s", ErrMoveConflict, fmt.Sprintf("a move in state \"%s\" cannot be aborted", quotedInner(rec.Move.State))) |
| 1485 | } |
| 1486 | rec.Move.State = MoveAborting |
| 1487 | return nil |
| 1488 | } |
| 1489 | } |
| 1490 | |
| 1491 | // MoveGuardedAbortComplete clears an aborting move and records the outcome. |
| 1492 | func MoveGuardedAbortComplete(g MoveGuard, now time.Time) func(*Record) error { |
| 1493 | return func(rec *Record) error { |
| 1494 | if err := g.check(rec); err != nil { |
| 1495 | return err |
| 1496 | } |
| 1497 | if rec.Move.State != MoveAborting { |
| 1498 | return fmt.Errorf("%w: abort-complete is only legal from aborting", ErrMoveConflict) |
| 1499 | } |
| 1500 | rec.LastMove = &MoveOutcome{ID: g.ID, Outcome: "aborted", At: now.UTC()} |
| 1501 | rec.Move = nil |
| 1502 | return nil |
| 1503 | } |
| 1504 | } |
| 1505 | |
| 1506 | // cutOrder is the cut's phase order up to archived. flipped is not a step |
| 1507 | // of it: the flip is MoveFlip's, one record write that also moves the |
| 1508 | // region, so MoveCutPhase never writes it. |
| 1509 | var cutOrder = []string{CutRequested, CutAdmitted, CutStopping, CutStopped, CutStaged, CutArchived} |
| 1510 | |
| 1511 | var landingOrder = []string{LandChecking, LandParking, LandBuilding, LandPublished, LandLocal, LandConfigured, LandRestored, LandProbing, LandProbed} |
| 1512 | |
| 1513 | // LandingOrder is the landing's phase order (§7.6), a copy: the claim's |
| 1514 | // op.phase mirrors landing_phase one step at a time, and a worker that |
| 1515 | // finds the claim behind the record walks it forward along this order |
| 1516 | // (§7.5, the landing's syncClaimPhase). |
| 1517 | func LandingOrder() []string { return append([]string(nil), landingOrder...) } |
| 1518 | |
| 1519 | func nextIn(order []string, from, to string) bool { |
| 1520 | for i := 0; i+1 < len(order); i++ { |
| 1521 | if order[i] == from && order[i+1] == to { |
| 1522 | return true |
| 1523 | } |
| 1524 | } |
| 1525 | return false |
| 1526 | } |
| 1527 | |
| 1528 | // MoveCutPhase advances cut_phase one step, in order, under the guard |
| 1529 | // (§7.4). stamp may fill the fields the phase records; it runs after the |
| 1530 | // checks, on the journal the step has just advanced. |
| 1531 | func MoveCutPhase(g MoveGuard, from, to string, stamp func(*MoveState)) func(*Record) error { |
| 1532 | return func(rec *Record) error { |
| 1533 | if err := g.check(rec); err != nil { |
| 1534 | return err |
| 1535 | } |
| 1536 | if rec.Move.State != MoveCutting || rec.Move.CutPhase != from || !nextIn(cutOrder, from, to) { |
| 1537 | return fmt.Errorf("%w: cut is %s/%s, step expects %s to %s", ErrMoveConflict, rec.Move.State, rec.Move.CutPhase, from, to) |
| 1538 | } |
| 1539 | rec.Move.CutPhase = to |
| 1540 | if stamp != nil { |
| 1541 | stamp(rec.Move) |
| 1542 | } |
| 1543 | return nil |
| 1544 | } |
| 1545 | } |
| 1546 | |
| 1547 | // MoveAdmit is §7.4 requested to admitted: the budget, the absolute |
| 1548 | // deadline and the probe attempts T11 admitted (H10), in one write. It pins nothing: the inputs were pinned at |
| 1549 | // readiness (MoveGuardedPromoteReady, or MoveGuardedRepin after a back to |
| 1550 | // ready), and the source compares the current inputs with that pin before |
| 1551 | // this CAS. A cut with no pin is refused. |
| 1552 | func MoveAdmit(g MoveGuard, now time.Time, budgetS int, deadline time.Time, attempts int) func(*Record) error { |
| 1553 | return func(rec *Record) error { |
| 1554 | if err := g.check(rec); err != nil { |
| 1555 | return err |
| 1556 | } |
| 1557 | if rec.Move.Inputs == nil { |
| 1558 | return fmt.Errorf("%w: the move has no pinned inputs to admit against", ErrMoveConflict) |
| 1559 | } |
| 1560 | if attempts < 0 { |
| 1561 | return fmt.Errorf("%w: a negative count of probe attempts", ErrMoveConflict) |
| 1562 | } |
| 1563 | return MoveCutPhase(g, CutRequested, CutAdmitted, func(m *MoveState) { |
| 1564 | t, d := now.UTC(), deadline.UTC() |
| 1565 | m.CutAt, m.CutBudgetS, m.CutDeadline, m.CutAttempts = &t, budgetS, &d, attempts |
| 1566 | // A refusal an earlier cut recorded (MoveBackToReadyRefused) is |
| 1567 | // over once a cut admits. |
| 1568 | m.Reason = "" |
| 1569 | })(rec) |
| 1570 | } |
| 1571 | } |
| 1572 | |
| 1573 | // RefusedReplicaLayout is the reason MoveBackToReadyRefused records when |
| 1574 | // the claim holder's location is not the activator's own lease prefix |
| 1575 | // (moves Task 5 fix round 1, ruling T5-lease): a scoped location, |
| 1576 | // instances/<host>/<incarnation>, whose replica the cut cannot stop under |
| 1577 | // the writer's lease. Moving such a host waits for a later plan. A |
| 1578 | // recorded sentence (copy record MV-N167), like the Revert reasons. |
| 1579 | const RefusedReplicaLayout = "the instance keeps its data under a per-incarnation path, which moves cannot cut yet" |
| 1580 | |
| 1581 | // The cut guard's refusals (Paul's ruling, 2026-10-11), recorded by |
| 1582 | // MoveBackToReadyRefused before anything stops. RefusedTargetUnarmed: the |
| 1583 | // target's latest report does not state that its box can land the move and |
| 1584 | // roll it back (moves.TargetReport.Armed; a report from an older box never |
| 1585 | // does). RefusedSourceUnarmed: the source box cannot run the recovery a |
| 1586 | // failed cut would need. Recorded sentences (copy record MV-N441, MV-N442). |
| 1587 | const ( |
| 1588 | RefusedTargetUnarmed = "the target box cannot land the move or roll it back yet" |
| 1589 | RefusedSourceUnarmed = "the source box cannot recover the move if the cut fails" |
| 1590 | ) |
| 1591 | |
| 1592 | // MoveBackToReadyRefused is MoveBackToReady with the reason recorded on the |
| 1593 | // move, for a refusal that stands until something outside the move |
| 1594 | // changes (the operator reads it in `carlos move status`). reason must be |
| 1595 | // a recorded constant; any other text is refused and nothing is written. |
| 1596 | func MoveBackToReadyRefused(g MoveGuard, reason string) func(*Record) error { |
| 1597 | return func(rec *Record) error { |
| 1598 | switch reason { |
| 1599 | case RefusedReplicaLayout, RefusedTargetUnarmed, RefusedSourceUnarmed: |
| 1600 | default: |
| 1601 | return fmt.Errorf("provision: refusal reason %q has no recorded sentence", reason) |
| 1602 | } |
| 1603 | if err := MoveBackToReady(g)(rec); err != nil { |
| 1604 | return err |
| 1605 | } |
| 1606 | // Each reason is written as its constant, so the copy gate reads |
| 1607 | // the recorded sentence (as MoveRevert writes its reasons). |
| 1608 | switch reason { |
| 1609 | case RefusedTargetUnarmed: |
| 1610 | rec.Move.Reason = RefusedTargetUnarmed |
| 1611 | case RefusedSourceUnarmed: |
| 1612 | rec.Move.Reason = RefusedSourceUnarmed |
| 1613 | default: |
| 1614 | rec.Move.Reason = RefusedReplicaLayout |
| 1615 | } |
| 1616 | return nil |
| 1617 | } |
| 1618 | } |
| 1619 | |
| 1620 | // MoveBackToReady undoes an admission whose checks are no longer fresh or |
| 1621 | // whose inputs moved (§7.4 admitted). Legal from requested or admitted |
| 1622 | // only: nothing has stopped yet. It clears the cut fields and the pinned |
| 1623 | // inputs; the next readiness round pins fresh ones (MoveGuardedRepin). |
| 1624 | func MoveBackToReady(g MoveGuard) func(*Record) error { |
| 1625 | return func(rec *Record) error { |
| 1626 | if err := g.check(rec); err != nil { |
| 1627 | return err |
| 1628 | } |
| 1629 | if rec.Move.State != MoveCutting || (rec.Move.CutPhase != CutRequested && rec.Move.CutPhase != CutAdmitted) { |
| 1630 | return fmt.Errorf("%w: back to ready is legal from requested or admitted only", ErrMoveConflict) |
| 1631 | } |
| 1632 | m := rec.Move |
| 1633 | m.State, m.CutPhase, m.CutAt, m.CutBudgetS, m.CutDeadline, m.Inputs = MoveReady, "", nil, 0, nil, nil |
| 1634 | return nil |
| 1635 | } |
| 1636 | } |
| 1637 | |
| 1638 | // MoveRecordSource writes the stopped source's baselines (§7.4 stopped). |
| 1639 | func MoveRecordSource(g MoveGuard, src MoveSource) func(*Record) error { |
| 1640 | return func(rec *Record) error { |
| 1641 | if err := g.check(rec); err != nil { |
| 1642 | return err |
| 1643 | } |
| 1644 | if rec.Move.State != MoveCutting || rec.Move.CutPhase != CutStopped { |
| 1645 | return fmt.Errorf("%w: source digests are recorded at stopped", ErrMoveConflict) |
| 1646 | } |
| 1647 | s := src |
| 1648 | if src.Files != nil { |
| 1649 | f := *src.Files |
| 1650 | s.Files = &f |
| 1651 | } |
| 1652 | rec.Move.Source = &s |
| 1653 | return nil |
| 1654 | } |
| 1655 | } |
| 1656 | |
| 1657 | // MoveFlip is the flip (§7.4, §7.5): region = to, state = landing, the |
| 1658 | // landing deadline and landing_phase = checking, in one record write. The |
| 1659 | // claim follows (hostclaim.Flip). |
| 1660 | // |
| 1661 | // It refuses any state but cutting/archived, whoever holds the guard. So a |
| 1662 | // record decision written first, such as the C7 conversion's pre-flip guard |
| 1663 | // or a revert, wins over a flip already in flight: the late flip finds the |
| 1664 | // state changed and is refused, and the region is untouched. |
| 1665 | func MoveFlip(g MoveGuard, now, landingDeadline time.Time) func(*Record) error { |
| 1666 | return func(rec *Record) error { |
| 1667 | if err := g.check(rec); err != nil { |
| 1668 | return err |
| 1669 | } |
| 1670 | if rec.Move.State != MoveCutting || rec.Move.CutPhase != CutArchived { |
| 1671 | return fmt.Errorf("%w: the flip follows archived", ErrMoveConflict) |
| 1672 | } |
| 1673 | t, d := now.UTC(), landingDeadline.UTC() |
| 1674 | rec.Region = rec.Move.To |
| 1675 | m := rec.Move |
| 1676 | m.State, m.CutPhase, m.FlippedAt, m.LandingDeadline, m.LandingPhase = MoveLanding, CutFlipped, &t, &d, LandChecking |
| 1677 | return nil |
| 1678 | } |
| 1679 | } |
| 1680 | |
| 1681 | // MoveLandingPhase advances landing_phase one step, in order, under the |
| 1682 | // guard (§7.6). stamp runs after the checks. |
| 1683 | func MoveLandingPhase(g MoveGuard, from, to string, stamp func(*MoveState)) func(*Record) error { |
| 1684 | return func(rec *Record) error { |
| 1685 | if err := g.check(rec); err != nil { |
| 1686 | return err |
| 1687 | } |
| 1688 | if rec.Move.State != MoveLanding || rec.Move.LandingPhase != from || !nextIn(landingOrder, from, to) { |
| 1689 | return fmt.Errorf("%w: landing is %s/%s, step expects %s to %s", ErrMoveConflict, rec.Move.State, rec.Move.LandingPhase, from, to) |
| 1690 | } |
| 1691 | rec.Move.LandingPhase = to |
| 1692 | if stamp != nil { |
| 1693 | stamp(rec.Move) |
| 1694 | } |
| 1695 | return nil |
| 1696 | } |
| 1697 | } |
| 1698 | |
| 1699 | // The reasons a revert may record. MoveState.Reason reaches an operator |
| 1700 | // (`carlos move status`), so it is a recorded sentence (copy record MV-N158 |
| 1701 | // onward), never error text; the copy gate reads every write to a field |
| 1702 | // named Reason and cannot read a parameter. A later task that needs another |
| 1703 | // reason adds a constant, a case below and a row in the copy record. |
| 1704 | const ( |
| 1705 | RevertStopEvidenceBudget = "stop evidence budget" |
| 1706 | // RevertCutDeadline: a phase after stopping found cut_deadline passed. |
| 1707 | RevertCutDeadline = "the cut ran past its deadline" |
| 1708 | // RevertDeferredRounds: three final mirror rounds found the source |
| 1709 | // changing under them, or moved more than the priced bar. |
| 1710 | RevertDeferredRounds = "the final copy kept finding changes at the source" |
| 1711 | // RevertArchiveRefused: the stopped files directory cannot be archived |
| 1712 | // (a special file, over the cap, too many entries, a path too long). |
| 1713 | RevertArchiveRefused = "the stopped files could not be archived" |
| 1714 | ) |
| 1715 | |
| 1716 | // MoveRevert starts the pre-flip recovery (§7.4 reverting): from a cut |
| 1717 | // phase that has begun stopping the source. Before that, back to ready is |
| 1718 | // the exit; after the flip, a rollback is. reason must be one of the |
| 1719 | // Revert constants above: any other text is refused, and nothing is written. |
| 1720 | func MoveRevert(g MoveGuard, reason string) func(*Record) error { |
| 1721 | return func(rec *Record) error { |
| 1722 | if err := g.check(rec); err != nil { |
| 1723 | return err |
| 1724 | } |
| 1725 | if rec.Move.State != MoveCutting { |
| 1726 | return fmt.Errorf("%w: revert is a cut's recovery", ErrMoveConflict) |
| 1727 | } |
| 1728 | switch rec.Move.CutPhase { |
| 1729 | case CutStopping, CutStopped, CutStaged, CutArchived: |
| 1730 | default: |
| 1731 | return fmt.Errorf("%w: revert is legal from stopping to archived only", ErrMoveConflict) |
| 1732 | } |
| 1733 | switch reason { |
| 1734 | case RevertStopEvidenceBudget: |
| 1735 | rec.Move.Reason = RevertStopEvidenceBudget |
| 1736 | case RevertCutDeadline: |
| 1737 | rec.Move.Reason = RevertCutDeadline |
| 1738 | case RevertDeferredRounds: |
| 1739 | rec.Move.Reason = RevertDeferredRounds |
| 1740 | case RevertArchiveRefused: |
| 1741 | rec.Move.Reason = RevertArchiveRefused |
| 1742 | default: |
| 1743 | return fmt.Errorf("provision: revert reason %q has no recorded sentence", reason) |
| 1744 | } |
| 1745 | rec.Move.State = MoveReverting |
| 1746 | return nil |
| 1747 | } |
| 1748 | } |
| 1749 | |
| 1750 | // The reasons a landing may record for the rollback it cannot yet start |
| 1751 | // (Plan 2 Task 10). As with the revert's, each reaches an operator through |
| 1752 | // `carlos move status`, so it is a recorded sentence with a copy row, and |
| 1753 | // anything else is refused with nothing written. |
| 1754 | const ( |
| 1755 | // RollbackConfigChanged: the held render's bundle or overlay etags are |
| 1756 | // not the ones pinned at readiness (§7.6 configured). |
| 1757 | RollbackConfigChanged = "configuration changed during the move" |
| 1758 | // RollbackLandingDeadline: landing_deadline passed before restored. |
| 1759 | RollbackLandingDeadline = "the landing ran past its deadline" |
| 1760 | // RollbackClaimWired: the target runs with CARLOS_HOST_CLAIM on, and |
| 1761 | // the held writes a landing makes have no claim selection yet (G16). |
| 1762 | RollbackClaimWired = "this box cannot land a move while host claims are on" |
| 1763 | // RollbackLandingLayout: the held row is not at the flat database path |
| 1764 | // Phase 1 lands at (G11, D10). |
| 1765 | RollbackLandingLayout = "the landing row is not at the flat path" |
| 1766 | // RollbackInstallMismatch: the built database does not hash to the |
| 1767 | // install manifest's sha256, or does not pass its integrity check. |
| 1768 | RollbackInstallMismatch = "the installed database does not match the build" |
| 1769 | // RollbackFilesOccupied: the files directory holds files, or <host>.env |
| 1770 | // exists, and this landing did not write them (no earlier stay of this |
| 1771 | // installation registered them, so nothing retired them). They are not |
| 1772 | // the landing's to clear (final review, finding 1). |
| 1773 | RollbackFilesOccupied = "the files directory already holds files this move did not write" |
| 1774 | // RollbackManifestOccupied: the prepared files manifest's key under the |
| 1775 | // location already holds a manifest this landing did not write (moves |
| 1776 | // Task 6, C8). The prepared manifest is create-only and the landing's |
| 1777 | // replay reads its own bytes back; anything else there is not the |
| 1778 | // landing's to replace. |
| 1779 | RollbackManifestOccupied = "the destination already holds a files manifest this move did not write" |
| 1780 | // RollbackProbeAttempts: the probe used every attempt it was given |
| 1781 | // without the app answering (§7.7). |
| 1782 | RollbackProbeAttempts = "the probe ran out of attempts" |
| 1783 | // RollbackCommitDeadline: the probe passed, but the commit would have |
| 1784 | // come after the cut or landing deadline (§7.9). |
| 1785 | RollbackCommitDeadline = "the commit came after the deadline" |
| 1786 | // RollbackProbeUnpriced: the admission recorded no probe attempts for |
| 1787 | // the landing (H10: MoveAdmit's CutAttempts is absent), so the probe |
| 1788 | // cannot be bounded. |
| 1789 | RollbackProbeUnpriced = "the cut recorded no probe attempts for this landing" |
| 1790 | ) |
| 1791 | |
| 1792 | // MoveMarkRollback records on a landing record why the landing stopped |
| 1793 | // (§7.6: "any failure from checking to probing, or landing_deadline, |
| 1794 | // triggers a rollback"), leaving state and landing_phase as they are: |
| 1795 | // Plan 4's rollback transition (§7.8) is what replaces this one call site, |
| 1796 | // and until it exists the record carries the reason and the landing pass |
| 1797 | // leaves the record alone. The same sentence written again is a no-op; a |
| 1798 | // different one over a marked record is a conflict, so the first reason |
| 1799 | // stands. reason must be one of the Rollback constants above. |
| 1800 | func MoveMarkRollback(g MoveGuard, reason string) func(*Record) error { |
| 1801 | return func(rec *Record) error { |
| 1802 | if err := g.check(rec); err != nil { |
| 1803 | return err |
| 1804 | } |
| 1805 | if rec.Move.State != MoveLanding { |
| 1806 | return fmt.Errorf("%w: a rollback marker is a landing's", ErrMoveConflict) |
| 1807 | } |
| 1808 | if rec.Move.Reason != "" && rec.Move.Reason != reason { |
| 1809 | return fmt.Errorf("%w: the landing already records a reason", ErrMoveConflict) |
| 1810 | } |
| 1811 | switch reason { |
| 1812 | case RollbackConfigChanged: |
| 1813 | rec.Move.Reason = RollbackConfigChanged |
| 1814 | case RollbackLandingDeadline: |
| 1815 | rec.Move.Reason = RollbackLandingDeadline |
| 1816 | case RollbackClaimWired: |
| 1817 | rec.Move.Reason = RollbackClaimWired |
| 1818 | case RollbackLandingLayout: |
| 1819 | rec.Move.Reason = RollbackLandingLayout |
| 1820 | case RollbackInstallMismatch: |
| 1821 | rec.Move.Reason = RollbackInstallMismatch |
| 1822 | case RollbackFilesOccupied: |
| 1823 | rec.Move.Reason = RollbackFilesOccupied |
| 1824 | case RollbackManifestOccupied: |
| 1825 | rec.Move.Reason = RollbackManifestOccupied |
| 1826 | case RollbackProbeAttempts: |
| 1827 | rec.Move.Reason = RollbackProbeAttempts |
| 1828 | case RollbackCommitDeadline: |
| 1829 | rec.Move.Reason = RollbackCommitDeadline |
| 1830 | case RollbackProbeUnpriced: |
| 1831 | rec.Move.Reason = RollbackProbeUnpriced |
| 1832 | default: |
| 1833 | return fmt.Errorf("provision: rollback reason %q has no recorded sentence", reason) |
| 1834 | } |
| 1835 | return nil |
| 1836 | } |
| 1837 | } |
| 1838 | |
| 1839 | // MoveProbeAttempt records, on a landing in its probing phase, how many |
| 1840 | // probe attempts have begun (§7.7). The count only goes up: the same n again |
| 1841 | // is a replay and changes nothing, a smaller one is a late write from an |
| 1842 | // earlier attempt and is refused. It is refused too once a rollback reason |
| 1843 | // is recorded, because the probe is then over. A record at probing says |
| 1844 | // only that the probe is running and how many attempts it has made; whether |
| 1845 | // the attempt's first start has been journalled is the claim's to say |
| 1846 | // (hostclaim.WakeRetryAttempt), not the record's. |
| 1847 | func MoveProbeAttempt(g MoveGuard, n int) func(*Record) error { |
| 1848 | return func(rec *Record) error { |
| 1849 | if err := g.check(rec); err != nil { |
| 1850 | return err |
| 1851 | } |
| 1852 | m := rec.Move |
| 1853 | if m.State != MoveLanding || m.LandingPhase != LandProbing || m.Reason != "" { |
| 1854 | return fmt.Errorf("%w: probe attempts are counted while a landing is probing", ErrMoveConflict) |
| 1855 | } |
| 1856 | have := 0 |
| 1857 | if m.Landing != nil { |
| 1858 | have = m.Landing.Attempts |
| 1859 | } |
| 1860 | if n < 1 || n < have { |
| 1861 | return fmt.Errorf("%w: the attempt count does not go down", ErrMoveConflict) |
| 1862 | } |
| 1863 | if m.Landing == nil { |
| 1864 | m.Landing = &MoveLandingJournal{} |
| 1865 | } |
| 1866 | m.Landing.Attempts = n |
| 1867 | return nil |
| 1868 | } |
| 1869 | } |
| 1870 | |
| 1871 | // ErrCutDeadline is a commit attempted after the deadline that bounds it |
| 1872 | // (§7.4, §7.9): the record CAS is the decision, so the deadline cannot be |
| 1873 | // passed between the record write and the claim write. |
| 1874 | var ErrCutDeadline = fmt.Errorf("%w: the commit deadline has passed", ErrMoveConflict) |
| 1875 | |
| 1876 | // MoveCommit is the commit (§7.9): landing/probed to committed, in one |
| 1877 | // record write. It is refused when a rollback reason is already recorded, so |
| 1878 | // a recorded rollback wins over a commit that raced it, and when now is |
| 1879 | // after the earlier of cut_deadline and landing_deadline (ruling H8: §7.9 |
| 1880 | // names the first and §7.8 makes the second a rollback trigger, so the |
| 1881 | // earlier bounds the commit). A record with no cut_deadline cannot be shown |
| 1882 | // to be in time and is refused. now equal to the deadline is in time. |
| 1883 | // |
| 1884 | // A replay finds the move already committed and is an ErrMoveConflict; the |
| 1885 | // caller re-reads the record and accepts committed for its own move (H22). |
| 1886 | func MoveCommit(g MoveGuard, now time.Time) func(*Record) error { |
| 1887 | return func(rec *Record) error { |
| 1888 | if err := g.check(rec); err != nil { |
| 1889 | return err |
| 1890 | } |
| 1891 | m := rec.Move |
| 1892 | if m.State != MoveLanding || m.LandingPhase != LandProbed { |
| 1893 | return fmt.Errorf("%w: commit follows probed", ErrMoveConflict) |
| 1894 | } |
| 1895 | if m.Reason != "" { |
| 1896 | return fmt.Errorf("%w: a rollback is recorded for this landing", ErrMoveConflict) |
| 1897 | } |
| 1898 | if m.CutDeadline == nil || now.After(*m.CutDeadline) || (m.LandingDeadline != nil && now.After(*m.LandingDeadline)) { |
| 1899 | return ErrCutDeadline |
| 1900 | } |
| 1901 | t := now.UTC() |
| 1902 | m.State, m.CommittedAt = MoveCommitted, &t |
| 1903 | return nil |
| 1904 | } |
| 1905 | } |
| 1906 | |
| 1907 | // committedStamp stamps one of a committed move's obligations. The first |
| 1908 | // stamp stands: a replay changes nothing. |
| 1909 | func committedStamp(g MoveGuard, now time.Time, field func(*MoveState) **time.Time) func(*Record) error { |
| 1910 | return func(rec *Record) error { |
| 1911 | if err := g.check(rec); err != nil { |
| 1912 | return err |
| 1913 | } |
| 1914 | if rec.Move.State != MoveCommitted { |
| 1915 | return fmt.Errorf("%w: this follows the commit", ErrMoveConflict) |
| 1916 | } |
| 1917 | if p := field(rec.Move); *p == nil { |
| 1918 | t := now.UTC() |
| 1919 | *p = &t |
| 1920 | } |
| 1921 | return nil |
| 1922 | } |
| 1923 | } |
| 1924 | |
| 1925 | // MoveOpened stamps opened_at on a committed move: the target is serving |
| 1926 | // (§7.9). |
| 1927 | func MoveOpened(g MoveGuard, now time.Time) func(*Record) error { |
| 1928 | return committedStamp(g, now, func(m *MoveState) **time.Time { return &m.OpenedAt }) |
| 1929 | } |
| 1930 | |
| 1931 | // MoveSourceParked stamps source_parked_at on a committed move: the source |
| 1932 | // has parked what it kept (§7.9). |
| 1933 | func MoveSourceParked(g MoveGuard, now time.Time) func(*Record) error { |
| 1934 | return committedStamp(g, now, func(m *MoveState) **time.Time { return &m.SourceParkedAt }) |
| 1935 | } |
| 1936 | |
| 1937 | // LandingDeadlineOf is the deadline a landing before restored works to: |
| 1938 | // the earlier of the cut's absolute deadline and landing_deadline. Spec |
| 1939 | // §7.4 has every pre-commit phase on both participants check the absolute |
| 1940 | // deadline; the flip's landing_deadline is a second, separate bound, never |
| 1941 | // an extension of it (final review, finding 3). which names the one that |
| 1942 | // applies, for the log. The box's landing and rollback decisions (host's |
| 1943 | // landingDeadlineOf) and the move's timings (moves.Timings) share it |
| 1944 | // (Plan 6 preflight L20). |
| 1945 | func LandingDeadlineOf(m *MoveState) (d *time.Time, which string) { |
| 1946 | if m.CutDeadline != nil { |
| 1947 | d, which = m.CutDeadline, "cut_deadline" |
| 1948 | } |
| 1949 | if m.LandingDeadline != nil && (d == nil || m.LandingDeadline.Before(*d)) { |
| 1950 | d, which = m.LandingDeadline, "landing_deadline" |
| 1951 | } |
| 1952 | return d, which |
| 1953 | } |
| 1954 | |
| 1955 | // MoveOfflineS is a move's offline time as its close records it (index D11): |
| 1956 | // from cut_at to the open that ends the move. That is a committed move's |
| 1957 | // opened_at (MoveClose), a rollback's source reopen, rollback.opened_at |
| 1958 | // (MoveRolledBackClose), or a revert's reopened_at (MoveRevertClose). ok is |
| 1959 | // false while either end is unstamped. The three closes and the move's |
| 1960 | // timings (moves.Timings) share it (Plan 6 preflight L20). |
| 1961 | func MoveOfflineS(m *MoveState) (seconds float64, ok bool) { |
| 1962 | if m == nil || m.CutAt == nil { |
| 1963 | return 0, false |
| 1964 | } |
| 1965 | var end *time.Time |
| 1966 | switch m.State { |
| 1967 | case MoveCommitted: |
| 1968 | end = m.OpenedAt |
| 1969 | case MoveRollingBack: |
| 1970 | if m.Rollback != nil { |
| 1971 | end = m.Rollback.OpenedAt |
| 1972 | } |
| 1973 | case MoveReverting: |
| 1974 | end = m.ReopenedAt |
| 1975 | } |
| 1976 | if end == nil { |
| 1977 | return 0, false |
| 1978 | } |
| 1979 | return end.Sub(*m.CutAt).Seconds(), true |
| 1980 | } |
| 1981 | |
| 1982 | // MoveClose is the close (§7.9 step 5, index D11): a committed move with |
| 1983 | // both obligations stamped clears Move and leaves the outcome in LastMove, |
| 1984 | // in the same write. The region is already the target's from the flip; a |
| 1985 | // record whose region is not is refused rather than closed on a lie. The |
| 1986 | // offline time is opened_at minus cut_at (MoveOfflineS), zero when cut_at |
| 1987 | // was never stamped. Plan 5 calls it once no obligation under the move |
| 1988 | // remains. |
| 1989 | func MoveClose(g MoveGuard, now time.Time) func(*Record) error { |
| 1990 | return func(rec *Record) error { |
| 1991 | if err := g.check(rec); err != nil { |
| 1992 | return err |
| 1993 | } |
| 1994 | m := rec.Move |
| 1995 | if m.State != MoveCommitted || m.OpenedAt == nil || m.SourceParkedAt == nil { |
| 1996 | return fmt.Errorf("%w: close follows the open and the source park", ErrMoveConflict) |
| 1997 | } |
| 1998 | if rec.Region != m.To { |
| 1999 | return fmt.Errorf("%w: the record's region is not the target's", ErrMoveConflict) |
| 2000 | } |
| 2001 | out := &MoveOutcome{ID: g.ID, Outcome: "done", At: now.UTC()} |
| 2002 | if s, ok := MoveOfflineS(m); ok { |
| 2003 | out.OfflineS = s |
| 2004 | } |
| 2005 | rec.LastMove, rec.Move = out, nil |
| 2006 | return nil |
| 2007 | } |
| 2008 | } |
| 2009 | |
| 2010 | // RollbackTargetFenced is the recorded reason for a verified target fence. |
| 2011 | const RollbackTargetFenced = "target fenced" |
| 2012 | |
| 2013 | // RollbackDeclaredLost is the rollback reason the declare-lost door |
| 2014 | // records (§7.10): the target box was declared lost before the move |
| 2015 | // committed. A recorded sentence, so it names no box; the box is in the |
| 2016 | // loss record LossRef names. |
| 2017 | const RollbackDeclaredLost = "the target box was declared lost" |
| 2018 | |
| 2019 | // The reasons a move may record when it stops for an operator (§7.8 stuck, |
| 2020 | // §7.4 reverting). Each reaches `carlos move status` through |
| 2021 | // Move.StuckReason, so each is a recorded sentence with a copy row, and each |
| 2022 | // belongs to the phases MoveMarkStuck names for it. |
| 2023 | const ( |
| 2024 | // StuckBaselineMismatch: the source's files or timers do not hash to |
| 2025 | // what it recorded when it stopped (INV-4). A rollback's clean return |
| 2026 | // at restoring or verified, or a revert's verify. |
| 2027 | StuckBaselineMismatch = "the source no longer matches what it recorded when it stopped" |
| 2028 | // StuckRecoveryProbeAttempts: a recovery probe, a rollback's or a |
| 2029 | // revert's, used every attempt it was given without the app answering |
| 2030 | // (§7.7, nine). |
| 2031 | StuckRecoveryProbeAttempts = "the recovery probe ran out of attempts" |
| 2032 | // StuckRevertProbeBudget: a revert's probes failed for 15 minutes. |
| 2033 | StuckRevertProbeBudget = "the source did not answer within 15 minutes of moving back" |
| 2034 | // StuckReturnFilesOccupied: a dirty return's install (Plan 4 Task 3, |
| 2035 | // ruling J13) found the files directory or <host>.env holding what it |
| 2036 | // did not write after the source's own pre-move files were parked. A |
| 2037 | // rollback whose source is at local. |
| 2038 | StuckReturnFilesOccupied = "the files directory holds files the return did not write" |
| 2039 | // StuckReturnManifestOccupied: a dirty return's install found the |
| 2040 | // prepared files manifest's key under the source's location holding a |
| 2041 | // manifest it did not write (moves Task 6, C8). A rollback whose source |
| 2042 | // is at local. |
| 2043 | StuckReturnManifestOccupied = "the source already holds a files manifest the return did not write" |
| 2044 | // StuckReturnInstallMismatch: the database a dirty return built from its |
| 2045 | // lane does not verify. A rollback whose source is at configured, where |
| 2046 | // the database is installed. |
| 2047 | StuckReturnInstallMismatch = "the returned database does not match the build" |
| 2048 | // StuckReturnUnknownStamp: a dirty return's install found the source's |
| 2049 | // inventory listing paths under a stamp that is neither the source's own |
| 2050 | // S nor one the claim shows superseded on this box (live, unknown, or |
| 2051 | // another holder's). Nothing is moved. A rollback whose source is at |
| 2052 | // local. |
| 2053 | StuckReturnUnknownStamp = "the source holds files of an installation the return cannot account for" |
| 2054 | // The target's dirty return (Plan 4 Task 4, fix round 1, ruling 2): |
| 2055 | // what the target must send back cannot be built from what it holds, |
| 2056 | // so no retry by itself can help. Each is a dirty rollback's whose |
| 2057 | // target is at returning; the target keeps its phase and everything it |
| 2058 | // holds, and a retry runs the return again. |
| 2059 | // |
| 2060 | // StuckReturnTargetDatabase: the stopped database the return must |
| 2061 | // snapshot is missing or not a database. |
| 2062 | StuckReturnTargetDatabase = "the target's stopped database is missing" |
| 2063 | // StuckReturnTargetChanged: the database, its position, the files or |
| 2064 | // the timers are no longer what the target recorded when it stopped. |
| 2065 | StuckReturnTargetChanged = "the target changed after it stopped" |
| 2066 | // StuckReturnTargetFiles: the target's files directory cannot be |
| 2067 | // archived (a special file, over the cap, too many entries, a path too |
| 2068 | // long). |
| 2069 | StuckReturnTargetFiles = "the target's files cannot be archived" |
| 2070 | // StuckReleaseTargetFiles (Plan 5 Task 9a): the target that served |
| 2071 | // stopped, and its files directory cannot be archived when it is |
| 2072 | // digested for its stopped report (a special file, over the cap, too |
| 2073 | // many entries, a path too long). A rollback whose target is at |
| 2074 | // releasing, the source still waiting; the target keeps its phase and |
| 2075 | // everything it holds, and a retry runs the release again. |
| 2076 | StuckReleaseTargetFiles = "the target's files cannot be archived now that it has stopped" |
| 2077 | ) |
| 2078 | |
| 2079 | // The refusals of this section that reach an operator through a door |
| 2080 | // (phase1Conflict passes each through; copy record, Plan 4). |
| 2081 | const ( |
| 2082 | // RefusedCommitted: MoveRollBack found the record committed. The commit |
| 2083 | // won the race (§7.9); the target is authoritative. |
| 2084 | RefusedCommitted = "the move has committed. it cannot roll back" |
| 2085 | // RefusedNotStuck: MoveRetry on a move that is not stuck. |
| 2086 | RefusedNotStuck = "the move is not stuck. there is nothing to retry" |
| 2087 | // RefusedNothingToDeclare: MoveRollbackTargetLost from a target phase |
| 2088 | // that is past the window a loss can be declared in. |
| 2089 | RefusedNothingToDeclare = "the target holds nothing the move still needs. there is nothing to declare lost" |
| 2090 | ) |
| 2091 | |
| 2092 | // rollbackReleaseBudget bounds the target's release (§7.8): the console |
| 2093 | // pages on ReleaseDeadline. |
| 2094 | const rollbackReleaseBudget = 15 * time.Minute |
| 2095 | |
| 2096 | // MoveRollBack is the rollback decision (§7.8): a landing, at any landing |
| 2097 | // phase, to rolling_back, in one record write that also moves the region |
| 2098 | // back to From. It is the landing's decision against the commit's: a |
| 2099 | // committed record refuses it (RefusedCommitted), and once it is written |
| 2100 | // the commit is refused, because the state is no longer landing. |
| 2101 | // |
| 2102 | // rbID is the rollback id the caller minted (moves.NewRollbackID). The |
| 2103 | // rollback's reason is the landing's recorded mark when there is one (the |
| 2104 | // first reason stands, as in MoveMarkRollback), else reason, which must be |
| 2105 | // a recorded Rollback constant (a reason given beside a mark must be one |
| 2106 | // too, though the mark wins). The mark stays on Move.Reason: it is copied, |
| 2107 | // not moved. LandingPhase and Landing are kept (the target's release reads |
| 2108 | // them). A replay finds rolling_back and is an |
| 2109 | // ErrMoveConflict; the caller re-reads. |
| 2110 | func MoveRollBack(g MoveGuard, rbID, reason string, now time.Time) func(*Record) error { |
| 2111 | return func(rec *Record) error { |
| 2112 | if err := g.check(rec); err != nil { |
| 2113 | return err |
| 2114 | } |
| 2115 | m := rec.Move |
| 2116 | if m.State == MoveCommitted { |
| 2117 | return fmt.Errorf("%w: %s", ErrMoveConflict, RefusedCommitted) |
| 2118 | } |
| 2119 | if m.State != MoveLanding { |
| 2120 | return fmt.Errorf("%w: a rollback is a landing's", ErrMoveConflict) |
| 2121 | } |
| 2122 | if !rollbackIDShape.MatchString(rbID) || now.IsZero() { |
| 2123 | return fmt.Errorf("provision: malformed rollback %q", rbID) |
| 2124 | } |
| 2125 | if reason != "" && !recordedRollbackReason(reason) { |
| 2126 | return fmt.Errorf("provision: rollback reason %q has no recorded sentence", reason) |
| 2127 | } |
| 2128 | why := reason |
| 2129 | if m.Reason != "" { |
| 2130 | why = m.Reason |
| 2131 | } |
| 2132 | rb := &MoveRollback{ID: rbID, TargetPhase: RbTargetReleasing, SourcePhase: RbSourceWaiting} |
| 2133 | switch why { |
| 2134 | case RollbackConfigChanged: |
| 2135 | rb.Reason = RollbackConfigChanged |
| 2136 | case RollbackLandingDeadline: |
| 2137 | rb.Reason = RollbackLandingDeadline |
| 2138 | case RollbackClaimWired: |
| 2139 | rb.Reason = RollbackClaimWired |
| 2140 | case RollbackLandingLayout: |
| 2141 | rb.Reason = RollbackLandingLayout |
| 2142 | case RollbackInstallMismatch: |
| 2143 | rb.Reason = RollbackInstallMismatch |
| 2144 | case RollbackFilesOccupied: |
| 2145 | rb.Reason = RollbackFilesOccupied |
| 2146 | case RollbackManifestOccupied: |
| 2147 | rb.Reason = RollbackManifestOccupied |
| 2148 | case RollbackProbeAttempts: |
| 2149 | rb.Reason = RollbackProbeAttempts |
| 2150 | case RollbackCommitDeadline: |
| 2151 | rb.Reason = RollbackCommitDeadline |
| 2152 | case RollbackProbeUnpriced: |
| 2153 | rb.Reason = RollbackProbeUnpriced |
| 2154 | case RollbackTargetFenced: |
| 2155 | rb.Reason = RollbackTargetFenced |
| 2156 | case RollbackDeclaredLost: |
| 2157 | rb.Reason = RollbackDeclaredLost |
| 2158 | default: |
| 2159 | return fmt.Errorf("provision: rollback reason %q has no recorded sentence", why) |
| 2160 | } |
| 2161 | t := now.UTC() |
| 2162 | d := t.Add(rollbackReleaseBudget) |
| 2163 | rb.RequestedAt, rb.ReleaseDeadline = &t, &d |
| 2164 | rec.Region = m.From |
| 2165 | m.State, m.Rollback = MoveRollingBack, rb |
| 2166 | return nil |
| 2167 | } |
| 2168 | } |
| 2169 | |
| 2170 | // recordedRollbackReason reports whether r is one of the sentences |
| 2171 | // MoveRollBack records. |
| 2172 | func recordedRollbackReason(r string) bool { |
| 2173 | switch r { |
| 2174 | case RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch, |
| 2175 | RollbackFilesOccupied, RollbackManifestOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost, RollbackTargetFenced, RollbackProbeUnpriced: |
| 2176 | return true |
| 2177 | } |
| 2178 | return false |
| 2179 | } |
| 2180 | |
| 2181 | // rollingBack checks the guard and that the record is a rolling_back move |
| 2182 | // with its journal. |
| 2183 | func rollingBack(g MoveGuard, rec *Record) (*MoveRollback, error) { |
| 2184 | if err := g.check(rec); err != nil { |
| 2185 | return nil, err |
| 2186 | } |
| 2187 | if rec.Move.State != MoveRollingBack || rec.Move.Rollback == nil { |
| 2188 | return nil, fmt.Errorf("%w: the move is %s, not rolling back", ErrMoveConflict, rec.Move.State) |
| 2189 | } |
| 2190 | return rec.Move.Rollback, nil |
| 2191 | } |
| 2192 | |
| 2193 | // rbTargetOrder is the target's rollback order (§7.8 target table). A |
| 2194 | // clean release skips returning and returned; a dirty one may not. |
| 2195 | var rbTargetOrder = []string{RbTargetReleasing, RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning, RbTargetCleaned} |
| 2196 | |
| 2197 | // MoveRollbackTargetPhase advances the target's rollback phase one step |
| 2198 | // under the guard (§7.8). Beyond the order: released goes to returning |
| 2199 | // only when Dirty and to cleaning only when not, and cleaning waits for the |
| 2200 | // source's RecoveredAt (Review Focus 4). It never leaves lost: a declared |
| 2201 | // loss has its own closures (MoveRollbackLossCommitted, |
| 2202 | // MoveLostTargetParked). stamp runs after the checks. |
| 2203 | func MoveRollbackTargetPhase(g MoveGuard, from, to string, stamp func(*MoveRollback)) func(*Record) error { |
| 2204 | return func(rec *Record) error { |
| 2205 | rb, err := rollingBack(g, rec) |
| 2206 | if err != nil { |
| 2207 | return err |
| 2208 | } |
| 2209 | ok := rb.TargetPhase == from |
| 2210 | switch { |
| 2211 | case !ok: |
| 2212 | case from == RbTargetReleased && to == RbTargetCleaning: |
| 2213 | ok = !rb.Dirty |
| 2214 | case from == RbTargetReleased && to == RbTargetReturning: |
| 2215 | ok = rb.Dirty |
| 2216 | default: |
| 2217 | ok = nextIn(rbTargetOrder, from, to) |
| 2218 | } |
| 2219 | if ok && to == RbTargetCleaning && rb.RecoveredAt == nil { |
| 2220 | ok = false |
| 2221 | } |
| 2222 | if !ok { |
| 2223 | return fmt.Errorf("%w: the target's rollback is %s (dirty %t), step expects %s to %s", ErrMoveConflict, rb.TargetPhase, rb.Dirty, from, to) |
| 2224 | } |
| 2225 | rb.TargetPhase = to |
| 2226 | if stamp != nil { |
| 2227 | stamp(rb) |
| 2228 | } |
| 2229 | return nil |
| 2230 | } |
| 2231 | } |
| 2232 | |
| 2233 | // The source's rollback orders (§7.8 source table): the clean return |
| 2234 | // verifies its own copy, the dirty return installs the target's. |
| 2235 | var ( |
| 2236 | rbSourceHead = []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring} |
| 2237 | rbSourceClean = []string{RbSourceRestoring, RbSourceVerified, RbSourceProbing} |
| 2238 | rbSourceDirty = []string{RbSourceRestoring, RbSourceParking, RbSourceBuilding, RbSourcePublished, RbSourceLocal, RbSourceConfigured, RbSourceRestored, RbSourceProbing} |
| 2239 | rbSourceTail = []string{RbSourceProbing, RbSourceProbed, RbSourceRecovered, RbSourceOpened} |
| 2240 | ) |
| 2241 | |
| 2242 | // MoveRollbackSourcePhase advances the source's rollback phase one step |
| 2243 | // under the guard (§7.8). The branch after restoring is the target's |
| 2244 | // Dirty: clean verifies, dirty installs. waiting ends only on the target's |
| 2245 | // evidence, released clean or returned dirty; a target at lost is never |
| 2246 | // taken (§7.10), only the released a loss commit writes. stamp runs after |
| 2247 | // the checks. |
| 2248 | func MoveRollbackSourcePhase(g MoveGuard, from, to string, stamp func(*MoveRollback)) func(*Record) error { |
| 2249 | return func(rec *Record) error { |
| 2250 | rb, err := rollingBack(g, rec) |
| 2251 | if err != nil { |
| 2252 | return err |
| 2253 | } |
| 2254 | branch := rbSourceClean |
| 2255 | if rb.Dirty { |
| 2256 | branch = rbSourceDirty |
| 2257 | } |
| 2258 | ok := rb.SourcePhase == from && (nextIn(rbSourceHead, from, to) || nextIn(branch, from, to) || nextIn(rbSourceTail, from, to)) |
| 2259 | if ok && from == RbSourceWaiting { |
| 2260 | ok = (rb.TargetPhase == RbTargetReleased && !rb.Dirty) || (rb.TargetPhase == RbTargetReturned && rb.Dirty) |
| 2261 | } |
| 2262 | if !ok { |
| 2263 | return fmt.Errorf("%w: the source's rollback is %s (target %s, dirty %t), step expects %s to %s", ErrMoveConflict, rb.SourcePhase, rb.TargetPhase, rb.Dirty, from, to) |
| 2264 | } |
| 2265 | rb.SourcePhase = to |
| 2266 | if stamp != nil { |
| 2267 | stamp(rb) |
| 2268 | } |
| 2269 | return nil |
| 2270 | } |
| 2271 | } |
| 2272 | |
| 2273 | // MoveRecordRecoveryInputs records the inputs a rollback's recovery probe |
| 2274 | // is about to render (§7.7, J2-R2-5; Plan 4 Task 5, Task 3b concern 3): the |
| 2275 | // probe renders the current bundle, overlay and release and records them |
| 2276 | // first, so the held render and the launch it then writes are bound to what |
| 2277 | // the record says (MovesClaimPolicy). It is legal only on this rollback |
| 2278 | // (rbID) while its source is at probing, and writes Rollback.RecoveryInputs |
| 2279 | // alone: a dirty return's configured recorded its own, and a restarted |
| 2280 | // probe's are the ones it renders now; neither is ever compared with |
| 2281 | // move.inputs. The release must be named. in is copied. |
| 2282 | func MoveRecordRecoveryInputs(g MoveGuard, rbID string, in MoveInputs) func(*Record) error { |
| 2283 | return func(rec *Record) error { |
| 2284 | if in.Release == "" { |
| 2285 | return fmt.Errorf("provision: recovery inputs name the release") |
| 2286 | } |
| 2287 | rb, err := rollingBack(g, rec) |
| 2288 | if err != nil { |
| 2289 | return err |
| 2290 | } |
| 2291 | if rb.ID != rbID || rb.SourcePhase != RbSourceProbing { |
| 2292 | return fmt.Errorf("%w: recovery inputs are recorded by rollback %s at probing (the record is %s at %s)", ErrMoveConflict, rbID, rb.ID, rb.SourcePhase) |
| 2293 | } |
| 2294 | cp := in |
| 2295 | cp.BundleETags = make(map[string]string, len(in.BundleETags)) |
| 2296 | for k, v := range in.BundleETags { |
| 2297 | cp.BundleETags[k] = v |
| 2298 | } |
| 2299 | rb.RecoveryInputs = &cp |
| 2300 | return nil |
| 2301 | } |
| 2302 | } |
| 2303 | |
| 2304 | // MoveMarkStuck stops a recovery for an operator (§7.8 stuck, §7.4 |
| 2305 | // reverting): rolling_back or reverting to stuck, with a recorded reason |
| 2306 | // in Move.StuckReason, the one field it writes besides the state and |
| 2307 | // StuckAt (Move.Reason and Rollback.Reason keep the cause). Each reason |
| 2308 | // belongs to its phases: |
| 2309 | // |
| 2310 | // - StuckBaselineMismatch: a rollback whose source is at restoring or |
| 2311 | // verified (a clean return's verification, INV-4), or a revert (its |
| 2312 | // verify); |
| 2313 | // - StuckRecoveryProbeAttempts: a rollback whose source is at probing; |
| 2314 | // - StuckReturnFilesOccupied, StuckReturnManifestOccupied and |
| 2315 | // StuckReturnUnknownStamp: a dirty rollback whose source is at local |
| 2316 | // (the shared installer, Plan 4 Task 3; the prepared manifest, Task 6); |
| 2317 | // - StuckReturnInstallMismatch: a dirty rollback whose source is at |
| 2318 | // configured; |
| 2319 | // - StuckReturnTargetDatabase, StuckReturnTargetChanged and |
| 2320 | // StuckReturnTargetFiles: a dirty rollback whose target is at returning |
| 2321 | // (the source still waiting for it); the target phase is kept, so a |
| 2322 | // retry runs the return again; |
| 2323 | // - StuckReleaseTargetFiles: a rollback whose target is at releasing, |
| 2324 | // the source still waiting (Plan 5 Task 9a); the target phase is kept, |
| 2325 | // so a retry runs the release again; |
| 2326 | // - StuckRevertProbeBudget: a revert; |
| 2327 | // - StuckRecoveryProbeAttempts: a revert too, whose probe has the recovery |
| 2328 | // probe's nine attempts as well as its fifteen minutes (§7.7; Plan 4 |
| 2329 | // Task 6 fix round 1). |
| 2330 | // |
| 2331 | // A rollback records the source phase in StuckAt, which MoveRetry returns |
| 2332 | // to; a revert has no Rollback journal and resumes from the claim's |
| 2333 | // rev_phase (ruling J20). The same reason again on a stuck record is a |
| 2334 | // replay; another is a conflict. Any other text is refused, and nothing is |
| 2335 | // written. |
| 2336 | func MoveMarkStuck(g MoveGuard, reason string) func(*Record) error { |
| 2337 | return func(rec *Record) error { |
| 2338 | if err := g.check(rec); err != nil { |
| 2339 | return err |
| 2340 | } |
| 2341 | m := rec.Move |
| 2342 | var why string |
| 2343 | switch reason { |
| 2344 | case StuckBaselineMismatch: |
| 2345 | why = StuckBaselineMismatch |
| 2346 | case StuckRecoveryProbeAttempts: |
| 2347 | why = StuckRecoveryProbeAttempts |
| 2348 | case StuckRevertProbeBudget: |
| 2349 | why = StuckRevertProbeBudget |
| 2350 | case StuckReturnFilesOccupied: |
| 2351 | why = StuckReturnFilesOccupied |
| 2352 | case StuckReturnManifestOccupied: |
| 2353 | why = StuckReturnManifestOccupied |
| 2354 | case StuckReturnInstallMismatch: |
| 2355 | why = StuckReturnInstallMismatch |
| 2356 | case StuckReturnUnknownStamp: |
| 2357 | why = StuckReturnUnknownStamp |
| 2358 | case StuckReturnTargetDatabase: |
| 2359 | why = StuckReturnTargetDatabase |
| 2360 | case StuckReturnTargetChanged: |
| 2361 | why = StuckReturnTargetChanged |
| 2362 | case StuckReturnTargetFiles: |
| 2363 | why = StuckReturnTargetFiles |
| 2364 | case StuckReleaseTargetFiles: |
| 2365 | why = StuckReleaseTargetFiles |
| 2366 | default: |
| 2367 | return fmt.Errorf("provision: stuck reason %q has no recorded sentence", reason) |
| 2368 | } |
| 2369 | if m.State == MoveStuck && m.StuckReason == why { |
| 2370 | return nil |
| 2371 | } |
| 2372 | ok := false |
| 2373 | switch m.State { |
| 2374 | case MoveRollingBack: |
| 2375 | if rb := m.Rollback; rb != nil { |
| 2376 | switch why { |
| 2377 | case StuckBaselineMismatch: |
| 2378 | ok = rb.SourcePhase == RbSourceRestoring || rb.SourcePhase == RbSourceVerified |
| 2379 | case StuckRecoveryProbeAttempts: |
| 2380 | ok = rb.SourcePhase == RbSourceProbing |
| 2381 | case StuckReturnFilesOccupied, StuckReturnManifestOccupied, StuckReturnUnknownStamp: |
| 2382 | ok = rb.Dirty && rb.SourcePhase == RbSourceLocal |
| 2383 | case StuckReturnInstallMismatch: |
| 2384 | ok = rb.Dirty && rb.SourcePhase == RbSourceConfigured |
| 2385 | case StuckReturnTargetDatabase, StuckReturnTargetChanged, StuckReturnTargetFiles: |
| 2386 | ok = rb.Dirty && rb.TargetPhase == RbTargetReturning && rb.SourcePhase == RbSourceWaiting |
| 2387 | case StuckReleaseTargetFiles: |
| 2388 | ok = rb.TargetPhase == RbTargetReleasing && rb.SourcePhase == RbSourceWaiting |
| 2389 | } |
| 2390 | } |
| 2391 | case MoveReverting: |
| 2392 | ok = why == StuckBaselineMismatch || why == StuckRevertProbeBudget || why == StuckRecoveryProbeAttempts |
| 2393 | } |
| 2394 | if !ok { |
| 2395 | return fmt.Errorf("%w: the move cannot stop here for that reason", ErrMoveConflict) |
| 2396 | } |
| 2397 | if m.Rollback != nil { |
| 2398 | m.Rollback.StuckAt = m.Rollback.SourcePhase |
| 2399 | } |
| 2400 | m.State = MoveStuck |
| 2401 | m.StuckReason = why |
| 2402 | return nil |
| 2403 | } |
| 2404 | } |
| 2405 | |
| 2406 | // MoveRetry is the operator's --retry (§7.8, Task 7): stuck back to the |
| 2407 | // recovery it stopped. A rollback returns to rolling_back with the source |
| 2408 | // at the phase StuckAt names, so a mismatch verifies again and a probe |
| 2409 | // probes again; a revert (no Rollback) returns to reverting. It clears |
| 2410 | // Move.StuckReason and StuckAt: the move's cause (Move.Reason) and the |
| 2411 | // rollback's (Rollback.Reason) stay. It leaves RecoveryRetryEpoch alone, so |
| 2412 | // the legacy (flag-off) retry writes the same bytes it always did; the |
| 2413 | // claim-backed retry is MoveRetryRecovery. |
| 2414 | func MoveRetry(g MoveGuard) func(*Record) error { |
| 2415 | return moveRetry(g, false) |
| 2416 | } |
| 2417 | |
| 2418 | // MoveRetryRecovery is MoveRetry for the claim-backed retry: in the same |
| 2419 | // transition it advances RecoveryRetryEpoch, which grants the recovery probe |
| 2420 | // a fresh durable start budget. |
| 2421 | func MoveRetryRecovery(g MoveGuard) func(*Record) error { |
| 2422 | return moveRetry(g, true) |
| 2423 | } |
| 2424 | |
| 2425 | func moveRetry(g MoveGuard, advanceEpoch bool) func(*Record) error { |
| 2426 | return func(rec *Record) error { |
| 2427 | if err := g.check(rec); err != nil { |
| 2428 | return err |
| 2429 | } |
| 2430 | m := rec.Move |
| 2431 | if m.State != MoveStuck { |
| 2432 | return fmt.Errorf("%w: %s", ErrMoveConflict, RefusedNotStuck) |
| 2433 | } |
| 2434 | if advanceEpoch && m.RecoveryRetryEpoch == ^uint64(0) { |
| 2435 | return ErrMoveConflict |
| 2436 | } |
| 2437 | if rb := m.Rollback; rb != nil { |
| 2438 | if rb.StuckAt == "" { |
| 2439 | return fmt.Errorf("%w: the stuck rollback records no phase to return to", ErrMoveConflict) |
| 2440 | } |
| 2441 | m.State, rb.SourcePhase, rb.StuckAt = MoveRollingBack, rb.StuckAt, "" |
| 2442 | } else { |
| 2443 | m.State = MoveReverting |
| 2444 | } |
| 2445 | m.StuckReason = "" |
| 2446 | if advanceEpoch { |
| 2447 | m.RecoveryRetryEpoch++ |
| 2448 | } |
| 2449 | return nil |
| 2450 | } |
| 2451 | } |
| 2452 | |
| 2453 | // MoveRollbackTargetLost is the declared-loss decision on the record |
| 2454 | // (§7.10): the target phase goes to lost, recording the loss record key |
| 2455 | // (or the termination fence record key) and when. Legal while the target |
| 2456 | // still holds something the source needs: releasing, returning, or |
| 2457 | // released dirty and not yet returned. Dirty is left as it was. The same |
| 2458 | // lossRef again is a replay; another is a conflict. The source's recovery |
| 2459 | // does not take lost: MoveRollbackLossCommitted moves it on once the claim |
| 2460 | // carries the declaration. |
| 2461 | func MoveRollbackTargetLost(g MoveGuard, lossRef string, now time.Time) func(*Record) error { |
| 2462 | return func(rec *Record) error { |
| 2463 | if lossRef == "" || now.IsZero() { |
| 2464 | return fmt.Errorf("provision: malformed loss declaration") |
| 2465 | } |
| 2466 | rb, err := rollingBack(g, rec) |
| 2467 | if err != nil { |
| 2468 | return err |
| 2469 | } |
| 2470 | switch { |
| 2471 | case rb.TargetPhase == RbTargetLost && rb.LossRef == lossRef: |
| 2472 | return nil |
| 2473 | case rb.TargetPhase == RbTargetLost: |
| 2474 | return fmt.Errorf("%w: another loss is recorded for this target", ErrMoveConflict) |
| 2475 | case rb.TargetPhase == RbTargetReleasing, rb.TargetPhase == RbTargetReturning, rb.TargetPhase == RbTargetReleased && rb.Dirty: |
| 2476 | default: |
| 2477 | return fmt.Errorf("%w: %s", ErrMoveConflict, RefusedNothingToDeclare) |
| 2478 | } |
| 2479 | t := now.UTC() |
| 2480 | rb.TargetPhase, rb.LossRef, rb.LostAt = RbTargetLost, lossRef, &t |
| 2481 | return nil |
| 2482 | } |
| 2483 | } |
| 2484 | |
| 2485 | // MoveRollbackLossCommitted closes a declared loss on the record (§7.10): |
| 2486 | // lost to released, clean, with LossRef as the evidence, so the source's |
| 2487 | // waiting takes it as a clean release. The caller writes it only after the |
| 2488 | // claim carries the declaration (its loss record committed). A replay |
| 2489 | // finds released with that evidence and changes nothing. |
| 2490 | func MoveRollbackLossCommitted(g MoveGuard) func(*Record) error { |
| 2491 | return func(rec *Record) error { |
| 2492 | rb, err := rollingBack(g, rec) |
| 2493 | if err != nil { |
| 2494 | return err |
| 2495 | } |
| 2496 | if rb.TargetPhase == RbTargetReleased && rb.LossRef != "" && rb.Evidence == rb.LossRef && !rb.Dirty { |
| 2497 | return nil |
| 2498 | } |
| 2499 | if rb.TargetPhase != RbTargetLost || rb.LossRef == "" { |
| 2500 | return fmt.Errorf("%w: the target's rollback is %s, not lost", ErrMoveConflict, rb.TargetPhase) |
| 2501 | } |
| 2502 | rb.TargetPhase, rb.Dirty, rb.Evidence = RbTargetReleased, false, rb.LossRef |
| 2503 | return nil |
| 2504 | } |
| 2505 | } |
| 2506 | |
| 2507 | // MoveRolledBackClose is a rollback's close (§7.8 rolled_back, index D11): |
| 2508 | // a rolling_back move whose source has recovered and opened and whose |
| 2509 | // target has parked clears Move and leaves the outcome in LastMove, in the |
| 2510 | // same write. The region is already From (MoveRollBack); a record whose |
| 2511 | // region is not is refused. The offline time is the rollback's OpenedAt |
| 2512 | // minus CutAt (MoveOfflineS), zero when CutAt was never stamped. |
| 2513 | func MoveRolledBackClose(g MoveGuard, now time.Time) func(*Record) error { |
| 2514 | return func(rec *Record) error { |
| 2515 | rb, err := rollingBack(g, rec) |
| 2516 | if err != nil { |
| 2517 | return err |
| 2518 | } |
| 2519 | m := rec.Move |
| 2520 | if rb.RecoveredAt == nil || rb.OpenedAt == nil || rb.TargetParkedAt == nil { |
| 2521 | return fmt.Errorf("%w: close follows the recovery, the open and the target park", ErrMoveConflict) |
| 2522 | } |
| 2523 | if rec.Region != m.From { |
| 2524 | return fmt.Errorf("%w: the record's region is not the source's", ErrMoveConflict) |
| 2525 | } |
| 2526 | out := &MoveOutcome{ID: g.ID, Outcome: "rolled_back", At: now.UTC()} |
| 2527 | if s, ok := MoveOfflineS(m); ok { |
| 2528 | out.OfflineS = s |
| 2529 | } |
| 2530 | rec.LastMove, rec.Move = out, nil |
| 2531 | return nil |
| 2532 | } |
| 2533 | } |
| 2534 | |
| 2535 | // MoveRevertReopened stamps ReopenedAt on a reverting move: its source |
| 2536 | // answered the probe (rev_phase probed, ruling J26). The first stamp |
| 2537 | // stands. |
| 2538 | func MoveRevertReopened(g MoveGuard, now time.Time) func(*Record) error { |
| 2539 | return func(rec *Record) error { |
| 2540 | if err := g.check(rec); err != nil { |
| 2541 | return err |
| 2542 | } |
| 2543 | if rec.Move.State != MoveReverting { |
| 2544 | return fmt.Errorf("%w: the reopen is a revert's", ErrMoveConflict) |
| 2545 | } |
| 2546 | if rec.Move.ReopenedAt == nil { |
| 2547 | t := now.UTC() |
| 2548 | rec.Move.ReopenedAt = &t |
| 2549 | } |
| 2550 | return nil |
| 2551 | } |
| 2552 | } |
| 2553 | |
| 2554 | // MoveRecordRevertInputs records the inputs a revert's probe is about to |
| 2555 | // render (Plan 4 Task 6; §7.7, J18): the current bundle, overlay and release, |
| 2556 | // written before the render so the held render and the launch are bound to |
| 2557 | // what the record says. It is legal on a reverting move that has not |
| 2558 | // reopened (a revert has no Rollback journal, so the inputs are the move's |
| 2559 | // own RecoveryInputs), and writes nothing else; a later probe's replace an |
| 2560 | // earlier one's. They are never compared with Inputs. The release must be |
| 2561 | // named. in is copied. |
| 2562 | func MoveRecordRevertInputs(g MoveGuard, in MoveInputs) func(*Record) error { |
| 2563 | return func(rec *Record) error { |
| 2564 | if in.Release == "" { |
| 2565 | return fmt.Errorf("provision: recovery inputs name the release") |
| 2566 | } |
| 2567 | if err := g.check(rec); err != nil { |
| 2568 | return err |
| 2569 | } |
| 2570 | if rec.Move.State != MoveReverting || rec.Move.ReopenedAt != nil { |
| 2571 | return fmt.Errorf("%w: a revert's recovery inputs are recorded before it reopens", ErrMoveConflict) |
| 2572 | } |
| 2573 | cp := in |
| 2574 | cp.BundleETags = make(map[string]string, len(in.BundleETags)) |
| 2575 | for k, v := range in.BundleETags { |
| 2576 | cp.BundleETags[k] = v |
| 2577 | } |
| 2578 | rec.Move.RecoveryInputs = &cp |
| 2579 | return nil |
| 2580 | } |
| 2581 | } |
| 2582 | |
| 2583 | // MoveRevertClose is a revert's close (§7.4 reverting, index D11): the |
| 2584 | // caller has seen the claim's rev_phase at opened; the record needs its |
| 2585 | // ReopenedAt. Move is cleared and LastMove says rolled_back, the offline |
| 2586 | // time counted from CutAt to the reopen (MoveOfflineS). A revert never |
| 2587 | // flipped, so the region is From; a record whose region is not is refused. |
| 2588 | func MoveRevertClose(g MoveGuard, now time.Time) func(*Record) error { |
| 2589 | return func(rec *Record) error { |
| 2590 | if err := g.check(rec); err != nil { |
| 2591 | return err |
| 2592 | } |
| 2593 | m := rec.Move |
| 2594 | if m.State != MoveReverting || m.ReopenedAt == nil { |
| 2595 | return fmt.Errorf("%w: a revert closes after its source reopens", ErrMoveConflict) |
| 2596 | } |
| 2597 | if rec.Region != m.From { |
| 2598 | return fmt.Errorf("%w: the record's region is not the source's", ErrMoveConflict) |
| 2599 | } |
| 2600 | out := &MoveOutcome{ID: g.ID, Outcome: "rolled_back", At: now.UTC()} |
| 2601 | if s, ok := MoveOfflineS(m); ok { |
| 2602 | out.OfflineS = s |
| 2603 | } |
| 2604 | rec.LastMove, rec.Move = out, nil |
| 2605 | return nil |
| 2606 | } |
| 2607 | } |
| 2608 | |
| 2609 | // InstanceStoreEnv is the environment name an instance's private object |
| 2610 | // store hangs off: the record's own Environment, or its Slug when it has |
| 2611 | // none. Twin of the create door's inline rule (internal/console/ |
| 2612 | // apiinstances.go, "env, derived := rec.Environment, false"); the door is |
| 2613 | // not refactored to call this, so keep the two in step. Shared by the |
| 2614 | // console's move door and the host round, which cannot see a console-local |
| 2615 | // helper. |
| 2616 | func InstanceStoreEnv(rec *Record) string { |
| 2617 | if rec.Environment != "" { |
| 2618 | return rec.Environment |
| 2619 | } |
| 2620 | return rec.Slug |
| 2621 | } |
| 2622 | |
| 2623 | // MovePromoteReady is the source box's preparing→ready promotion. |
| 2624 | func MovePromoteReady(now time.Time) func(*Record) error { |
| 2625 | return func(rec *Record) error { |
| 2626 | if rec.Move == nil || rec.Move.State != MovePreparing { |
| 2627 | return fmt.Errorf("%w: ready is only reached from preparing", ErrMoveConflict) |
| 2628 | } |
| 2629 | t := now.UTC() |
| 2630 | rec.Move.State = MoveReady |
| 2631 | rec.Move.ReadyAt = &t |
| 2632 | return nil |
| 2633 | } |
| 2634 | } |
| 2635 | |
| 2636 | // MoveMarkCutting is the console's cut verb: legal only from ready — the |
| 2637 | // tenant freezes writes first, and a cut from preparing would cut over a |
| 2638 | // replica that was never caught up. |
| 2639 | func MoveMarkCutting() func(*Record) error { |
| 2640 | return func(rec *Record) error { |
| 2641 | if rec.Move == nil || rec.Move.State != MoveReady { |
| 2642 | return fmt.Errorf("%w: cut is only legal from ready", ErrMoveConflict) |
| 2643 | } |
| 2644 | rec.Move.State = MoveCutting |
| 2645 | return nil |
| 2646 | } |
| 2647 | } |
| 2648 | |
| 2649 | // MoveAbort marks a move for abort — legal before the point of no return. |
| 2650 | // The clear itself is the source box's (MoveAbortComplete), after it has |
| 2651 | // deleted the staged copy. |
| 2652 | // |
| 2653 | // The legal states are listed, not the illegal ones, so a state this |
| 2654 | // console has never heard of is refused (G16, instance-moves design §6.1). |
| 2655 | // A newer box writes "landing" after the flip, while the target restores |
| 2656 | // from staging. Aborting it would send the target down the aborting arm to |
| 2657 | // delete that staging before it promoted, and the first wake would seed an |
| 2658 | // empty database. |
| 2659 | func MoveAbort() func(*Record) error { |
| 2660 | return func(rec *Record) error { |
| 2661 | if rec.Move == nil { |
| 2662 | return fmt.Errorf("%w: %s", ErrMoveConflict, refusedNoMove) |
| 2663 | } |
| 2664 | switch rec.Move.State { |
| 2665 | case MovePreparing, MoveReady: |
| 2666 | case MoveCutting: |
| 2667 | return fmt.Errorf("%w: past the point of no return — the cut completes forward", ErrMoveConflict) |
| 2668 | case MoveAborting: |
| 2669 | return fmt.Errorf("%w: already aborting", ErrMoveConflict) |
| 2670 | default: |
| 2671 | return fmt.Errorf("%w: a move in state %q cannot be aborted", ErrMoveConflict, rec.Move.State) |
| 2672 | } |
| 2673 | rec.Move.State = MoveAborting |
| 2674 | return nil |
| 2675 | } |
| 2676 | } |
| 2677 | |
| 2678 | // MoveAbortComplete is the source box's half of a legacy abort: staging is |
| 2679 | // gone, the move field clears. It refuses a per-op move (Plan 5 Task 1): that |
| 2680 | // completes through MoveGuardedAbortComplete once the target region has |
| 2681 | // parked its lane and the claim is released, and its source never deletes |
| 2682 | // the target's staging. |
| 2683 | func MoveAbortComplete() func(*Record) error { |
| 2684 | return func(rec *Record) error { |
| 2685 | if rec.Move == nil || rec.Move.State != MoveAborting { |
| 2686 | return fmt.Errorf("%w: abort-complete is only legal from aborting", ErrMoveConflict) |
| 2687 | } |
| 2688 | if !rec.Move.Legacy() { |
| 2689 | return fmt.Errorf("%w: a move with an id completes its abort through the guarded transition", ErrMoveConflict) |
| 2690 | } |
| 2691 | rec.Move = nil |
| 2692 | return nil |
| 2693 | } |
| 2694 | } |
| 2695 | |
| 2696 | // MoveComplete is the source's terminal act: Region flips to the target and |
| 2697 | // Move clears, in ONE write — every reader that knows nothing about moves |
| 2698 | // sees the old region right up until it sees the new one. |
| 2699 | func MoveComplete() func(*Record) error { |
| 2700 | return func(rec *Record) error { |
| 2701 | if rec.Move == nil || rec.Move.State != MoveCutting { |
| 2702 | return fmt.Errorf("%w: complete is only legal from cutting", ErrMoveConflict) |
| 2703 | } |
| 2704 | rec.Region = rec.Move.To |
| 2705 | rec.Move = nil |
| 2706 | return nil |
| 2707 | } |
| 2708 | } |
| 2709 | |
| 2710 | // DeleteRecord retires an instance's route authorization. Object deletion |
| 2711 | // only — the parked database and blobs under the regional bucket's prefix |
| 2712 | // are deliberately untouched (design §3.2); Store's Delete is idempotent, |
| 2713 | // so retiring an already-gone record is not an error. |
| 2714 | func DeleteRecord(ctx context.Context, s release.Store, acct, app, host string) error { |
| 2715 | return s.Delete(ctx, RecordKey(acct, app, host)) |
| 2716 | } |
| 2717 | |
| 2718 | // UpdateRecord rewrites an EXISTING instance record, guarded by the etag the |
| 2719 | // caller read it under. Three writes now touch a record and each has its own |
| 2720 | // contract, deliberately: |
| 2721 | // |
| 2722 | // - CreateRecord is create-only ("" etag): a lost race is a permanent 409, |
| 2723 | // because two signups for one host must never silently merge. |
| 2724 | // - SetRecordRegion is an internal read-modify-write that retries once: it |
| 2725 | // stamps ONE field from a placement decision made elsewhere, so re-reading |
| 2726 | // and re-applying on a lost race changes nothing about what it meant. |
| 2727 | // - This one surfaces the conflict. Its caller is the console's member-facing |
| 2728 | // PATCH door, where the read and the write are one decision — a member sent |
| 2729 | // the upstream set they wanted GIVEN the record they were shown — so a |
| 2730 | // silent retry would re-apply that decision on top of a body nobody read, |
| 2731 | // which is the clobber PutManifestIfMatch's own doc comment records the |
| 2732 | // cost of. |
| 2733 | // |
| 2734 | // The record is marshaled here, by the same json.Marshal every other writer |
| 2735 | // uses: the caller hands over a whole provision.Record (already |
| 2736 | // ValidateShape'd) and never hand-builds the object body, so field ORDER and |
| 2737 | // omitempty behaviour — which the registry's joined addrs column compares |
| 2738 | // verbatim — cannot drift between the create path and this one. |
| 2739 | // |
| 2740 | // An empty etag is rejected rather than quietly becoming a create-only Put: |
| 2741 | // this verb is "rewrite what I read", and a caller that reached here with no |
| 2742 | // etag read nothing. |
| 2743 | func claimRawUpdateRecord(ctx context.Context, s release.Store, rec Record, etag string) error { |
| 2744 | if etag == "" { |
| 2745 | return fmt.Errorf("provision: update %s/%s/%s: no etag — an update must be guarded by the read it was decided from", |
| 2746 | rec.Account, rec.App, rec.Host) |
| 2747 | } |
| 2748 | prior, currentETag, err := s.Get(ctx, RecordKey(rec.Account, rec.App, rec.Host)) |
| 2749 | if err != nil { |
| 2750 | return err |
| 2751 | } |
| 2752 | if currentETag != etag { |
| 2753 | return release.ErrCASConflict |
| 2754 | } |
| 2755 | before, err := DecodeRecord(prior) |
| 2756 | if err != nil { |
| 2757 | return fmt.Errorf("provision: record %s/%s/%s: %w", rec.Account, rec.App, rec.Host, err) |
| 2758 | } |
| 2759 | if !protectedFieldsEqual(before, rec) { |
| 2760 | return ErrMaintenanceProjection |
| 2761 | } |
| 2762 | if !seedEqual(before.Seed, rec.Seed) { |
| 2763 | return ErrSeedProtected |
| 2764 | } |
| 2765 | body, err := json.Marshal(rec) |
| 2766 | if err != nil { |
| 2767 | return err |
| 2768 | } |
| 2769 | _, _, err = WriteRecord(ctx, s, RecordKey(rec.Account, rec.App, rec.Host), prior, body, etag) |
| 2770 | return err |
| 2771 | } |
| 2772 | |
| 2773 | // GetRecord reads and validates one instance record. A body whose own |
| 2774 | // Host/App/Account disagree with the key path it was read from is refused |
| 2775 | // rather than trusted — the same "never guessed at" discipline |
| 2776 | // AccountManifest's sqid-in-path-and-body carries. |
| 2777 | func GetRecord(ctx context.Context, s release.Store, acct, app, host string) (*Record, error) { |
| 2778 | rec, _, err := GetRecordETag(ctx, s, acct, app, host) |
| 2779 | return rec, err |
| 2780 | } |
| 2781 | |
| 2782 | // GetRecordETag is GetRecord carrying the etag its bytes were read under — |
| 2783 | // for a caller that must write back GUARDED against what it read |
| 2784 | // (UpdateRecord), not against whatever is there at write time. Same seam, |
| 2785 | // same reason, as GetManifestETag beside PutManifestIfMatch. |
| 2786 | func GetRecordETag(ctx context.Context, s release.Store, acct, app, host string) (*Record, string, error) { |
| 2787 | raw, etag, err := s.Get(ctx, RecordKey(acct, app, host)) |
| 2788 | if err != nil { |
| 2789 | return nil, "", err |
| 2790 | } |
| 2791 | rec, err := DecodeRecord(raw) |
| 2792 | if err != nil { |
| 2793 | return nil, "", fmt.Errorf("provision: record %s/%s/%s: %w", acct, app, host, err) |
| 2794 | } |
| 2795 | if rec.Account != acct || rec.App != app || rec.Host != host { |
| 2796 | return nil, "", fmt.Errorf("provision: record %s/%s/%s: record disagrees with its own path", acct, app, host) |
| 2797 | } |
| 2798 | return &rec, etag, nil |
| 2799 | } |
| 2800 | |
| 2801 | // recordHosts picks the record hosts out of a listing of RecordsPrefix: |
| 2802 | // the keys of the form <prefix><host>.json, in listing order. Anything else |
| 2803 | // (a key nested below the level, a name without .json) is not a record at |
| 2804 | // this level. Both ListRecords and ListRecordsStrict read through it, so |
| 2805 | // the two never disagree about what a record key looks like. |
| 2806 | func recordHosts(keys []string, prefix string) []string { |
| 2807 | var hosts []string |
| 2808 | for _, key := range keys { |
| 2809 | rest, ok := strings.CutPrefix(key, prefix) |
| 2810 | if !ok || rest == "" || strings.Contains(rest, "/") { |
| 2811 | continue |
| 2812 | } |
| 2813 | host, ok := strings.CutSuffix(rest, ".json") |
| 2814 | if !ok || host == "" { |
| 2815 | continue |
| 2816 | } |
| 2817 | hosts = append(hosts, host) |
| 2818 | } |
| 2819 | return hosts |
| 2820 | } |
| 2821 | |
| 2822 | // ListRecords lists an app's instance records, sorted by host, capped at |
| 2823 | // max with truncated reporting whether more existed. The cap is decided on |
| 2824 | // the KEY count — how many instances/<host>.json objects exist under the |
| 2825 | // prefix — not on how many bodies successfully parse: a handful of |
| 2826 | // malformed bodies among the first max keys must not silently pull in |
| 2827 | // extra keys past the cap to compensate, and truncated must not depend on |
| 2828 | // which records happen to be malformed today. A malformed or |
| 2829 | // path-disagreeing body (GetRecord's own checks) is skipped, not fatal. |
| 2830 | // |
| 2831 | // A negative max is clamped to 0 rather than treated as "no cap" (a caller |
| 2832 | // passing an unvalidated int — a console handler's query param, most |
| 2833 | // likely — must degrade safely, not panic on the slice below or list |
| 2834 | // everything by accident). max=0 lists nothing but still reports |
| 2835 | // truncated when any keys exist, the same "cap decided on key count" rule |
| 2836 | // applied at its floor. |
| 2837 | func ListRecords(ctx context.Context, s release.Store, acct, app string, max int) (recs []Record, truncated bool, err error) { |
| 2838 | recs, truncated, _, err = ListRecordsComplete(ctx, s, acct, app, max) |
| 2839 | return recs, truncated, err |
| 2840 | } |
| 2841 | |
| 2842 | // ListRecordsComplete is ListRecords that also names each host whose record |
| 2843 | // could not be read (a store error, a malformed body, a body that disagrees |
| 2844 | // with its path). A caller that must know whether the list is the whole app |
| 2845 | // (the deploy watch's completeness check, Astra plan 6 R3) reads skipped. |
| 2846 | // Hosts cut by the cap are not skipped; truncated reports those. |
| 2847 | func ListRecordsComplete(ctx context.Context, s release.Store, acct, app string, max int) (recs []Record, truncated bool, skipped []string, err error) { |
| 2848 | if max < 0 { |
| 2849 | max = 0 |
| 2850 | } |
| 2851 | prefix := RecordsPrefix(acct, app) |
| 2852 | keys, err := s.List(ctx, prefix) |
| 2853 | if err != nil { |
| 2854 | return nil, false, nil, err |
| 2855 | } |
| 2856 | hosts := recordHosts(keys, prefix) |
| 2857 | sort.Strings(hosts) |
| 2858 | if len(hosts) > max { |
| 2859 | truncated = true |
| 2860 | hosts = hosts[:max] |
| 2861 | } |
| 2862 | for _, host := range hosts { |
| 2863 | rec, err := GetRecord(ctx, s, acct, app, host) |
| 2864 | if err != nil { |
| 2865 | skipped = append(skipped, host) |
| 2866 | continue |
| 2867 | } |
| 2868 | recs = append(recs, *rec) |
| 2869 | } |
| 2870 | return recs, truncated, skipped, nil |
| 2871 | } |
| 2872 | |
| 2873 | // ErrRecordsTruncated is ListRecordsStrict's answer when the app holds more |
| 2874 | // record keys than the caller's cap. |
| 2875 | var ErrRecordsTruncated = errors.New("provision: the app has more records than the cap allows") |
| 2876 | |
| 2877 | // ListRecordsStrict is ListRecords for a caller that decides something from |
| 2878 | // the whole record set, or from a record's absence: whether another record |
| 2879 | // shares an environment, whether an import is in flight. ListRecords is |
| 2880 | // built for display and quietly drops what it cannot show; this one never |
| 2881 | // does, and the difference is in three places. |
| 2882 | // |
| 2883 | // - A listing that fails, or a record key that will not Get (any error, |
| 2884 | // including not-found: the key was listed a moment ago), is an error and |
| 2885 | // no records come back. ListRecords skips such a record. |
| 2886 | // - More record keys than max is the error ErrRecordsTruncated, not a |
| 2887 | // flag the caller might forget to read. max < 0 is clamped to 0, as in |
| 2888 | // ListRecords, so an app with any record fails at a cap of 0. |
| 2889 | // - A record that reads but does not decode, or whose body disagrees with |
| 2890 | // its own path, is not returned and not skipped silently: its host (the |
| 2891 | // host in the key) is in unreadable. A caller must treat each such host |
| 2892 | // as a record that could be anything, and fail closed on it. |
| 2893 | // |
| 2894 | // recs is sorted by host. unreadable is non-nil whenever err is nil. |
| 2895 | func ListRecordsStrict(ctx context.Context, s release.Store, acct, app string, max int) (recs []Record, unreadable map[string]bool, err error) { |
| 2896 | if max < 0 { |
| 2897 | max = 0 |
| 2898 | } |
| 2899 | prefix := RecordsPrefix(acct, app) |
| 2900 | keys, err := s.List(ctx, prefix) |
| 2901 | if err != nil { |
| 2902 | return nil, nil, fmt.Errorf("provision: list records of %s/%s: %w", acct, app, err) |
| 2903 | } |
| 2904 | hosts := recordHosts(keys, prefix) |
| 2905 | if len(hosts) > max { |
| 2906 | return nil, nil, ErrRecordsTruncated |
| 2907 | } |
| 2908 | sort.Strings(hosts) |
| 2909 | unreadable = map[string]bool{} |
| 2910 | for _, host := range hosts { |
| 2911 | raw, _, err := s.Get(ctx, RecordKey(acct, app, host)) |
| 2912 | if err != nil { |
| 2913 | return nil, nil, fmt.Errorf("provision: read record %s: %w", RecordKey(acct, app, host), err) |
| 2914 | } |
| 2915 | rec, err := DecodeRecord(raw) |
| 2916 | if err != nil || rec.Account != acct || rec.App != app || rec.Host != host { |
| 2917 | unreadable[host] = true |
| 2918 | continue |
| 2919 | } |
| 2920 | recs = append(recs, rec) |
| 2921 | } |
| 2922 | return recs, unreadable, nil |
| 2923 | } |
| 2924 | |
| 2925 | // SocketDir is where a hibernating instance's control socket lives — the |
| 2926 | // same tmpfs every other exec-managed route already uses |
| 2927 | // (host/bootstrap.go's tmpfiles.d entry). |
| 2928 | const SocketDir = "/run/carlos" |
| 2929 | |
| 2930 | // DefaultSocket and DefaultDBPath are the route conventions `carlos add` |
| 2931 | // and the reconciler's mint (Task 3) both build on — the "honest shared |
| 2932 | // core" design §9 names, so a hand-added route and a provisioned one land |
| 2933 | // on identical path shapes. |
| 2934 | // |
| 2935 | // filepath.Join, not concatenation: dataDir is CARLOS_DATA_DIR, an operator- |
| 2936 | // typed env var, and "/data/" is an ordinary way to write a directory. |
| 2937 | // Concatenating gave "/data//h.db" — the same file under a second name, and |
| 2938 | // two names for one database is how a route and litestream's box-wide config |
| 2939 | // come to disagree about what they are replicating. (The literals runAdd |
| 2940 | // used before this helper existed were Join'd; this keeps that.) |
| 2941 | func DefaultSocket(host string) string { return filepath.Join(SocketDir, host+".sock") } |
| 2942 | func DefaultDBPath(dataDir, host string) string { return filepath.Join(dataDir, host+".db") } |
| 2943 | |
| 2944 | // MaxInstanceID bounds InstanceID. A unix socket address holds MaxUnixPath |
| 2945 | // bytes of path and a host may be 253 bytes, so the directories a socket |
| 2946 | // lives under are keyed by a bounded identifier, never by the host itself |
| 2947 | // (isolated-exec design §3.1, codex R5-3). |
| 2948 | const MaxInstanceID = 40 |
| 2949 | |
| 2950 | // MaxUnixPath is sockaddr_un's sun_path (108 bytes) minus the NUL Go |
| 2951 | // insists on room for. |
| 2952 | const MaxUnixPath = 107 |
| 2953 | |
| 2954 | // ControlSocketDir and ControlSocketPath are the root instance-control |
| 2955 | // channel (design §5). Deliberately NOT under SocketDir: an instance's |
| 2956 | // namespace replaces /run/carlos with an empty tmpfs, and nothing an |
| 2957 | // instance can reach must resolve to this path. |
| 2958 | // |
| 2959 | // Defined in internal/controlsock (a leaf package with no imports) and |
| 2960 | // re-exported here unchanged: provision already imports internal/release |
| 2961 | // (for the store), and internal/unixdial — imported by internal/release's |
| 2962 | // unix dial sites — needs this path without pulling provision (and so |
| 2963 | // release) back in. |
| 2964 | const ( |
| 2965 | ControlSocketDir = controlsock.Dir |
| 2966 | ControlSocketPath = controlsock.Path |
| 2967 | ) |
| 2968 | |
| 2969 | // InstanceID is the bounded per-host identifier every isolated-exec |
| 2970 | // directory, socket, unit instance and slice is keyed by. A host of at |
| 2971 | // most MaxInstanceID bytes is its own identifier. A longer host becomes |
| 2972 | // "_" + its first 22 bytes + "-" + 16 hex digits of its sha256, also |
| 2973 | // MaxInstanceID bytes. |
| 2974 | // |
| 2975 | // The underscore is the whole reason the two forms can share one |
| 2976 | // namespace: no valid host contains one (controlname labels are |
| 2977 | // [a-z0-9-]), so a hashed identifier can never equal any short host, and |
| 2978 | // two hashed identifiers collide only on a 64-bit hash collision between |
| 2979 | // two long hosts with the same 22-byte prefix (codex R6-1). Anything that |
| 2980 | // finds two rows with one identifier treats them as a conflict, never as |
| 2981 | // the same host. |
| 2982 | func InstanceID(host string) string { |
| 2983 | if len(host) <= MaxInstanceID { |
| 2984 | return host |
| 2985 | } |
| 2986 | sum := sha256.Sum256([]byte(host)) |
| 2987 | return "_" + host[:22] + "-" + hex.EncodeToString(sum[:8]) |
| 2988 | } |
| 2989 | |
| 2990 | // InstanceDir is the per-host data directory of the nested layout: |
| 2991 | // <dataDir>/instances/<InstanceID(host)>. File names INSIDE it keep the |
| 2992 | // full host (a file name may be 255 bytes; a socket path may not). |
| 2993 | func InstanceDir(dataDir, host string) string { |
| 2994 | return filepath.Join(dataDir, "instances", InstanceID(host)) |
| 2995 | } |
| 2996 | |
| 2997 | // NestedDBPath is the database under InstanceDir. It is what the layout |
| 2998 | // pass writes into a migrated row's DBPath; DefaultDBPath stays flat in |
| 2999 | // this release (design §8.1). |
| 3000 | func NestedDBPath(dataDir, host string) string { |
| 3001 | return filepath.Join(InstanceDir(dataDir, host), host+".db") |
| 3002 | } |
| 3003 | |
| 3004 | // InstanceRunDir is the per-host socket directory of the nested layout. |
| 3005 | func InstanceRunDir(host string) string { |
| 3006 | return filepath.Join(SocketDir, "instances", InstanceID(host)) |
| 3007 | } |
| 3008 | |
| 3009 | // NestedSocket is the instance's listener under InstanceRunDir; its |
| 3010 | // control socket is NestedSocket + ".control", as for every exec route. |
| 3011 | func NestedSocket(host string) string { |
| 3012 | return filepath.Join(InstanceRunDir(host), "app.sock") |
| 3013 | } |
| 3014 | |
| 3015 | // NestedPaths reports whether a row's database or socket sits anywhere |
| 3016 | // under the nested layout's trees: <dataDir>/instances/ or |
| 3017 | // <SocketDir>/instances/. It reads the path shape only, whatever the row's |
| 3018 | // flags, backing or exact file names say. |
| 3019 | // |
| 3020 | // That is deliberately looser than activator.Classify, which calls a row |
| 3021 | // nested only when both paths equal the nested defaults. The downgrade |
| 3022 | // guards (design §8.2, "That order is enforced") use this test instead: |
| 3023 | // a row can leave management or be hand-edited and still have its files |
| 3024 | // under the instance tree, and a release before the layout, an unflag or |
| 3025 | // a re-add would mishandle it all the same. Do not merge the two. |
| 3026 | // |
| 3027 | // Both paths are cleaned first, so a hand-edited row with a doubled slash |
| 3028 | // or a dot segment is judged by the place it names. |
| 3029 | func NestedPaths(dataDir string, rt registry.Route) bool { |
| 3030 | return strings.HasPrefix(filepath.Clean(rt.DBPath), filepath.Join(dataDir, "instances")+"/") || |
| 3031 | strings.HasPrefix(filepath.Clean(rt.Socket), filepath.Join(SocketDir, "instances")+"/") |
| 3032 | } |
| 3033 | |
| 3034 | // LogSocket is the agent's per-role log listener for a host (design |
| 3035 | // §6.4): outside every tenant mount, keyed by the identifier. |
| 3036 | func LogSocket(host, role string) string { |
| 3037 | return filepath.Join(SocketDir, "logs", InstanceID(host)+"."+role+".sock") |
| 3038 | } |
| 3039 | |
| 3040 | // LayoutJournalPath is the migration journal for a host in flight |
| 3041 | // (isolated-exec design §8.2): under <dataDir>/layout, outside every tenant |
| 3042 | // mount. Its presence is what makes a host `fenced` (§8.1); plan 4 writes |
| 3043 | // it, and every consumer that must not act on a half-migrated host checks |
| 3044 | // for it here. |
| 3045 | func LayoutJournalPath(dataDir, host string) string { |
| 3046 | return filepath.Join(dataDir, "layout", host+".json") |
| 3047 | } |
| 3048 | |
| 3049 | // LayoutJournalExists reports whether a journal is at LayoutJournalPath. |
| 3050 | // It Lstats, so a dangling symlink there counts, and any error other than |
| 3051 | // "not exist" counts too: a journal this process cannot even stat is not |
| 3052 | // proof there isn't one. |
| 3053 | func LayoutJournalExists(dataDir, host string) bool { |
| 3054 | _, err := os.Lstat(LayoutJournalPath(dataDir, host)) |
| 3055 | return !errors.Is(err, os.ErrNotExist) |
| 3056 | } |
| 3057 | |
| 3058 | // IsControlEndpoint reports whether path is the control channel or |
| 3059 | // anything else under its directory. The edge and the agent refuse to |
| 3060 | // dial such a path as an upstream (design §5.4): a registry row pointing |
| 3061 | // at it is refused before connect(). Delegates to internal/controlsock; |
| 3062 | // see the ControlSocketDir/ControlSocketPath comment above for why. |
| 3063 | func IsControlEndpoint(path string) bool { |
| 3064 | return controlsock.IsEndpoint(path) |
| 3065 | } |
| 3066 | |
| 3067 | // LongestSocketAddress is the longest unix socket path any helper here |
| 3068 | // can produce, over the longest valid host, so a test can pin it under |
| 3069 | // MaxUnixPath rather than trusting arithmetic in a comment. |
| 3070 | func LongestSocketAddress() int { |
| 3071 | host := strings.Repeat("x", 253) |
| 3072 | longest := 0 |
| 3073 | for _, p := range []string{ |
| 3074 | NestedSocket(host), |
| 3075 | NestedSocket(host) + ".control", |
| 3076 | LogSocket(host, "sidecar"), // the longest role name |
| 3077 | } { |
| 3078 | if len(p) > longest { |
| 3079 | longest = len(p) |
| 3080 | } |
| 3081 | } |
| 3082 | return longest |
| 3083 | } |
| 3084 | func PutManifest(ctx context.Context, s release.Store, acct, app string, m Manifest) (claimResult0 error) { |
| 3085 | claimResult0 = release.WithAppMutation(ctx, s, acct, app, "provision.PutManifest", func(claimContext context.Context) error { |
| 3086 | claimResult0 = claimRawPutManifest(claimContext, s, acct, app, m) |
| 3087 | return claimResult0 |
| 3088 | }) |
| 3089 | return |
| 3090 | } |
| 3091 | func PutManifestIfMatch(ctx context.Context, s release.Store, acct, app string, m Manifest, etag string) (claimResult0 error) { |
| 3092 | claimResult0 = release.WithAppMutation(ctx, s, acct, app, "provision.PutManifestIfMatch", func(claimContext context.Context) error { |
| 3093 | claimResult0 = claimRawPutManifestIfMatch(claimContext, s, acct, app, m, etag) |
| 3094 | return claimResult0 |
| 3095 | }) |
| 3096 | return |
| 3097 | } |
| 3098 | |
| 3099 | // quotedInner is strconv.Quote without the surrounding quotes: the approved |
| 3100 | // sentence (MV-N42) carries its own quotes, and the state inside them keeps |
| 3101 | // %q's escaping of quotes, backslashes and control characters. |
| 3102 | func quotedInner(s string) string { |
| 3103 | q := strconv.Quote(s) |
| 3104 | return q[1 : len(q)-1] |
| 3105 | } |
| 3106 | |
amadan