| 1 | package provision |
| 2 | |
| 3 | // Reconcile turns bucket instance records into registry routes, for THIS |
| 4 | // agent's own region and pool set — plus, since the multi-region ingress |
| 5 | // design's Part 3, the serving classes every platform box materializes |
| 6 | // wherever they are homed (see "serving classes" below). Spec: |
| 7 | // docs/superpowers/specs/2026-08-04-instance-provisioning-design.md §4. |
| 8 | // |
| 9 | // DNS seam (spec §10): multi-region deployments write the per-slug CNAME at |
| 10 | // provision time in the console's create handler, not here — the reconcile |
| 11 | // pass stays DNS-free so a box never needs Route53 credentials. |
| 12 | |
| 13 | import ( |
| 14 | "context" |
| 15 | "crypto/ed25519" |
| 16 | "encoding/json" |
| 17 | "errors" |
| 18 | "fmt" |
| 19 | "log" |
| 20 | "net" |
| 21 | "os" |
| 22 | "slices" |
| 23 | "strings" |
| 24 | "sync" |
| 25 | "time" |
| 26 | |
| 27 | "github.com/carlosframework/platform/internal/domains" |
| 28 | "github.com/carlosframework/platform/internal/registry" |
| 29 | "github.com/carlosframework/platform/internal/release" |
| 30 | "github.com/carlosframework/platform/internal/seeds" |
| 31 | ) |
| 32 | |
| 33 | // ReconcileConfig is everything one pass reads its world through — the same |
| 34 | // injection-seam shape domains.Config uses (tests supply fixtures, the host |
| 35 | // agent supplies production values). |
| 36 | type ReconcileConfig struct { |
| 37 | ClaimEnabled bool |
| 38 | ClaimInstall func(context.Context, ClaimRecord, func(registry.ClaimOwnership) error) error |
| 39 | ClaimRetire func(context.Context, registry.Route) error |
| 40 | |
| 41 | // Region is this agent's region code. "" is the unconstrained region — |
| 42 | // a box that hasn't been told its place serves only region-less |
| 43 | // records, never someone else's residency promise. |
| 44 | Region string |
| 45 | // Pools are the pools this box serves. Empty means the shared "" pool |
| 46 | // only — NOT "every pool" (poolServed below). |
| 47 | Pools []string |
| 48 | // DataDir feeds DefaultDBPath ("/data" in production). |
| 49 | DataDir string |
| 50 | // Now is the clock seam; nil = time.Now. |
| 51 | Now func() time.Time |
| 52 | // Inbound is this box's inbound-work declaration (CARLOS_INBOUND, design |
| 53 | // §8). "" and InboundAny are today's behaviour exactly — an inert |
| 54 | // default, the apps-domain-style gate this repo already uses — and |
| 55 | // InboundNone means the box serves sidecar work only: no certs, no |
| 56 | // inbound ports, possibly a private subnet with egress-only networking. |
| 57 | // Read by ReconcileSidecars' placement gate (matches a record's own |
| 58 | // "no-inbound" placement request against this box) AND by Reconcile's |
| 59 | // mint gate below (refuses ALL routed instance minting outright — an |
| 60 | // ordinary instance record carries no placement vocabulary to match). |
| 61 | // |
| 62 | // Callers building this from a raw env reading MUST pass it through |
| 63 | // NormalizeInbound first (host/agent.go's provisionTickConfig does) — |
| 64 | // both gates below compare this field with exact string equality |
| 65 | // against InboundNone, so an un-folded value would defeat them silently. |
| 66 | Inbound string |
| 67 | // Layout answers which layout this box's intent names, "" or |
| 68 | // release.LayoutModeFlat for flat and release.LayoutModeNested for |
| 69 | // nested, read from the deploy store the pass already has in hand |
| 70 | // (isolated-exec design §8.6 step 5, brought forward in plan 5): a row |
| 71 | // minted on a nested box is minted at the nested paths, never flat |
| 72 | // and then migrated, so a new tenant never runs flat beside nested |
| 73 | // siblings, not even for one pass. An error defers every mint on this |
| 74 | // pass rather than guessing: a box that cannot tell its layout must |
| 75 | // not mint a flat row the fleet may want nested. Nil is flat (tests, |
| 76 | // and a caller with no intent to read). |
| 77 | Layout func(ctx context.Context, s release.Store) (string, error) |
| 78 | // DryRun reports what the pass would do and writes nothing — not to the |
| 79 | // registry, not anywhere. The backfill mitigation (design §4.3): a |
| 80 | // production route is adopted only after someone has read what adopting |
| 81 | // it would mean, because adoption is what grants the removal phase |
| 82 | // authority over that row. |
| 83 | DryRun bool |
| 84 | // ReleasePub verifies the production flag before a trash marker is |
| 85 | // allowed to retire an app's routes (#340), the same guard and the same |
| 86 | // key domains.Config carries for the same decision. nil leaves the |
| 87 | // retirement UNGUARDED rather than disabling it — domains §1c's posture, |
| 88 | // carried over deliberately so the two passes cannot reach opposite |
| 89 | // conclusions about the same app: a keyless deployment has no way to |
| 90 | // make the guard pass, and holding every trash forever there would break |
| 91 | // hiding entirely. Nothing destructive to the bucket happens on this |
| 92 | // path; the auto-purge that does still lives in domains, still refuses |
| 93 | // without a key. |
| 94 | ReleasePub ed25519.PublicKey |
| 95 | // ConfigGate answers whether this box can provide the config of the |
| 96 | // route a record would mint (amadan #213): nil error means yes, an error |
| 97 | // means do not mint it this pass. It is handed the route the mint would |
| 98 | // create (routeFor / sidecarRouteFor), so it reads App, Account, |
| 99 | // Environment and the shape off the one renderer, and it decides for |
| 100 | // itself which shapes need config at all. A refused record is retried |
| 101 | // every pass, so a box that gains the key mints on its next tick. The |
| 102 | // gate owns its own logging and operator-visible record; this pass only |
| 103 | // honours the answer. Existing routes are not unminted by a refusal: the |
| 104 | // wake-time gate (activator.ExecBackend.ConfigGate) is what stops one |
| 105 | // starting. Nil: no gate, every record mints as before. |
| 106 | ConfigGate func(ctx context.Context, rt registry.Route) error |
| 107 | // Hold answers whether a record's row on this box belongs to a running |
| 108 | // move (moves spec §5.2, §9.6, §19 K9) and, if so, the hold status and |
| 109 | // the op holding it. When ok, a row minted for rec is minted held |
| 110 | // (status, HoldOp op) at flat paths, never ready and never nested |
| 111 | // (Phase 1 lands flat, G11); an unheld row of this pass's own is held |
| 112 | // with SetHoldIf (Upsert never changes a hold on conflict, G10) once a |
| 113 | // fresh read of the record, under the host's hold lock |
| 114 | // (WithHostHoldLock), still asks for it, except under a recovery hold |
| 115 | // (restoring, releasing), which only mints (holdExisting); a row |
| 116 | // already held, by this op or another, is left as it is. Nil: no move |
| 117 | // holds anything, main's pass exactly. |
| 118 | Hold func(rec Record) (status, op string, ok bool) |
| 119 | // KeepParticipant answers whether a host's claim names this box as a |
| 120 | // participant of a running operation (claim spec §8, moves §9.6): the |
| 121 | // removal phase never removes such a row, whatever the record's region |
| 122 | // says. The operation removes it at its own finalization. Nil: none is. |
| 123 | KeepParticipant func(host string) bool |
| 124 | // HoldWritten is called after every hold this pass writes (a held mint |
| 125 | // or a SetHoldIf that landed), so the caller can touch the |
| 126 | // stage-requested marker: root re-judges what it staged under the new |
| 127 | // hold (registry.SetHold's contract, G10). Nil: nobody is told. |
| 128 | HoldWritten func(host string) |
| 129 | // RunSeen is called with every in-scope record of an untrashed app that |
| 130 | // carries a run, once per pass, before the pass acts on the record (run |
| 131 | // spec §5.2: pickup rides reconcile's own record reads, so an idle box |
| 132 | // pays nothing extra). It must not block: the box's runs worker only |
| 133 | // notes the record and acts after the pass, by which time a row this |
| 134 | // pass minted is in the registry. Nil: no one is told. |
| 135 | RunSeen func(rec Record) |
| 136 | // Seeds is the seeded-instances wiring (seedreconcile.go). Nil: every |
| 137 | // seeded record is left alone. |
| 138 | Seeds *SeedHooks |
| 139 | |
| 140 | // beforeHoldWrite is the tests' pause under the host's hold lock, just |
| 141 | // before holdExisting's conditional write (TestReconcileHoldWrite...). |
| 142 | // Nil in the agent. |
| 143 | beforeHoldWrite func(host string) |
| 144 | } |
| 145 | |
| 146 | // InboundAny and InboundNone are the closed vocabulary ReconcileConfig.Inbound |
| 147 | // is read against — the BOX's half of design §8's two independent halves (the |
| 148 | // record's half is PlacementAny/PlacementNoInbound). "" reads as InboundAny: |
| 149 | // a box that has not been told is a box that serves, which is every box that |
| 150 | // exists today. |
| 151 | const ( |
| 152 | InboundAny = "any" |
| 153 | InboundNone = "none" |
| 154 | ) |
| 155 | |
| 156 | // NormalizeInbound folds a raw CARLOS_INBOUND reading (whitespace and case) |
| 157 | // and maps it onto the closed vocabulary above. Review fix (Task 10 |
| 158 | // finding): the box-side comparisons this feeds are exact string equality |
| 159 | // against InboundNone, so an un-folded "None"/"NONE"/"off" fell through as |
| 160 | // if the variable were unset — a box meant to be no-inbound would silently |
| 161 | // get handed traffic it structurally cannot serve, the same silent outage |
| 162 | // this task exists to prevent, just triggered from the other direction. |
| 163 | // |
| 164 | // An UNRECOGNISED value (anything other than "", "any" or "none" once |
| 165 | // folded) is treated as InboundNone — restrictive, not inert — and logged |
| 166 | // loudly, naming the offending value. The only way to produce an |
| 167 | // unrecognised value is to have typed one: CARLOS_INBOUND is never set by |
| 168 | // anything but an operator's own hand, and an operator only sets this |
| 169 | // variable at all when they mean to restrict. So the restrictive reading |
| 170 | // matches intent in the overwhelmingly likely case, and it is this repo's |
| 171 | // house style for an unparseable declaration already (a provision manifest |
| 172 | // with schema > 1 is refused outright, never guessed at). The failure mode |
| 173 | // of guessing wrong here is loud and bounded — new mints are refused with a |
| 174 | // log line naming host and reason, nothing already running is evacuated — |
| 175 | // where the opposite default's failure mode is a box quietly accepting |
| 176 | // traffic it cannot serve. |
| 177 | func NormalizeInbound(raw string) string { |
| 178 | v := strings.ToLower(strings.TrimSpace(raw)) |
| 179 | switch v { |
| 180 | case "", InboundAny, InboundNone: |
| 181 | return v |
| 182 | default: |
| 183 | log.Printf("provision: CARLOS_INBOUND=%q is not a recognised value (want \"\", %q or %q); "+ |
| 184 | "treating this box as CARLOS_INBOUND=%s (no-inbound, restrictive) rather than guessing it means unrestricted", |
| 185 | raw, InboundAny, InboundNone, InboundNone) |
| 186 | return InboundNone |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | // acctApp identifies one app within one account — the unit both the record |
| 191 | // scan and the manifest/seed cache below key on, since app names are unique |
| 192 | // only within an account, never globally. |
| 193 | type acctApp struct{ acct, app string } |
| 194 | |
| 195 | // trashState is what one app's trash marker means to this pass. Two fields |
| 196 | // and not one bool, because the two questions have different safe answers: |
| 197 | // see the trash cache in Reconcile for which is which. |
| 198 | type trashState struct { |
| 199 | declineMint bool |
| 200 | removable bool |
| 201 | } |
| 202 | |
| 203 | // productionClear reports whether a trashed app's routes may be retired — |
| 204 | // i.e. whether we can positively see that it is NOT flagged production. |
| 205 | // Dropping every route of a production app is an outage, which is precisely |
| 206 | // what the flag exists to prevent, so an unreadable flag refuses in the same |
| 207 | // direction as an ON one: the state an attacker with delete-but-not-sign can |
| 208 | // manufacture is exactly the unreadable one. |
| 209 | // |
| 210 | // A nil key is the keyless box, which acts on the marker — domains §1c's |
| 211 | // posture, and the comment on ReconcileConfig.ReleasePub says why. |
| 212 | // |
| 213 | // Both refusals log EVERY pass, deliberately: an operator who wrote that |
| 214 | // marker on purpose has to be told why nothing happened, or they will assume |
| 215 | // deletion is broken. |
| 216 | func productionClear(ctx context.Context, s release.Store, pub ed25519.PublicKey, acct, app string) bool { |
| 217 | if pub == nil { |
| 218 | return true |
| 219 | } |
| 220 | flag, err := release.GetProduction(ctx, s, pub, acct, app) |
| 221 | switch { |
| 222 | case err != nil: |
| 223 | log.Printf("provision: cannot read the production flag for %s/%s — refusing to act on the trash marker: %v", acct, app, err) |
| 224 | return false |
| 225 | case flag != nil && flag.Production: |
| 226 | log.Printf("provision: trash marker on production app %s/%s — refusing to act on it; turn production off first", acct, app) |
| 227 | return false |
| 228 | } |
| 229 | return true |
| 230 | } |
| 231 | |
| 232 | // ---- serving classes ------------------------------------------------------- |
| 233 | // |
| 234 | // Which boxes materialize a record. The multi-region ingress design's Part 3 |
| 235 | // table, verbatim, and the ONE place this pass states it: |
| 236 | // |
| 237 | // Everywhere static (bake-served): the edge resolves the app's |
| 238 | // channel pointer and serves the shipped tar out of the |
| 239 | // deployment store every box already reads, so no box |
| 240 | // needs the home box to serve it. |
| 241 | // Everywhere, probed addr routes whose upstreams are ALL publicly dialable: |
| 242 | // each edge runs its own prober and dials the backends |
| 243 | // directly, so per-edge live sets are per-edge by design. |
| 244 | // Home-boxed exec, unit-owned, socket-activated, tunnel- and |
| 245 | // fleet-backed: the process exists on exactly ONE box, |
| 246 | // and a request that lands elsewhere backhauls to it |
| 247 | // (internal/edge's hop). Unchanged invariant. |
| 248 | // |
| 249 | // Everything here decides SCOPE only. Nothing about the route a record mints |
| 250 | // changes with the box that mints it — routeFor stays the one renderer, so a |
| 251 | // far box's copy and the home box's copy are the same route, column for |
| 252 | // column, including the record's own Region. |
| 253 | // |
| 254 | // Single-box inertness: with one box there is no far box, every record is in |
| 255 | // scope for the same reason it was before, and the widened half of the gate |
| 256 | // never fires. |
| 257 | |
| 258 | // everywhereClass reports whether rec's serving CLASS is one every platform |
| 259 | // box materializes. It answers the class question only — the tenancy |
| 260 | // questions (pool, fleet placement) are inScope's, below. |
| 261 | // |
| 262 | // Conservative by construction: anything this function cannot positively |
| 263 | // classify as servable from a box that holds nothing but the bucket is |
| 264 | // home-boxed. That is the direction a wrong answer must fail in — a route that |
| 265 | // stays home-boxed is served exactly as it is today (and backhauled from |
| 266 | // elsewhere), while one wrongly promoted to everywhere is a box proxying to an |
| 267 | // upstream it cannot reach. |
| 268 | func everywhereClass(rec Record) bool { |
| 269 | // A replica is the residence rule made structural (#162): it belongs to |
| 270 | // the box its placement names and nowhere else, and this widening is the |
| 271 | // mint gate for every non-fleet box. ValidateShape refuses a replica |
| 272 | // with an upstream, so the addr arm below could never fire on one — but |
| 273 | // "could never through THIS door" is not "cannot", and a replica of a |
| 274 | // static kind would pass the kind arm above if the kind rail ever |
| 275 | // moved. The role check is FIRST and absolute. |
| 276 | if rec.Role == "replica" { |
| 277 | return false |
| 278 | } |
| 279 | // Unit names the systemd unit that owns this route's process (issue |
| 280 | // #110); adoption CYCLES it. That unit exists on the home box, so a far |
| 281 | // box adopting the row would be stamping authority over a unit name that |
| 282 | // means something else there, or nothing at all. |
| 283 | if rec.Unit != "" { |
| 284 | return false |
| 285 | } |
| 286 | // Belt and braces: ValidateShape already refuses hibernation on both |
| 287 | // classes below (a static route has no process to wake, an addr-backed one |
| 288 | // has no socket), and this pass validates every record before the gate. A |
| 289 | // hibernating route needs the activator and the socket only its home box |
| 290 | // has, so if that door ever moved, this must not be the hole it opens. |
| 291 | if rec.HibernateOrDefault() { |
| 292 | return false |
| 293 | } |
| 294 | if rec.KindOrDefault() == "static" { |
| 295 | return true |
| 296 | } |
| 297 | ups := rec.AddrsOrDefault() |
| 298 | if len(ups) == 0 { |
| 299 | return false // socket-backed: the exec child lives on one box |
| 300 | } |
| 301 | for _, a := range ups { |
| 302 | if !publiclyDialable(a) { |
| 303 | return false |
| 304 | } |
| 305 | } |
| 306 | return true |
| 307 | } |
| 308 | |
| 309 | // everywhereRoute is everywhereClass read off a registry ROW instead of a |
| 310 | // record — the removal phase's half of the same question, asked when the |
| 311 | // record is gone and the row is all that is left to ask. |
| 312 | // |
| 313 | // The two must agree, or the widened mint gate leaves immortal routes: a far |
| 314 | // box that mints an everywhere-class route and then cannot see it as one is a |
| 315 | // box serving a retired site forever. They agree by construction today because |
| 316 | // routeFor is the one renderer between them, and |
| 317 | // TestEverywhereClassAgreesAcrossRecordAndRow locks that: a class rule added to |
| 318 | // one of these functions alone recreates exactly the immortality bug this pair |
| 319 | // was written to close, with every other test still green. |
| 320 | // |
| 321 | // They are separate functions rather than one over a shared shape because the |
| 322 | // row carries columns no record can state (Backing, Tunnel, Caps), and a row |
| 323 | // that has GROWN one of those since it was minted is deliberately read as |
| 324 | // home-boxed here. State that cost honestly, because it is bigger than a stale |
| 325 | // row: an operator who runs `carlos route -grant` (or -backing/-tunnel) against |
| 326 | // the FAR box's copy takes that copy out of removal scope permanently. Deleting |
| 327 | // the record then retires the route on its home box and leaves the far box |
| 328 | // still SERVING the retired site, grants intact, with no record left anywhere |
| 329 | // to explain it — so `carlos remove` stops being fleet-wide the moment anyone |
| 330 | // touches a far copy. The alternative (removing rows carrying operator columns) |
| 331 | // hands an unattended bucket-record deletion the grant purge registry.Delete |
| 332 | // documents as human-gated, which is worse. The conservative gate stands; the |
| 333 | // gap is tracked in the bookkeeping issue, not papered over here. |
| 334 | func everywhereRoute(rt registry.Route) bool { |
| 335 | // The replica mark is everywhereClass's role check, read off the row |
| 336 | // (the record whose class it was may be exactly what is gone) — the |
| 337 | // same FIRST-and-absolute position, for the same reason. |
| 338 | if rt.Replica { |
| 339 | return false |
| 340 | } |
| 341 | if rt.Unit != "" || rt.Hibernate || rt.Backing != "" || rt.Tunnel != "" || rt.Caps != "" { |
| 342 | return false |
| 343 | } |
| 344 | if rt.Kind == "static" { |
| 345 | return true |
| 346 | } |
| 347 | ups := rt.AddrList() |
| 348 | if len(ups) == 0 { |
| 349 | return false |
| 350 | } |
| 351 | for _, a := range ups { |
| 352 | if !publiclyDialable(a) { |
| 353 | return false |
| 354 | } |
| 355 | } |
| 356 | return true |
| 357 | } |
| 358 | |
| 359 | // publiclyDialable reports whether one upstream is an address every edge in |
| 360 | // the deployment can reach — the "publicly-reachable upstreams" half of the |
| 361 | // spec's table. |
| 362 | // |
| 363 | // A NAME is not dialable for this purpose even if it resolves to a public |
| 364 | // address today: validAddrsElement's doc comment already records why a name |
| 365 | // cannot be judged at all here (it is free to move after the check), and the |
| 366 | // legacy Addr column is full of names pointing at box-local backends. It stays |
| 367 | // home-boxed, which is where those routes are serving from today. |
| 368 | // |
| 369 | // Loopback and link-local are the spec's own examples (another box's |
| 370 | // 127.0.0.1 is not this box's). RFC1918 and ULA are refused for exactly the |
| 371 | // same reason one layer out: a private address is private to ONE network, and |
| 372 | // two boxes in two regions are two networks. IsGlobalUnicast covers the rest |
| 373 | // (unspecified, multicast, loopback, link-local). |
| 374 | func publiclyDialable(addr string) bool { |
| 375 | h, _, err := net.SplitHostPort(addr) |
| 376 | if err != nil { |
| 377 | return false |
| 378 | } |
| 379 | ip := net.ParseIP(h) |
| 380 | if ip == nil { |
| 381 | return false |
| 382 | } |
| 383 | return ip.IsGlobalUnicast() && !ip.IsPrivate() |
| 384 | } |
| 385 | |
| 386 | // inScope answers "is this region/pool pair this pass's business" — the one |
| 387 | // gate the record scan and the removal phase both ask, so a record this box |
| 388 | // mints is a record this box can retire. |
| 389 | // |
| 390 | // Pool is compared FIRST and never widened. It is a TENANCY boundary, not a |
| 391 | // geography: a pool names the boxes a tenant's routes are allowed to land on |
| 392 | // (customer fleets, dedicated boxes), and "everywhere" in the spec's table |
| 393 | // means every box that already serves this tenant, not every box in the world. |
| 394 | // |
| 395 | // Region equality is the unchanged home-box test. It is widened for the |
| 396 | // everywhere classes, and only between PLATFORM regions: a fleet placement |
| 397 | // ("fleet/<name>", registry.FleetPlacement) is the other tenancy boundary and |
| 398 | // is refused from both sides — the platform's boxes do not pick up a |
| 399 | // customer fleet's routes, and a customer's box does not pick up the |
| 400 | // platform's. |
| 401 | func (cfg ReconcileConfig) inScope(region, pool string, everywhere bool) bool { |
| 402 | if !poolServed(cfg.Pools, pool) { |
| 403 | return false |
| 404 | } |
| 405 | if region == cfg.Region { |
| 406 | return true |
| 407 | } |
| 408 | if !everywhere { |
| 409 | return false |
| 410 | } |
| 411 | _, recordOnFleet := registry.FleetPlacement(region) |
| 412 | _, boxOnFleet := registry.FleetPlacement(cfg.Region) |
| 413 | return !recordOnFleet && !boxOnFleet |
| 414 | } |
| 415 | |
| 416 | // RouteFor is the one answer to "what route does this record describe". The |
| 417 | // mint path builds from it and the adoption match test (design §4.2) |
| 418 | // compares against it — one function, so a shape the reconciler would mint |
| 419 | // and a shape it will adopt can never drift apart. |
| 420 | // |
| 421 | // Socket and DBPath are DERIVED, never declared: DefaultSocket and |
| 422 | // DefaultDBPath exist so a hand-added route and a provisioned one land on |
| 423 | // identical path shapes, and their doc comment records what two names for |
| 424 | // one database cost. Deriving DBPath EXPLICITLY here — rather than leaving it |
| 425 | // empty on a route that has one — is also what keeps ops.FleetDBs' |
| 426 | // hibernating-exclusion correct: internal/ops/litestream.go builds that |
| 427 | // exclusion set from `r.Hibernate && r.DBPath != ""`, so a hibernating route |
| 428 | // whose row carried no path would be swept up by the dataDir glob and |
| 429 | // replicated box-wide as if it never slept. |
| 430 | // |
| 431 | // nested mints the derived paths at the nested layout's helpers instead |
| 432 | // (InstanceDir, NestedSocket): the shape the layout pass would otherwise |
| 433 | // move a flat row to. Only the mint branch passes true, from |
| 434 | // ReconcileConfig.Layout; the drift and adoption comparisons derive flat |
| 435 | // and judge a nested row by NestedPaths. |
| 436 | func RouteFor(rec Record, dataDir, seed string, nested bool, policy *UpdatePolicy) registry.Route { |
| 437 | rt := registry.Route{ |
| 438 | Host: rec.Host, Slug: rec.Slug, Kind: rec.KindOrDefault(), Unit: rec.Unit, |
| 439 | App: rec.App, Account: rec.Account, Pool: rec.Pool, Region: rec.Region, |
| 440 | Channel: rec.Channel, Environment: rec.Environment, |
| 441 | Hibernate: rec.HibernateOrDefault(), StatusNamed: rec.StatusNamedOrDefault(), |
| 442 | Steering: rec.Steering, |
| 443 | Replicas: replicaSet(rec.Replicas), |
| 444 | Replica: rec.Role == "replica", |
| 445 | Gate: rec.Gate, |
| 446 | Provisioned: true, Seed: seed, Status: "ready", |
| 447 | // The record's own settings over the app's policy (settings spec |
| 448 | // §3.2); "" for default settings, so an untouched fleet mints the |
| 449 | // row it always did. |
| 450 | UpdateSettings: EffectiveUpdates(rec, policy).Column(), |
| 451 | SeedSource: SeedColumn(rec), |
| 452 | } |
| 453 | if rec.DerivesPaths() { |
| 454 | rt.Socket = DefaultSocket(rec.Host) |
| 455 | rt.DBPath = DefaultDBPath(dataDir, rec.Host) |
| 456 | // Only an activator-managed row has a nested shape: an unflagged |
| 457 | // socket route is another supervisor's process, whose paths stay |
| 458 | // flat in every release (design §8.2 step 1). |
| 459 | if nested && rt.Hibernate { |
| 460 | rt.Socket = NestedSocket(rec.Host) |
| 461 | rt.DBPath = NestedDBPath(dataDir, rec.Host) |
| 462 | } |
| 463 | } |
| 464 | rt.Addr, rt.Addrs, rt.Health = upstreamCols(rec) |
| 465 | return rt |
| 466 | } |
| 467 | |
| 468 | // mintLayoutReader answers "mint nested?" for one pass, asking cfg.Layout |
| 469 | // at most once and only when a mint needs the answer. A nil Layout is |
| 470 | // flat. The answer is cached error and all: a store that failed once this |
| 471 | // pass is not asked again per record. |
| 472 | func mintLayoutReader(ctx context.Context, s release.Store, cfg ReconcileConfig) func() (bool, error) { |
| 473 | var ( |
| 474 | asked bool |
| 475 | nested bool |
| 476 | err error |
| 477 | ) |
| 478 | return func() (bool, error) { |
| 479 | if cfg.Layout == nil { |
| 480 | return false, nil |
| 481 | } |
| 482 | if !asked { |
| 483 | asked = true |
| 484 | var mode string |
| 485 | mode, err = cfg.Layout(ctx, s) |
| 486 | nested = err == nil && mode == release.LayoutModeNested |
| 487 | } |
| 488 | return nested, err |
| 489 | } |
| 490 | } |
| 491 | |
| 492 | // upstreamCols renders a record's THREE upstream columns exactly as a minted |
| 493 | // route carries them: the legacy single addr, the comma-joined addr set, and |
| 494 | // the probe-config JSON. |
| 495 | // |
| 496 | // It exists as its own function because TWO paths now write these columns — |
| 497 | // the mint (routeFor, above) and the owned-row converge (convergeOwned, which |
| 498 | // calls registry.SetUpstreams) — and a second, independent derivation is how |
| 499 | // a route that was minted correctly comes to be "converged" onto a different |
| 500 | // value every pass, forever. One derivation, two callers. |
| 501 | // |
| 502 | // All three move together because they are one fact seen three ways: addr and |
| 503 | // addrs are two spellings of "where this route proxies" (registry's AddrList |
| 504 | // folds addr in whenever addrs is empty), and health decides which of those |
| 505 | // upstreams the edge will use. A record that migrates from the legacy addr to |
| 506 | // an addrs set — which is exactly what the console's PATCH door does to a |
| 507 | // pre-existing single-upstream tenant — must clear the row's addr in the same |
| 508 | // write, or the row keeps a decommissioned upstream that resurfaces through |
| 509 | // the fold the moment the set is cleared again. |
| 510 | // |
| 511 | // Addrs is read from the record's EXPLICIT field only, never |
| 512 | // AddrsOrDefault's Addr-folding — a legacy Addr-only record must mint (and |
| 513 | // converge to) a byte-identical route to before these fields existed |
| 514 | // (registry.Route.Addrs documents the fold-back that keeps single-upstream |
| 515 | // callers working off Addr alone). Health, by contrast, goes through |
| 516 | // HealthOrDefault: a multi-upstream route with no stated probe MUST be |
| 517 | // probed, or the edge has no way to tell which of its upstreams is dark. |
| 518 | func upstreamCols(rec Record) (addr, addrs, health string) { |
| 519 | addr = rec.Addr |
| 520 | if len(rec.Addrs) > 0 { |
| 521 | addrs = strings.Join(rec.Addrs, ",") |
| 522 | } |
| 523 | // WithDefaults, not the raw pointer: a record with explicit-but-partial |
| 524 | // probe config (e.g. only Path set) must still mint a route whose |
| 525 | // Health JSON carries the platform's fully-materialized defaults for |
| 526 | // every field it left at zero. An un-defaulted interval_ms of 0 |
| 527 | // reaching the edge would mean a tight-loop probe against a customer |
| 528 | // upstream. |
| 529 | if h := rec.HealthOrDefault(); h != nil { |
| 530 | withDefaults := h.WithDefaults() |
| 531 | // Health is a struct of strings and ints — json.Marshal cannot fail |
| 532 | // on it (no channels, funcs, cyclic pointers, unsupported types). |
| 533 | // The err == nil guard exists to fail CLOSED on the impossible case |
| 534 | // rather than panic: leaving health == "" would mean the route |
| 535 | // mints unprobed, not that a bad value reaches the edge. |
| 536 | if b, err := json.Marshal(withDefaults); err == nil { |
| 537 | health = string(b) |
| 538 | } |
| 539 | } |
| 540 | return addr, addrs, health |
| 541 | } |
| 542 | |
| 543 | // shapeMatches reports whether an existing route is the same ROUTE the |
| 544 | // record describes — the gate on adoption (design §4.2), and so the gate on |
| 545 | // this pass acquiring removal authority over a row it did not mint. |
| 546 | // |
| 547 | // Channel, Environment, Hibernate and Steering are deliberately absent: those |
| 548 | // are the columns the record is taking ownership OF, and a hand-added route |
| 549 | // on the wrong channel is the whole reason to adopt one. Requiring them to |
| 550 | // agree would make adoption impossible in exactly the case it exists for — |
| 551 | // Steering named explicitly here because a record carrying |
| 552 | // steering:"latency" over an unprovisioned row is a real deployment shape |
| 553 | // (an operator hand-added the route before the console's steering PATCH |
| 554 | // door existed to touch it), and refusing that adoption forever would buy |
| 555 | // nothing: convergeOwned stamps Steering onto the row unconditionally the |
| 556 | // moment adoption succeeds (ruled 2026-08-23, plan amendment b485ec5, |
| 557 | // superseding this task's own first pass, which compared it here). |
| 558 | // |
| 559 | // Backing, Tunnel and Caps are deliberately required to be empty (unset) on |
| 560 | // the existing row — registry.Route's own doc comments name these three as |
| 561 | // the box-scoped operator columns, and a record has a field for none of |
| 562 | // them, so a record can never fully describe a row that carries one: |
| 563 | // - Backing "unit" names a carlos-app@<host> systemd tenant; adopting |
| 564 | // across it would claim a process shape the record says nothing about. |
| 565 | // - Tunnel is routing-decisive and OUTRANKS the Socket/Addr a record can |
| 566 | // state: internal/edge picks the tunnel proxy before it ever looks at |
| 567 | // Socket, so a tunnel-backed route can agree with a record on every |
| 568 | // other compared column while every byte it actually serves goes down |
| 569 | // a channel the record cannot see or reclaim. |
| 570 | // - Caps is the operator-granted, box-local capability grant column |
| 571 | // (port:25, host-role, ...). registry.Delete's purge of a route's |
| 572 | // grants is already treated as a human-gated, last-resort action |
| 573 | // elsewhere in this file (the shape-drift log below); adoption must not |
| 574 | // hand that purge to an unattended bucket-record deletion. |
| 575 | // |
| 576 | // A row this pass cannot state the shape of is a row this pass must not |
| 577 | // claim to describe, no matter how well everything else lines up. |
| 578 | // |
| 579 | // These three refusals guard the adoption INSTANT, not the row forever after: |
| 580 | // `carlos route -backing/-unit/-tunnel/-grant` still work on a Provisioned |
| 581 | // row, so an already-owned route can acquire one of them later. See the |
| 582 | // removal phase's comment in Reconcile for what that costs. |
| 583 | // |
| 584 | // Platform and Canary are likewise required false: those flags mark a row |
| 585 | // domains.Reconcile minted and re-mints/removes on its own authority |
| 586 | // (mintedRoute leaves Slug/Pool/Region/DBPath at their zero values, so a |
| 587 | // platform apex on an unconstrained-region box can otherwise agree with a |
| 588 | // record on every other compared column). Adoption must not hand this |
| 589 | // pass's removal authority to a row another pass already owns and |
| 590 | // periodically rebuilds — nor let it write Channel/Environment onto a row |
| 591 | // the CLI itself refuses (platformCanaryOwnedRow, cli.go) for the same |
| 592 | // reason. |
| 593 | // |
| 594 | // Everything else that identifies the route is compared, including the |
| 595 | // derived Socket and DBPath — two rows agreeing on kind and app but not on |
| 596 | // where the database lives are not the same route, and adopting across that |
| 597 | // difference would point a running process at a second copy of its own data. |
| 598 | func shapeMatches(existing, want registry.Route) bool { |
| 599 | return shapeMismatch(existing, want) == "" |
| 600 | } |
| 601 | |
| 602 | // shapeMismatch names the first column on which existing disagrees with |
| 603 | // want — including the box-scoped and pass-ownership guards shapeMatches |
| 604 | // documents — or "" when every compared column agrees. It backs both |
| 605 | // shapeMatches and the adoption refusal log: printing only a couple of |
| 606 | // columns there (the original message did app/kind/addr) leaves every |
| 607 | // refusal over any OTHER column looking identical and undiagnosable to the |
| 608 | // operator the refusal message exists for. |
| 609 | // |
| 610 | // `replication` is box-scoped and inexpressible in a record exactly like |
| 611 | // Backing/Tunnel/Caps, and is DELIBERATELY not listed here — see the |
| 612 | // asymmetry note above the removal loop below for why refusing adoption over |
| 613 | // it would cost the fleet the backfill and buy nothing. |
| 614 | func shapeMismatch(existing, want registry.Route) string { |
| 615 | switch { |
| 616 | case existing.Backing != "": |
| 617 | return fmt.Sprintf("backing %q (box-scoped, not expressible in a record)", existing.Backing) |
| 618 | case existing.Tunnel != "": |
| 619 | return fmt.Sprintf("tunnel %q (box-scoped, not expressible in a record)", existing.Tunnel) |
| 620 | case existing.Caps != "": |
| 621 | return fmt.Sprintf("caps %q (box-scoped, not expressible in a record)", existing.Caps) |
| 622 | case existing.Platform: |
| 623 | return "platform (owned by the domains reconcile pass)" |
| 624 | case existing.Canary: |
| 625 | return "canary (owned by the domains reconcile pass)" |
| 626 | // Replicas is deliberately absent, replication's twin: the column is |
| 627 | // CONVERGED in the owned arm (SetReplicas — the record is authoritative |
| 628 | // in both directions), so a divergence here is repaired, never a reason |
| 629 | // to refuse adoption. Listing it would opt every route with a settled |
| 630 | // replica set out of the backfill permanently and invisibly, the exact |
| 631 | // trade the replication paragraph below refuses — and a replica set |
| 632 | // grants no authority over the route's process; it names where the |
| 633 | // account's mirrors live. |
| 634 | case existing.App != want.App: |
| 635 | return fmt.Sprintf("app %q/%q", existing.App, want.App) |
| 636 | case existing.Account != want.Account: |
| 637 | return fmt.Sprintf("account %q/%q", existing.Account, want.Account) |
| 638 | case existing.Kind != want.Kind: |
| 639 | return fmt.Sprintf("kind %q/%q", existing.Kind, want.Kind) |
| 640 | case existing.Addr != want.Addr: |
| 641 | return fmt.Sprintf("addr %q/%q", existing.Addr, want.Addr) |
| 642 | case existing.Addrs != want.Addrs: |
| 643 | return fmt.Sprintf("addrs %q/%q", existing.Addrs, want.Addrs) |
| 644 | case existing.Health != want.Health: |
| 645 | return fmt.Sprintf("health %q/%q", existing.Health, want.Health) |
| 646 | case existing.Unit != want.Unit: |
| 647 | return fmt.Sprintf("unit %q/%q", existing.Unit, want.Unit) |
| 648 | case existing.Slug != want.Slug: |
| 649 | return fmt.Sprintf("slug %q/%q", existing.Slug, want.Slug) |
| 650 | case existing.Pool != want.Pool: |
| 651 | return fmt.Sprintf("pool %q/%q", existing.Pool, want.Pool) |
| 652 | case existing.Region != want.Region: |
| 653 | return fmt.Sprintf("region %q/%q", existing.Region, want.Region) |
| 654 | case existing.Socket != want.Socket: |
| 655 | return fmt.Sprintf("socket %q/%q", existing.Socket, want.Socket) |
| 656 | case existing.DBPath != want.DBPath: |
| 657 | return fmt.Sprintf("db %q/%q", existing.DBPath, want.DBPath) |
| 658 | default: |
| 659 | return "" |
| 660 | } |
| 661 | } |
| 662 | |
| 663 | // convergeOwned applies the record's Channel, Environment, StatusNamed, |
| 664 | // UPSTREAM SET and Steering onto a route it owns — the columns the record is |
| 665 | // authoritative for once it owns the row, whether ownership was just granted |
| 666 | // by adoption or was already held by an earlier pass. Both the owned-route |
| 667 | // converge branch and the adoption branch call this, so there is exactly one |
| 668 | // place these columns get written from a record, never two copies drifting |
| 669 | // apart. |
| 670 | // |
| 671 | // Addrs/Health converge UNCONDITIONALLY on divergence — no "only when the |
| 672 | // record states one" guard of the kind Unit carries — and that difference is |
| 673 | // deliberate. Unit has a supported out-of-band writer (`carlos route -unit`, |
| 674 | // issue #110) whose stamp a record that never mentioned the field must not |
| 675 | // treat as drift. The upstream set has no such writer: the record IS the |
| 676 | // desired state, the console's PATCH door is how it changes, and clearing the |
| 677 | // set (PATCH with an empty addrs) has to reach the row or that door would move |
| 678 | // the bucket and nothing else. Both values come from upstreamCols, the same |
| 679 | // derivation the mint uses, so a route minted from a record cannot be |
| 680 | // "converged" onto anything but what it already carries. |
| 681 | // |
| 682 | // Steering converges the same way, for the same reason: it has no supported |
| 683 | // out-of-band writer either (Task 3's console door and CLI both write the |
| 684 | // RECORD, never the row directly), so "" is as much an answer as "latency" — |
| 685 | // a record that clears its steering opt-in must reach the row, or the |
| 686 | // console's PATCH door would move the bucket and nothing else, the exact |
| 687 | // silent-stomp shape the upstream-set paragraph above already documents. |
| 688 | // |
| 689 | // StatusNamed converges only when the record STATES it (design §3, the same |
| 690 | // tri-state Hibernate carries): absent means "this record does not speak to |
| 691 | // the status page", and stamping a derived false over an operator's `carlos |
| 692 | // add -status-named` would be the silent stomp this design exists to end. |
| 693 | // Unlike Hibernate it is converged at the adoption instant too — it decides |
| 694 | // what a public page prints, not whether a live process is allowed to sleep. |
| 695 | // |
| 696 | // Every write logs on success as well as on failure. These are the writes |
| 697 | // that REVERT an operator's own `carlos route -environment` or `-hibernate`; |
| 698 | // a row that moved with nothing saying so is the exact failure class of the |
| 699 | // production incident behind #130, seen from the outside. |
| 700 | // |
| 701 | // dryRun is a parameter here, not left to callers, deliberately: this |
| 702 | // function is reached from two branches, and a dry-run guard living in only |
| 703 | // one of the two callers is exactly the single-unguarded-write failure mode |
| 704 | // the pass exists to prevent (design §4.3). Guarding here means both call |
| 705 | // sites are safe by construction, not by each one remembering to check. |
| 706 | func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record, policy *UpdatePolicy, policyKnown bool, dryRun bool) { |
| 707 | if rec.Channel != "" && existing.Channel != rec.Channel { |
| 708 | if dryRun { |
| 709 | log.Printf("provision: dry-run: would converge %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel) |
| 710 | } else if err := reg.SetChannel(rec.Host, rec.Channel); err != nil { |
| 711 | log.Printf("provision: converge channel on %s: %v", rec.Host, err) |
| 712 | } else { |
| 713 | log.Printf("provision: converged %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel) |
| 714 | } |
| 715 | } |
| 716 | if existing.Environment != rec.Environment { |
| 717 | if dryRun { |
| 718 | log.Printf("provision: dry-run: would converge %s environment %q -> %q", rec.Host, existing.Environment, rec.Environment) |
| 719 | } else if err := reg.SetEnvironment(rec.Host, rec.Environment); err != nil { |
| 720 | log.Printf("provision: converge environment on %s: %v", rec.Host, err) |
| 721 | } else { |
| 722 | log.Printf("provision: converged %s environment %q -> %q", rec.Host, existing.Environment, rec.Environment) |
| 723 | } |
| 724 | } |
| 725 | if rec.StatusNamed != nil && existing.StatusNamed != *rec.StatusNamed { |
| 726 | if dryRun { |
| 727 | log.Printf("provision: dry-run: would converge %s status-named %v -> %v", rec.Host, existing.StatusNamed, *rec.StatusNamed) |
| 728 | } else if err := reg.SetStatusNamed(rec.Host, *rec.StatusNamed); err != nil { |
| 729 | log.Printf("provision: converge status-named on %s: %v", rec.Host, err) |
| 730 | } else { |
| 731 | log.Printf("provision: converged %s status-named %v -> %v", rec.Host, existing.StatusNamed, *rec.StatusNamed) |
| 732 | } |
| 733 | } |
| 734 | // One UPDATE for all three columns, because they are one fact: where the |
| 735 | // route proxies (addr, addrs — two spellings of the same thing, which |
| 736 | // registry.AddrList folds together) and the probe that decides which of |
| 737 | // those upstreams the edge will send traffic to. Writing them separately |
| 738 | // would leave a window in which a route's new upstreams are live under the |
| 739 | // OLD probe's path/expected-status — a probe that can mark every new |
| 740 | // upstream dark — and leaving addr out entirely would leave the row saying |
| 741 | // two contradictory things forever (see upstreamCols). |
| 742 | if addr, addrs, health := upstreamCols(rec); existing.Addr != addr || |
| 743 | existing.Addrs != addrs || existing.Health != health { |
| 744 | if dryRun { |
| 745 | log.Printf("provision: dry-run: would converge %s upstreams addr %q -> %q, addrs %q -> %q (probe %q -> %q)", |
| 746 | rec.Host, existing.Addr, addr, existing.Addrs, addrs, existing.Health, health) |
| 747 | } else if err := reg.SetUpstreams(rec.Host, addr, addrs, health); err != nil { |
| 748 | log.Printf("provision: converge upstreams on %s: %v", rec.Host, err) |
| 749 | } else { |
| 750 | log.Printf("provision: converged %s upstreams addr %q -> %q, addrs %q -> %q (probe %q -> %q)", |
| 751 | rec.Host, existing.Addr, addr, existing.Addrs, addrs, existing.Health, health) |
| 752 | } |
| 753 | } |
| 754 | if existing.Steering != rec.Steering { |
| 755 | if dryRun { |
| 756 | log.Printf("provision: dry-run: would converge %s steering %q -> %q", rec.Host, existing.Steering, rec.Steering) |
| 757 | } else if err := reg.SetSteering(rec.Host, rec.Steering); err != nil { |
| 758 | log.Printf("provision: converge steering on %s: %v", rec.Host, err) |
| 759 | } else { |
| 760 | log.Printf("provision: converged %s steering %q -> %q", rec.Host, existing.Steering, rec.Steering) |
| 761 | } |
| 762 | } |
| 763 | // Replicas CONVERGES, where steering merely repoints: the bucket's value |
| 764 | // is authoritative in BOTH directions, including down to none — a set |
| 765 | // that shrank must actually shrink on the row, or the edges file goes on |
| 766 | // publishing a replica placement the account no longer has. This is the |
| 767 | // one deliberate widening the owned arm permits, and the reason is in |
| 768 | // Record.Replicas' doc comment: replica placement is the bucket's |
| 769 | // decision to own, not an operator column to preserve. |
| 770 | if want := replicaSet(rec.Replicas); existing.Replicas != want { |
| 771 | if dryRun { |
| 772 | log.Printf("provision: dry-run: would converge %s replicas %q -> %q", rec.Host, existing.Replicas, want) |
| 773 | } else if err := reg.SetReplicas(rec.Host, want); err != nil { |
| 774 | log.Printf("provision: converge replicas on %s: %v", rec.Host, err) |
| 775 | } else { |
| 776 | log.Printf("provision: converged %s replicas %q -> %q", rec.Host, existing.Replicas, want) |
| 777 | } |
| 778 | } |
| 779 | // The replica MARK converges the same both-ways way, from the same |
| 780 | // record: it decides how the edges file publishes the row (replica |
| 781 | // mark vs plain home), which is exactly the fact steering's fold sorts |
| 782 | // hosts by. Upsert would preserve it wrong (a re-mint carrying no role |
| 783 | // must DEMOTE a row that carried one), so the narrow setter owns it. |
| 784 | if want := rec.Role == "replica"; existing.Replica != want { |
| 785 | if dryRun { |
| 786 | log.Printf("provision: dry-run: would converge %s replica mark %v -> %v", rec.Host, existing.Replica, want) |
| 787 | } else if err := reg.SetReplica(rec.Host, want); err != nil { |
| 788 | log.Printf("provision: converge replica mark on %s: %v", rec.Host, err) |
| 789 | } else { |
| 790 | log.Printf("provision: converged %s replica mark %v -> %v", rec.Host, existing.Replica, want) |
| 791 | } |
| 792 | } |
| 793 | // Update settings (settings spec §4): the record over the app policy, |
| 794 | // both directions, including back to "". Left alone when the policy |
| 795 | // could not be read this pass. |
| 796 | if policyKnown { |
| 797 | if want := EffectiveUpdates(rec, policy).Column(); existing.UpdateSettings != want { |
| 798 | if dryRun { |
| 799 | log.Printf("provision: dry-run: would converge %s update settings %q -> %q", rec.Host, existing.UpdateSettings, want) |
| 800 | } else if err := reg.SetUpdateSettings(rec.Host, want); err != nil { |
| 801 | log.Printf("provision: converge update settings on %s: %v", rec.Host, err) |
| 802 | } else { |
| 803 | log.Printf("provision: converged %s update settings %q -> %q", rec.Host, existing.UpdateSettings, want) |
| 804 | } |
| 805 | } |
| 806 | } |
| 807 | } |
| 808 | |
| 809 | // Reconcile materializes bucket instance records into registry routes for |
| 810 | // THIS agent's region and pools: mint on appearance, converge seed, remove |
| 811 | // on record deletion — removal only for routes this pass (or an earlier one |
| 812 | // running the same scope) minted, marked by Provisioned, the same authority |
| 813 | // discipline as domains.Reconcile's canary removal. Per-item failures log |
| 814 | // and skip; a failed account-manifest or record read additionally disables |
| 815 | // the removal phase for the whole pass — a transient read failure must |
| 816 | // never mass-retire a fleet. Minting is additive and always safe, so it |
| 817 | // proceeds regardless. |
| 818 | func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig) error { |
| 819 | if cfg.Now == nil { |
| 820 | cfg.Now = time.Now |
| 821 | } |
| 822 | |
| 823 | // scan is the record-collection phase over st: the in-scope records by |
| 824 | // host, the hosts whose record is there but cannot be used (protect), |
| 825 | // and whether anything failed to read (readFailed, which disables |
| 826 | // removal for the whole pass). The pass scans s; a pass over a record |
| 827 | // view snapshot that has a removal to decide scans the bucket too |
| 828 | // (freshStore), so both judge every record by exactly these rules (S3 |
| 829 | // change-index design §4.4). |
| 830 | type recordScan struct { |
| 831 | recs map[string]Record |
| 832 | protect map[string]bool |
| 833 | readFailed bool |
| 834 | } |
| 835 | scan := func(st release.Store) (recordScan, error) { |
| 836 | known, readFailed, err := knownAccounts(ctx, st) |
| 837 | if err != nil { |
| 838 | return recordScan{}, err |
| 839 | } |
| 840 | |
| 841 | // --- collect in-scope records -------------------------------------------- |
| 842 | keys, partial, err := listAppSubtree(ctx, st, known, "instances/") |
| 843 | if err != nil { |
| 844 | return recordScan{}, fmt.Errorf("provision: list apps: %w", err) |
| 845 | } |
| 846 | if partial { |
| 847 | readFailed = true // unseen records must not read as retired |
| 848 | } |
| 849 | recs := map[string]Record{} // host -> record, in-scope only |
| 850 | // protect holds hosts whose record KEY parsed (ParseRecordKey succeeded) |
| 851 | // but whose body could not be trusted — unparseable JSON, or a body that |
| 852 | // disagrees with its own path. Such a host must NEVER be treated as "the |
| 853 | // bucket retired it": a non-additive Record encoding change reaching an |
| 854 | // older binary would make EVERY existing record fail exactly this way, |
| 855 | // and without this guard one tick would delete every provisioned route |
| 856 | // in scope. This is narrower than readFailed (which vetoes removal for |
| 857 | // the WHOLE pass): only the specific host with the unusable body is held |
| 858 | // back — every other record still converges/removes normally the same |
| 859 | // pass. |
| 860 | protect := map[string]bool{} |
| 861 | var recordKeys []string |
| 862 | for _, key := range keys { |
| 863 | if _, _, _, ok := ParseRecordKey(key, known); ok { |
| 864 | recordKeys = append(recordKeys, key) |
| 865 | } |
| 866 | } |
| 867 | for _, part := range chunks(recordKeys) { |
| 868 | bodies := getEach(ctx, st, part) |
| 869 | for _, key := range part { |
| 870 | acct, app, host, _ := ParseRecordKey(key, known) |
| 871 | raw, err := bodies[key].body, bodies[key].err |
| 872 | if err != nil { |
| 873 | log.Printf("provision: skipping record %s: read failed: %v", key, err) |
| 874 | // The record's absence from recs below must not read as "the |
| 875 | // bucket retired it" — a transient Get failure disables removal |
| 876 | // for the whole pass, same as an unreadable account manifest. |
| 877 | readFailed = true |
| 878 | continue |
| 879 | } |
| 880 | rec, err := DecodeRecord(raw) |
| 881 | if err != nil { |
| 882 | log.Printf("provision: skipping record %s: malformed: %v", key, err) |
| 883 | protect[host] = true |
| 884 | continue |
| 885 | } |
| 886 | if rec.Host != host || rec.Account != acct || rec.App != app { |
| 887 | log.Printf("provision: skipping record %s: disagrees with its own path (host=%q account=%q app=%q)", |
| 888 | key, rec.Host, rec.Account, rec.App) |
| 889 | protect[host] = true |
| 890 | continue |
| 891 | } |
| 892 | // The record IS the authorization, and this pass turns its host into |
| 893 | // a socket path and a database path (DefaultSocket/DefaultDBPath) |
| 894 | // and a registry primary key. The console's create door enforces |
| 895 | // this same shape — but a door is not a guarantee about what is |
| 896 | // already in the bucket: anything with bucket write access, and any |
| 897 | // record written by a different (older, or future) writer, arrives |
| 898 | // here unexamined. Both doors hold the invariant, neither trusts |
| 899 | // the other. Protected like a corrupted body, and for the same |
| 900 | // reason: a record that cannot be used is not a record that was |
| 901 | // retired. |
| 902 | if !ValidHostShape(rec.Host) { |
| 903 | log.Printf("provision: skipping record %s: host %q is not a hostname", key, rec.Host) |
| 904 | protect[host] = true |
| 905 | continue |
| 906 | } |
| 907 | // ValidateShape is the console create door's policy check — but a |
| 908 | // door is not a guarantee about what is already in the bucket (the |
| 909 | // same reasoning as the host-shape check just above). This pass |
| 910 | // materializes whatever it finds, so a record whose Kind/Addr/Unit/ |
| 911 | // Environment combination is malformed must be caught here too, or |
| 912 | // a bad value becomes a live route. Protected like a corrupted |
| 913 | // body: the record is there, it just cannot be used. |
| 914 | if err := ValidateShape(rec); err != nil { |
| 915 | log.Printf("provision: skipping record %s: invalid shape: %v", key, err) |
| 916 | protect[host] = true |
| 917 | continue |
| 918 | } |
| 919 | // ValidateShape deliberately does not inspect Channel (it validates |
| 920 | // shape fields, and Channel is not one), but a channel-less record |
| 921 | // must not become — or converge — a route pinned to no channel: |
| 922 | // that is issue #130's failure mode arriving through a different |
| 923 | // door than the one Task 3 closed. Protected like a corrupted body: |
| 924 | // the record is there, it just cannot be used. |
| 925 | if rec.Channel == "" { |
| 926 | log.Printf("provision: skipping record %s: empty channel", key) |
| 927 | protect[host] = true |
| 928 | continue |
| 929 | } |
| 930 | // Scope. For a home-boxed record this is the region/pool test it |
| 931 | // always was; for a static or public-addr one the region half widens |
| 932 | // to every platform box in the deployment (everywhereClass, above). |
| 933 | // A replica record narrows it: it belongs ONLY to the box whose |
| 934 | // placement is the record's own Region — the placement the create |
| 935 | // door validated as armed — so a replica record minted on the wrong |
| 936 | // region would mint a route with no mirror behind it and no lease |
| 937 | // speaking to it, a process holding live order data nobody |
| 938 | // authorized into that region (the residence rule, #162). Every |
| 939 | // other box skips it exactly like any other out-of-scope record; |
| 940 | // the replica's own box sees Region == cfg.Region through the |
| 941 | // ordinary test above. |
| 942 | if !cfg.inScope(rec.Region, rec.Pool, everywhereClass(rec)) { |
| 943 | continue // another agent's jurisdiction, not this pass's business |
| 944 | } |
| 945 | // Two accounts writing the same host is a collision this pass |
| 946 | // cannot arbitrate silently: whichever record this listing reaches |
| 947 | // FIRST wins the host, and the loser is skipped loudly rather than |
| 948 | // overwriting the winner in the map with no trace of the conflict. |
| 949 | if prev, dup := recs[rec.Host]; dup { |
| 950 | log.Printf("provision: duplicate host %s: keeping %s, skipping %s", rec.Host, |
| 951 | RecordKey(prev.Account, prev.App, prev.Host), RecordKey(rec.Account, rec.App, rec.Host)) |
| 952 | continue |
| 953 | } |
| 954 | recs[rec.Host] = rec |
| 955 | } |
| 956 | } |
| 957 | return recordScan{recs: recs, protect: protect, readFailed: readFailed}, nil |
| 958 | } |
| 959 | sc, err := scan(s) |
| 960 | if err != nil { |
| 961 | return err |
| 962 | } |
| 963 | recs, protect, readFailed := sc.recs, sc.protect, sc.readFailed |
| 964 | |
| 965 | // --- manifest seed cache, per (acct, app), lazily ------------------------- |
| 966 | // known=false (a genuine read failure, not "no manifest") means the seed |
| 967 | // value returned is a mint-only fallback: safe to hand a BRAND NEW route |
| 968 | // (nothing existing to protect; the next readable pass converges it), |
| 969 | // never safe to feed the converge branch below — SetSeed-ing an existing |
| 970 | // route to "" on a transient blip would erase a working seed for a |
| 971 | // reason that has nothing to do with the manifest actually changing. |
| 972 | // Each app with records costs two reads below (its manifest and its trash |
| 973 | // marker), made lazily, one at a time, inside the record loop. Both are |
| 974 | // read here first, eight at a time, for every app the records name; the |
| 975 | // lookups below then judge exactly what they judged before, from these |
| 976 | // results, and read for themselves only for an app not prefetched (a |
| 977 | // route's app with no record, at removal time). |
| 978 | // The manifest is kept only as the seed it renders to, never whole: the |
| 979 | // lookup below has always reduced it to that at once. |
| 980 | type manifestRead struct { |
| 981 | seed string |
| 982 | policy *UpdatePolicy |
| 983 | err error |
| 984 | } |
| 985 | type tombRead struct { |
| 986 | tomb *release.AppTombstone |
| 987 | err error |
| 988 | } |
| 989 | var recApps []string |
| 990 | appOf := map[string]acctApp{} |
| 991 | for _, rec := range recs { |
| 992 | k := acctApp{rec.Account, rec.App} |
| 993 | id := rec.Account + "/" + rec.App |
| 994 | if _, seen := appOf[id]; !seen { |
| 995 | appOf[id] = k |
| 996 | recApps = append(recApps, id) |
| 997 | } |
| 998 | } |
| 999 | slices.Sort(recApps) |
| 1000 | manifests := map[acctApp]manifestRead{} |
| 1001 | tombs := map[acctApp]tombRead{} |
| 1002 | var prefetchMu sync.Mutex |
| 1003 | forEach(recApps, func(id string) { |
| 1004 | k := appOf[id] |
| 1005 | // Trash first, as the record loop asks: an app in the trash, or one |
| 1006 | // whose marker will not read, is never minted for, so its manifest |
| 1007 | // was never read and is not read here either. |
| 1008 | tomb, terr := release.ReadAppTombstone(ctx, s, k.acct, k.app) |
| 1009 | prefetchMu.Lock() |
| 1010 | tombs[k] = tombRead{tomb, terr} |
| 1011 | prefetchMu.Unlock() |
| 1012 | if tomb != nil || terr != nil { |
| 1013 | return |
| 1014 | } |
| 1015 | m, merr := GetManifest(ctx, s, k.acct, k.app) |
| 1016 | r := manifestRead{err: merr} |
| 1017 | if merr == nil { |
| 1018 | r.seed = seedJSON(m) |
| 1019 | r.policy = m.Updates |
| 1020 | } |
| 1021 | prefetchMu.Lock() |
| 1022 | manifests[k] = r |
| 1023 | prefetchMu.Unlock() |
| 1024 | }) |
| 1025 | |
| 1026 | seedCache := map[acctApp]string{} |
| 1027 | seedKnown := map[acctApp]bool{} |
| 1028 | seenManifest := map[acctApp]bool{} |
| 1029 | seedFor := func(acct, app string) (seed string, known bool) { |
| 1030 | k := acctApp{acct, app} |
| 1031 | if seenManifest[k] { |
| 1032 | return seedCache[k], seedKnown[k] |
| 1033 | } |
| 1034 | seenManifest[k] = true |
| 1035 | var rendered string |
| 1036 | var err error |
| 1037 | if r, ok := manifests[k]; ok { |
| 1038 | rendered, err = r.seed, r.err |
| 1039 | } else { |
| 1040 | var m *Manifest |
| 1041 | if m, err = GetManifest(ctx, s, acct, app); err == nil { |
| 1042 | rendered = seedJSON(m) |
| 1043 | } |
| 1044 | } |
| 1045 | if err != nil { |
| 1046 | if errors.Is(err, os.ErrNotExist) { |
| 1047 | seedCache[k], seedKnown[k] = "", true // absent manifest: confirmed no seed |
| 1048 | return seedCache[k], seedKnown[k] |
| 1049 | } |
| 1050 | // A record already written is truth regardless — the manifest |
| 1051 | // only governs the API door — but a genuine read failure must |
| 1052 | // still hold back removal: this pass cannot tell a disabled |
| 1053 | // feature from a transient blip, so it treats it as the latter. |
| 1054 | log.Printf("provision: manifest %s/%s: read failed: %v", acct, app, err) |
| 1055 | readFailed = true |
| 1056 | return "", false |
| 1057 | } |
| 1058 | seedCache[k], seedKnown[k] = rendered, true |
| 1059 | return seedCache[k], seedKnown[k] |
| 1060 | } |
| 1061 | |
| 1062 | // policyFor is the app's update policy for this pass (settings spec |
| 1063 | // §3.2). known is false when the manifest could not be read: the caller |
| 1064 | // then leaves every row's update_settings as it is, because a read |
| 1065 | // failure is not evidence the policy went away. An absent manifest is |
| 1066 | // known: no policy. |
| 1067 | policyFor := func(acct, app string) (*UpdatePolicy, bool) { |
| 1068 | k := acctApp{acct, app} |
| 1069 | if r, ok := manifests[k]; ok { |
| 1070 | if r.err != nil { |
| 1071 | return nil, errors.Is(r.err, os.ErrNotExist) |
| 1072 | } |
| 1073 | return r.policy, true |
| 1074 | } |
| 1075 | m, err := GetManifest(ctx, s, acct, app) |
| 1076 | switch { |
| 1077 | case errors.Is(err, os.ErrNotExist): |
| 1078 | return nil, true |
| 1079 | case err != nil: |
| 1080 | return nil, false |
| 1081 | } |
| 1082 | return m.Updates, true |
| 1083 | } |
| 1084 | // policyUnread says a row was not minted because policyFor could not |
| 1085 | // read the app's policy. A read failure is not "no policy": minting now |
| 1086 | // would mint an instance that inherits manual as auto, and root would |
| 1087 | // adopt the head over its data's build. Absent is known. |
| 1088 | policyUnread := func(host string) { |
| 1089 | log.Printf("provision: %s: not minted this pass: the app's update policy could not be read", host) |
| 1090 | } |
| 1091 | |
| 1092 | // --- trash cache, per (acct, app), lazily --------------------------------- |
| 1093 | // An app in the trash provisions nothing, and its existing routes are |
| 1094 | // RETIRED — both halves are this pass's job (#340). |
| 1095 | // |
| 1096 | // Retirement used to be domains.Reconcile's alone, on the grounds that it |
| 1097 | // held the production-flag guard. That deferral was a live bug: the |
| 1098 | // domains pass runs only on the box holding the housekeeper lease |
| 1099 | // (agent.go's domainsLoop wraps it in runHousekeeperDuties), and its |
| 1100 | // registry write is local, so on a multi-box deployment every OTHER box |
| 1101 | // kept a trashed app's route forever and went on serving it from its own |
| 1102 | // edge. Single-box deployments were correct by accident. This pass runs |
| 1103 | // on every box and already reads the marker, so the trigger belongs here |
| 1104 | // — and the guard comes with it (cfg.ReleasePub). Both passes now remove |
| 1105 | // and neither mints, so they agree rather than fight. |
| 1106 | // |
| 1107 | // One marker, two DIFFERENT answers, and conflating them is the trap: |
| 1108 | // |
| 1109 | // declineMint — true also when the marker is UNREADABLE. Not minting |
| 1110 | // is the reversible direction, and a bucket writer who could plant |
| 1111 | // garbage here to stall minting can already retire the instance |
| 1112 | // outright by deleting its record. |
| 1113 | // removable — true ONLY for a marker positively read and present, |
| 1114 | // whose app is not flagged production. Removal is not reversible, so |
| 1115 | // an unreadable marker (or an unreadable flag) must never reach it. |
| 1116 | // |
| 1117 | // trashOver is that lookup over st, answering from pre where pre holds |
| 1118 | // the app: the pass's own over s and its prefetch, and the removal |
| 1119 | // phase's over the bucket when this pass reads a snapshot (below). |
| 1120 | trashOver := func(st release.Store, pre map[acctApp]tombRead) func(acct, app string) trashState { |
| 1121 | trashCache := map[acctApp]trashState{} |
| 1122 | trashSeen := map[acctApp]bool{} |
| 1123 | return func(acct, app string) trashState { |
| 1124 | k := acctApp{acct, app} |
| 1125 | if trashSeen[k] { |
| 1126 | return trashCache[k] |
| 1127 | } |
| 1128 | trashSeen[k] = true |
| 1129 | var tomb *release.AppTombstone |
| 1130 | var err error |
| 1131 | if r, ok := pre[k]; ok { |
| 1132 | tomb, err = r.tomb, r.err |
| 1133 | } else { |
| 1134 | tomb, err = release.ReadAppTombstone(ctx, st, acct, app) |
| 1135 | } |
| 1136 | switch { |
| 1137 | case err != nil: |
| 1138 | log.Printf("provision: trash marker for %s/%s unreadable: %v — not minting for this app", acct, app, err) |
| 1139 | trashCache[k] = trashState{declineMint: true} |
| 1140 | case tomb != nil: |
| 1141 | log.Printf("provision: %s/%s is in the trash — not minting instance routes for it", acct, app) |
| 1142 | trashCache[k] = trashState{declineMint: true, removable: productionClear(ctx, st, cfg.ReleasePub, acct, app)} |
| 1143 | } |
| 1144 | return trashCache[k] |
| 1145 | } |
| 1146 | } |
| 1147 | trashFor := trashOver(s, tombs) |
| 1148 | trashedFor := func(acct, app string) bool { return trashFor(acct, app).declineMint } |
| 1149 | |
| 1150 | // --- mint / converge / conflict ------------------------------------------- |
| 1151 | routes, err := reg.List() |
| 1152 | if err != nil { |
| 1153 | return fmt.Errorf("provision: list routes: %w", err) |
| 1154 | } |
| 1155 | byHost := map[string]registry.Route{} |
| 1156 | for _, rt := range routes { |
| 1157 | byHost[rt.Host] = rt |
| 1158 | } |
| 1159 | mintLayout := mintLayoutReader(ctx, s, cfg) |
| 1160 | // The import journal is read before the records are acted on, every |
| 1161 | // pass (seeded-instances spec §5.7, seedreconcile.go). A host it handled |
| 1162 | // is skipped by the loop below, and every journaled host by the removal |
| 1163 | // phase: the journal owns the row until its row is gone. An unreadable |
| 1164 | // journal could hide any host, so the removal phase does not run. |
| 1165 | journalHandled, journaled, journalOK := journalPass(ctx, s, reg, cfg, recs, readFailed, protect) |
| 1166 | if !journalOK { |
| 1167 | readFailed = true |
| 1168 | } |
| 1169 | |
| 1170 | for _, rec := range recs { |
| 1171 | if journalHandled[rec.Host] { |
| 1172 | continue |
| 1173 | } |
| 1174 | if trashedFor(rec.Account, rec.App) { |
| 1175 | continue |
| 1176 | } |
| 1177 | if cfg.RunSeen != nil && rec.ActiveRun != nil { |
| 1178 | cfg.RunSeen(rec) |
| 1179 | } |
| 1180 | // A mint-only fallback ("", the record-already-written-is-truth |
| 1181 | // case) is fine here even when the manifest is unreadable this |
| 1182 | // pass — there is no existing seed value a brand new route could |
| 1183 | // clobber, and a later pass converges it once the manifest reads. |
| 1184 | seed, seedIsKnown := seedFor(rec.Account, rec.App) |
| 1185 | existing, ok := byHost[rec.Host] |
| 1186 | // A seed is dispatched before anything else (seeded-instances spec |
| 1187 | // §3.2, codex 13): an aborted one is finished, an unready one is |
| 1188 | // skipped, and a seeded row never carries the manifest's seed |
| 1189 | // command. |
| 1190 | if rec.Seed != nil { |
| 1191 | var row *registry.Route |
| 1192 | if ok { |
| 1193 | row = &existing |
| 1194 | } |
| 1195 | switch seedAction(rec, row, cfg, mintLayout, journalOK) { |
| 1196 | case seedSkip: |
| 1197 | continue |
| 1198 | case seedSwap: |
| 1199 | // The swap mints the import's row, so it waits for the |
| 1200 | // app's update policy as a mint does (settings spec §3.2). |
| 1201 | p, pKnown := policyFor(rec.Account, rec.App) |
| 1202 | if !pKnown { |
| 1203 | policyUnread(rec.Host) |
| 1204 | continue |
| 1205 | } |
| 1206 | swapImport(ctx, s, reg, cfg, rec, row, mintLayout, p) |
| 1207 | continue |
| 1208 | case seedFinishAbort: |
| 1209 | // A --from seed this box minted is still running: its row |
| 1210 | // goes first, and a later pass that finds no row rolls the |
| 1211 | // seed back (seedreconcile.go). |
| 1212 | if ok && rec.Seed.Kind == seeds.KindReplica { |
| 1213 | retireAbortedSeedRow(ctx, reg, cfg, rec, existing) |
| 1214 | continue |
| 1215 | } |
| 1216 | finishAbortedSeed(ctx, s, reg, cfg, rec) |
| 1217 | continue |
| 1218 | } |
| 1219 | seed, seedIsKnown = "", true |
| 1220 | } |
| 1221 | // A move's hold is judged first (moves spec §9.6). A missing row is |
| 1222 | // minted held by mintHeld, which has a ready mint's refusals and |
| 1223 | // the same claim hook (below) around its write, so the hold and the |
| 1224 | // claim both have to allow the mint. An existing row is held in |
| 1225 | // place with a narrow status write, outside the claim hook as the |
| 1226 | // other status writers are, and then converged as usual: the owned |
| 1227 | // branch's narrow setters never touch status or hold_op. |
| 1228 | if cfg.Hold != nil { |
| 1229 | if status, op, held := cfg.Hold(rec); held { |
| 1230 | if !ok { |
| 1231 | p, known := policyFor(rec.Account, rec.App) |
| 1232 | mintHeld(ctx, reg, cfg, rec, seed, status, op, p, known) |
| 1233 | continue |
| 1234 | } |
| 1235 | holdExisting(ctx, s, reg, cfg, rec, existing, status, op) |
| 1236 | } |
| 1237 | } |
| 1238 | |
| 1239 | // Refusals that need no claim are decided BEFORE cfg.claimInstall: the |
| 1240 | // hook activates the host guard and writes inventory, effects that a |
| 1241 | // record this pass will not act on must never cause. Only work that |
| 1242 | // is going to touch the registry reaches it. |
| 1243 | var prior *registry.Route |
| 1244 | nested := false |
| 1245 | if ok { |
| 1246 | prior = &existing |
| 1247 | nested = NestedPaths(cfg.DataDir, existing) |
| 1248 | } else { |
| 1249 | // A box that declared CARLOS_INBOUND=none (design §8) has told |
| 1250 | // this pass what work it will ACCEPT, not made a claim about its |
| 1251 | // network — fleet boxes are already outbound-only dial in this |
| 1252 | // estate. Ordinary instance records carry no placement |
| 1253 | // vocabulary of their own (that is ReconcileSidecars' gate, |
| 1254 | // Task 9, over SidecarRecord.Placement); every routed instance |
| 1255 | // this pass mints needs inbound service, so a no-inbound box |
| 1256 | // must refuse ALL of them outright, loudly, rather than mint |
| 1257 | // something it can never serve — a misfiled record becomes a |
| 1258 | // log line here, never a silent outage. |
| 1259 | if cfg.Inbound == InboundNone { |
| 1260 | log.Printf("provision: refusing to mint route %s (app %q acct %q): this box declares "+ |
| 1261 | "CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic", rec.Host, rec.App, rec.Account) |
| 1262 | continue |
| 1263 | } |
| 1264 | // The box's layout decides the paths a fresh row is minted at. |
| 1265 | // Read once per pass, lazily, so a pass with nothing to mint |
| 1266 | // never reads it; a read that fails defers every mint on this |
| 1267 | // pass (ReconcileConfig.Layout). |
| 1268 | var lerr error |
| 1269 | if nested, lerr = mintLayout(); lerr != nil { |
| 1270 | log.Printf("provision: %s: not minting this pass: the box's layout intent could not be read: %v", rec.Host, lerr) |
| 1271 | continue |
| 1272 | } |
| 1273 | } |
| 1274 | // The route's whole shape now comes from the record (routeFor): |
| 1275 | // Kind, Addr, Unit, Environment, StatusNamed, Hibernate and the |
| 1276 | // path derivation. A record carrying none of them still mints the |
| 1277 | // exact route this branch hardcoded before they existed — that is |
| 1278 | // the additivity rule, pinned by TestRecordWireShapeIsAdditive and |
| 1279 | // TestReconcileMints. |
| 1280 | p, pKnown := policyFor(rec.Account, rec.App) |
| 1281 | if !ok && !pKnown { |
| 1282 | policyUnread(rec.Host) |
| 1283 | continue |
| 1284 | } |
| 1285 | nr := RouteFor(rec, cfg.DataDir, seed, nested, p) |
| 1286 | // A box that cannot provide this instance's config must not |
| 1287 | // admit it: it would start without its secrets, and a keyed |
| 1288 | // sibling would be left with a route that lost the race to one |
| 1289 | // that was never going to work. Refused before the claim hook, the |
| 1290 | // write and the dry-run line, so a dry run reports what a real pass |
| 1291 | // would do. |
| 1292 | if !ok && cfg.ConfigGate != nil { |
| 1293 | if err := cfg.ConfigGate(ctx, nr); err != nil { |
| 1294 | continue |
| 1295 | } |
| 1296 | } |
| 1297 | // An existing row this record neither owns nor may adopt is known to be |
| 1298 | // left alone before any claim work: with claims on, the hook would |
| 1299 | // activate the guard and write inventory only to be refused after. |
| 1300 | // (Flag off, the same decision is made inside the write below.) |
| 1301 | if ok && cfg.ClaimEnabled && !(existing.Provisioned && existing.App == rec.App && existing.Account == rec.Account) { |
| 1302 | if reason := shapeMismatch(existing, nr); reason != "" { |
| 1303 | log.Printf("provision: %s exists but is not the route this record describes (%s) — leaving it alone", |
| 1304 | rec.Host, reason) |
| 1305 | continue |
| 1306 | } |
| 1307 | } |
| 1308 | selected := ClaimRecord{Host: rec.Host, Key: RecordKey(rec.Account, rec.App, rec.Host), Family: "instance", Account: rec.Account, App: rec.App, CreatedAt: rec.CreatedAt, Value: rec, Route: nr, DryRun: cfg.DryRun} |
| 1309 | if err := cfg.claimInstall(ctx, selected, prior, func(owner registry.ClaimOwnership) error { |
| 1310 | switch { |
| 1311 | case !ok: |
| 1312 | nr.ClaimOwnership = owner |
| 1313 | if cfg.DryRun { |
| 1314 | log.Printf("provision: dry-run: would mint %s route %s (app %q acct %q channel %q)", |
| 1315 | nr.Kind, rec.Host, rec.App, rec.Account, rec.Channel) |
| 1316 | return nil |
| 1317 | } |
| 1318 | if err := reg.Upsert(nr); err != nil { |
| 1319 | log.Printf("provision: mint %s: %v", rec.Host, err) |
| 1320 | return nil |
| 1321 | } |
| 1322 | log.Printf("provision: minted %s route %s (app %q acct %q)", nr.Kind, rec.Host, rec.App, rec.Account) |
| 1323 | case existing.Provisioned && existing.App == rec.App && existing.Account == rec.Account: |
| 1324 | // The columns this record owns (design §4.1, issue #130). Narrow |
| 1325 | // UPDATEs, never a re-Upsert: a read-modify-write racing a |
| 1326 | // deploy's SetVersions or the activator's SetNextDue would |
| 1327 | // clobber those, and Upsert cannot express "clear Environment" |
| 1328 | // at all (it preserves a non-empty value by design). |
| 1329 | convergeOwned(reg, existing, rec, p, pKnown, cfg.DryRun) |
| 1330 | // Absent means "this record does not speak to hibernation" — |
| 1331 | // never a derivation stamped over an operator's own setting. |
| 1332 | // |
| 1333 | // Converging hibernation OFF also drops the route's replication |
| 1334 | // rung, inside registry.SetHibernate — a rung is read only on |
| 1335 | // the activator's wake path, so on a route the record has just |
| 1336 | // stopped hibernating it would describe nothing that happens |
| 1337 | // while `carlos routes` went on printing it. A record cannot |
| 1338 | // express the rung (it is box-scoped, like caps and tunnel), so |
| 1339 | // this pass cannot restore one either; the drop is LOGGED with |
| 1340 | // the flip rather than left for an operator to notice, because |
| 1341 | // it is the one operator-typed value a converge tick can |
| 1342 | // silently spend. |
| 1343 | // |
| 1344 | // A nested row keeps its flag whatever the record says: only a |
| 1345 | // row the activator manages is one the layout revert can stop |
| 1346 | // and move back, so clearing the flag would strand its files |
| 1347 | // under the instance tree (design §8.2, downgrade rules). A row |
| 1348 | // with a layout journal is mid-move whatever its paths say, and |
| 1349 | // recovery needs it managed just the same. |
| 1350 | if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate && !*rec.Hibernate && |
| 1351 | (NestedPaths(cfg.DataDir, existing) || LayoutJournalExists(cfg.DataDir, rec.Host)) { |
| 1352 | log.Printf("provision: %s: record asks to clear hibernate but the row's paths are nested; revert the layout first", rec.Host) |
| 1353 | } else if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate { |
| 1354 | dropped := "" |
| 1355 | if !*rec.Hibernate && existing.Replication != "" { |
| 1356 | dropped = fmt.Sprintf(" (replication %q cleared: only a hibernating route's rung is read)", existing.Replication) |
| 1357 | } |
| 1358 | if cfg.DryRun { |
| 1359 | log.Printf("provision: dry-run: would converge %s hibernate %v -> %v%s", rec.Host, existing.Hibernate, *rec.Hibernate, dropped) |
| 1360 | } else if err := reg.SetHibernate(rec.Host, *rec.Hibernate); err != nil { |
| 1361 | log.Printf("provision: converge hibernate on %s: %v", rec.Host, err) |
| 1362 | } else { |
| 1363 | log.Printf("provision: converged %s hibernate %v -> %v%s", rec.Host, existing.Hibernate, *rec.Hibernate, dropped) |
| 1364 | } |
| 1365 | } |
| 1366 | if existing.Gate != rec.Gate { |
| 1367 | if cfg.DryRun { |
| 1368 | log.Printf("provision: dry-run: would converge %s gate %s -> %s", rec.Host, gateWord(existing.Gate), gateWord(rec.Gate)) |
| 1369 | } else if err := reg.SetGate(rec.Host, rec.Gate); err != nil { |
| 1370 | log.Printf("provision: converge gate on %s: %v", rec.Host, err) |
| 1371 | } else { |
| 1372 | log.Printf("provision: converged %s gate %s -> %s", rec.Host, gateWord(existing.Gate), gateWord(rec.Gate)) |
| 1373 | } |
| 1374 | } |
| 1375 | // The seed source goes first, and the seed command follows only |
| 1376 | // once it has landed (seeded-instances preflight M10). A row |
| 1377 | // whose record has a seed gets its seed column before its seed |
| 1378 | // command is cleared, so between the two writes it is placeable |
| 1379 | // (seededRestore, never fresh). In the other order, or with the |
| 1380 | // source write failed and the command cleared anyway, the row |
| 1381 | // would hold neither, and a wake then could boot it empty. |
| 1382 | sourceConverged := true |
| 1383 | if col := SeedColumn(rec); existing.SeedSource != col { |
| 1384 | if cfg.DryRun { |
| 1385 | log.Printf("provision: dry-run: would converge %s seed source", rec.Host) |
| 1386 | } else if err := reg.SetSeedSource(rec.Host, col); err != nil { |
| 1387 | log.Printf("provision: converge seed source on %s: %v", rec.Host, err) |
| 1388 | sourceConverged = false |
| 1389 | } |
| 1390 | } |
| 1391 | // seedIsKnown guards against stamping a transient manifest-read |
| 1392 | // failure's "" fallback over a working seed — see seedFor. |
| 1393 | if sourceConverged && seedIsKnown && existing.Seed != seed { |
| 1394 | if cfg.DryRun { |
| 1395 | log.Printf("provision: dry-run: would converge %s seed", rec.Host) |
| 1396 | } else if err := reg.SetSeed(rec.Host, seed); err != nil { |
| 1397 | log.Printf("provision: converge seed on %s: %v", rec.Host, err) |
| 1398 | } |
| 1399 | } |
| 1400 | // Unit is compared only when the RECORD states one: `carlos |
| 1401 | // route -unit` (issue #110) is the supported way to stamp the |
| 1402 | // unit owning a route's process out of band, and a record that |
| 1403 | // never spoke to Unit must not treat that stamp as drift — or |
| 1404 | // this would fire on a healthy, correctly-stamped route every |
| 1405 | // pass, forever. |
| 1406 | // Addr is deliberately NOT compared here any more: it is a |
| 1407 | // CONVERGED column now (convergeOwned writes it beside Addrs and |
| 1408 | // Health through SetUpstreams), and a drift log that kept naming a |
| 1409 | // column the same pass had just fixed would fire forever on a |
| 1410 | // healthy row — the exact "undiagnosable refusal" this message was |
| 1411 | // widened to prevent. What remains are the columns nothing |
| 1412 | // converges in place: the route CLASS and the two filesystem paths |
| 1413 | // derived from it. |
| 1414 | // |
| 1415 | // A nested row's paths differ from the record's flat derivation |
| 1416 | // by design (the layout pass moved them), so when that is the |
| 1417 | // only difference the line says so, rather than pointing an |
| 1418 | // operator at `carlos remove`. |
| 1419 | want := RouteFor(rec, cfg.DataDir, seed, false, p) |
| 1420 | unitDrift := rec.Unit != "" && want.Unit != existing.Unit |
| 1421 | if want.Kind != existing.Kind || unitDrift || |
| 1422 | want.Socket != existing.Socket || want.DBPath != existing.DBPath { |
| 1423 | if want.Kind == existing.Kind && !unitDrift && NestedPaths(cfg.DataDir, existing) { |
| 1424 | log.Printf("provision: %s: row is on the nested layout (db %q socket %q); the record's shared paths are not drift", |
| 1425 | rec.Host, existing.DBPath, existing.Socket) |
| 1426 | } else { |
| 1427 | log.Printf("provision: %s: record describes a different route SHAPE than the row "+ |
| 1428 | "(kind %q/%q unit %q/%q socket %q/%q db %q/%q) — not converged in place; "+ |
| 1429 | "`carlos remove %s` purges this route's capability grants (registry.Delete's contract, "+ |
| 1430 | "not a bug) before the next pass re-mints it, so treat that as a last resort, not a routine fix", |
| 1431 | rec.Host, want.Kind, existing.Kind, want.Unit, existing.Unit, |
| 1432 | want.Socket, existing.Socket, want.DBPath, existing.DBPath, rec.Host) |
| 1433 | } |
| 1434 | } |
| 1435 | default: |
| 1436 | // A route exists that this pass did not mint. If it is the same |
| 1437 | // route the record describes, adopt it — that is what makes |
| 1438 | // backfilling the fleet a bucket-side write rather than a box |
| 1439 | // session (design §4.2). If it is not, leave it exactly as alone |
| 1440 | // as before: adoption grants this pass's removal phase authority |
| 1441 | // over the row, so the match test is the only thing standing |
| 1442 | // between a backfill and a retired production route. |
| 1443 | // A hand-added row on a nested box (`carlos add` there mints |
| 1444 | // nested too) matches the record at the nested paths; the row's |
| 1445 | // own shape says which derivation to compare against. |
| 1446 | want := RouteFor(rec, cfg.DataDir, seed, NestedPaths(cfg.DataDir, existing), p) |
| 1447 | if reason := shapeMismatch(existing, want); reason != "" { |
| 1448 | log.Printf("provision: %s exists but is not the route this record describes (%s) — leaving it alone", |
| 1449 | rec.Host, reason) |
| 1450 | return nil |
| 1451 | } |
| 1452 | if cfg.DryRun { |
| 1453 | log.Printf("provision: dry-run: would adopt existing route %s (app %q acct %q) — this record would own its channel and environment", |
| 1454 | rec.Host, rec.App, rec.Account) |
| 1455 | // convergeOwned reports channel and environment together, |
| 1456 | // and only the columns that actually differ (design §4.1) — |
| 1457 | // a fixed "converging channel X -> Y" line here would both |
| 1458 | // omit environment (the defect this fixes) and claim a |
| 1459 | // convergence that would not happen when the route is |
| 1460 | // already on the record's channel. Every write inside |
| 1461 | // convergeOwned is dry-run guarded there, so this call |
| 1462 | // writes nothing. |
| 1463 | convergeOwned(reg, existing, rec, p, pKnown, true) |
| 1464 | return nil |
| 1465 | } |
| 1466 | if err := reg.SetProvisioned(rec.Host, true); err != nil { |
| 1467 | log.Printf("provision: adopt %s: %v", rec.Host, err) |
| 1468 | return nil |
| 1469 | } |
| 1470 | log.Printf("provision: adopted existing route %s (app %q acct %q) — this record now owns its channel and environment", |
| 1471 | rec.Host, rec.App, rec.Account) |
| 1472 | // Hibernate is deliberately NOT converged here even when |
| 1473 | // explicit: this first pass over a hand-added route should |
| 1474 | // change what it serves and who owns the column, not whether a |
| 1475 | // live process is allowed to sleep. It converges on the next |
| 1476 | // pass through the owned branch above, by which point the |
| 1477 | // adoption is already visible in the log. |
| 1478 | convergeOwned(reg, existing, rec, p, pKnown, cfg.DryRun) |
| 1479 | } |
| 1480 | return nil |
| 1481 | }); err != nil { |
| 1482 | log.Printf("provision: claim install %s: %v", rec.Host, err) |
| 1483 | } |
| 1484 | } |
| 1485 | |
| 1486 | // --- removal, only when nothing failed to read this pass ------------------ |
| 1487 | // Three conditions, each load-bearing: Provisioned (the flag — not the |
| 1488 | // app, not the channel — grants removal authority; a hand-added route |
| 1489 | // never has it), scope (this box's own region/pool jurisdiction — a |
| 1490 | // foreign box's provisioned route is not this pass's business), and a |
| 1491 | // retirement the BUCKET decided rather than one this pass guessed — |
| 1492 | // either the record is gone, or the app carries a trash marker (#340, |
| 1493 | // and see the trash cache above for why the marker's two answers are |
| 1494 | // not the same answer). |
| 1495 | // |
| 1496 | // Scope is asked through the SAME inScope as the record scan above, with |
| 1497 | // the class read off the row (everywhereRoute) because the record whose |
| 1498 | // class it was may be exactly what is gone. Both halves must widen |
| 1499 | // together: |
| 1500 | // a far box that mints a static route under the widened mint gate and then |
| 1501 | // judges it out of scope here would serve a retired site forever. |
| 1502 | // |
| 1503 | // What reaches reg.Delete here may carry Backing, Tunnel or Caps even |
| 1504 | // though shapeMismatch refuses to ADOPT a row carrying any of them: those |
| 1505 | // refusals cover the adoption instant only. `carlos route -backing/-unit/ |
| 1506 | // -tunnel/-grant` all still work on a Provisioned row, so a route can |
| 1507 | // acquire an operator column after it was minted or adopted and remain |
| 1508 | // one unattended record-deletion away from registry.Delete's grant purge. |
| 1509 | // That asymmetry predates this pass's adoption step; it is stated here so |
| 1510 | // the adoption guard is not mistaken for a standing guarantee. |
| 1511 | // |
| 1512 | // `replication` (discussion #10) is the same KIND of column — box-scoped, |
| 1513 | // operator-typed, inexpressible in a record — and is deliberately exempt |
| 1514 | // from shapeMismatch rather than added to it. The three that are listed |
| 1515 | // there are listed for a reason that does not hold for this one: each |
| 1516 | // grants or relocates AUTHORITY over a route's process (a capability, a |
| 1517 | // process owner, an ingress path), and adoption is itself an authority |
| 1518 | // transfer, so handing bucket control a row that already carries one is |
| 1519 | // the thing worth stopping to look at. A replication rung grants nothing. |
| 1520 | // It names how often a database is copied off the box, and it survives |
| 1521 | // adoption unchanged: adoption's own write is SetProvisioned, the owned |
| 1522 | // branch converges through narrow setters that never touch this column, |
| 1523 | // and the one path that DOES clear it — registry.SetHibernate, when a |
| 1524 | // record turns hibernation off — is the honest answer rather than a loss, |
| 1525 | // since a non-hibernating route's rung is read by nobody. |
| 1526 | // |
| 1527 | // Listing it would mean an operator who tuned a large sleeping tenant's |
| 1528 | // rung had, by that act, opted the route out of the backfill this pass |
| 1529 | // exists to perform — permanently and invisibly, until someone thought to |
| 1530 | // clear a column to let a record adopt a row. That is the exact opposite |
| 1531 | // trade from the other three: it would strand precisely the hibernating |
| 1532 | // tenants the ladder was built for, in exchange for guarding an authority |
| 1533 | // this column does not carry. |
| 1534 | // |
| 1535 | // retire runs that phase over one set of facts: the records, the |
| 1536 | // protected hosts and the trash lookup. With stopAtFirst it removes |
| 1537 | // nothing and reports whether any row would go. A record view |
| 1538 | // snapshot's absence is not the bucket's (a write that missed its bump |
| 1539 | // is missing from a Complete snapshot until the backstop rebuild), so a |
| 1540 | // pass over a snapshot that would remove anything decides every removal |
| 1541 | // again from the bucket's own walk, below (S3 change-index design §4.4). |
| 1542 | retire := func(recs map[string]Record, protect map[string]bool, trashFor func(acct, app string) trashState, stopAtFirst bool) (wouldRemove bool) { |
| 1543 | for _, rt := range routes { |
| 1544 | // Removal authority is scoped to this pass's OWN kind. A sidecar |
| 1545 | // row is Provisioned too — that flag is what grants a pass removal |
| 1546 | // authority — and it can never appear in the INSTANCE record set, |
| 1547 | // so without this guard the two passes would delete each other's |
| 1548 | // rows on every tick, forever, with no error anywhere and nothing |
| 1549 | // in the logs that reads as a failure. ReconcileSidecars carries |
| 1550 | // the mirror of this guard; both directions are asserted |
| 1551 | // (TestInstanceReconcileNeverRemovesSidecarRows). |
| 1552 | if rt.Kind == SidecarKind { |
| 1553 | continue |
| 1554 | } |
| 1555 | if !rt.Provisioned || !cfg.inScope(rt.Region, rt.Pool, everywhereRoute(rt)) { |
| 1556 | continue |
| 1557 | } |
| 1558 | // Two triggers, asked in this order because the first is the |
| 1559 | // cheap one: the bucket retired the instance (its record is |
| 1560 | // gone), or the bucket retired the whole APP (a trash marker, |
| 1561 | // while the record stays put). Both name the bucket's own |
| 1562 | // decision; neither is a guess this pass made. |
| 1563 | // |
| 1564 | // The trash lookup needs both halves of the app's identity off |
| 1565 | // the row. A Provisioned row always carries them — routeFor |
| 1566 | // copies them from the record — so a row missing either is a |
| 1567 | // shape this pass did not mint, and not one it will retire. |
| 1568 | why := "record gone" |
| 1569 | if _, ok := recs[rt.Host]; ok { |
| 1570 | if rt.Account == "" || rt.App == "" || !trashFor(rt.Account, rt.App).removable { |
| 1571 | continue |
| 1572 | } |
| 1573 | why = "app in the trash" |
| 1574 | } |
| 1575 | if journaled[rt.Host] { |
| 1576 | continue // the import journal owns the row (seedreconcile.go) |
| 1577 | } |
| 1578 | if protect[rt.Host] { |
| 1579 | log.Printf("provision: not removing %s: its record exists but could not be read cleanly this pass", rt.Host) |
| 1580 | continue |
| 1581 | } |
| 1582 | // A participant of a running operation keeps its row whatever |
| 1583 | // the record's region says (moves spec §9.6, claim spec §8): a |
| 1584 | // move's source after the flip, its target during a rollback. |
| 1585 | // The operation deletes the row at its own finalization. |
| 1586 | if cfg.KeepParticipant != nil && cfg.KeepParticipant(rt.Host) { |
| 1587 | log.Printf("provision: not removing %s (%s): participant of a running operation", rt.Host, why) |
| 1588 | continue |
| 1589 | } |
| 1590 | if stopAtFirst { |
| 1591 | return true |
| 1592 | } |
| 1593 | if cfg.DryRun { |
| 1594 | log.Printf("provision: dry-run: would remove retired %s route %s (%s)", rt.Kind, rt.Host, why) |
| 1595 | continue |
| 1596 | } |
| 1597 | if err := cfg.claimRetire(ctx, reg, rt); err != nil { |
| 1598 | log.Printf("provision: remove retired %s route %s: %v", rt.Kind, rt.Host, err) |
| 1599 | continue |
| 1600 | } |
| 1601 | log.Printf("provision: removed retired %s route %s (%s)", rt.Kind, rt.Host, why) |
| 1602 | } |
| 1603 | return false |
| 1604 | } |
| 1605 | if readFailed || !retire(recs, protect, trashFor, onSnapshot(s)) { |
| 1606 | return nil |
| 1607 | } |
| 1608 | // A snapshot pass with a removal to make: the record walk again, over |
| 1609 | // the bucket, and the removal phase over what it read, by the walk's |
| 1610 | // own rules (invalid shapes and empty channels protected before scope, |
| 1611 | // hosts matched across every key, the bucket's trash marker). Anything |
| 1612 | // it cannot read keeps every row this pass, as in the walk. |
| 1613 | live := freshStore(s) |
| 1614 | lsc, err := scan(live) |
| 1615 | if err != nil || lsc.readFailed { |
| 1616 | log.Printf("provision: not removing any route this pass: the record view lacks a record or holds a trash marker, and the bucket's own walk could not be read cleanly%s", walkErrSuffix(err)) |
| 1617 | return nil |
| 1618 | } |
| 1619 | retire(lsc.recs, lsc.protect, trashOver(live, nil), false) |
| 1620 | return nil |
| 1621 | } |
| 1622 | |
| 1623 | // mintHeld mints a move participant's row held (moves spec §9.6 "mint"): the |
| 1624 | // record's own route at flat paths whatever the box's layout intent (Phase 1 |
| 1625 | // lands flat, G11), with status and hold_op set by the INSERT, so it is never |
| 1626 | // ready for even one pass. The refusals a ready mint has (no-inbound box, |
| 1627 | // the config gate, a dry run) apply unchanged, and so does its claim hook |
| 1628 | // (claim Plan 6b, cfg.claimInstall): the write runs inside it, after every |
| 1629 | // refusal that needs no claim, so with claims on the row carries the owner |
| 1630 | // the hook grants and the hook's guard and inventory effects happen only for |
| 1631 | // a mint that is going to land. With claims off the hook is a pass-through. |
| 1632 | // A status that is not a hold, or an empty op, mints nothing: a |
| 1633 | // participant's row is never minted ready. |
| 1634 | func mintHeld(ctx context.Context, reg *registry.Registry, cfg ReconcileConfig, rec Record, seed, status, op string, policy *UpdatePolicy, policyKnown bool) { |
| 1635 | if !registry.IsHold(status) || op == "" { |
| 1636 | log.Printf("provision: %s: not minting: the move's hold %q by %q is not a hold", rec.Host, status, op) |
| 1637 | return |
| 1638 | } |
| 1639 | if cfg.Inbound == InboundNone { |
| 1640 | log.Printf("provision: refusing to mint held route %s (app %q acct %q): this box declares "+ |
| 1641 | "CARLOS_INBOUND=none (no-inbound) and cannot serve routed instance traffic", rec.Host, rec.App, rec.Account) |
| 1642 | return |
| 1643 | } |
| 1644 | if !policyKnown { |
| 1645 | // The same wait as the ordinary mint: an instance that inherits |
| 1646 | // manual from its app must not land held as auto. |
| 1647 | log.Printf("provision: %s: not minted this pass: the app's update policy could not be read", rec.Host) |
| 1648 | return |
| 1649 | } |
| 1650 | nr := RouteFor(rec, cfg.DataDir, seed, false, policy) |
| 1651 | nr.Status, nr.HoldOp = status, op |
| 1652 | if cfg.ConfigGate != nil { |
| 1653 | if err := cfg.ConfigGate(ctx, nr); err != nil { |
| 1654 | return |
| 1655 | } |
| 1656 | } |
| 1657 | written := false |
| 1658 | selected := ClaimRecord{Host: rec.Host, Key: RecordKey(rec.Account, rec.App, rec.Host), Family: "instance", Account: rec.Account, App: rec.App, CreatedAt: rec.CreatedAt, Value: rec, Route: nr, DryRun: cfg.DryRun} |
| 1659 | if err := cfg.claimInstall(ctx, selected, nil, func(owner registry.ClaimOwnership) error { |
| 1660 | nr.ClaimOwnership = owner |
| 1661 | if cfg.DryRun { |
| 1662 | log.Printf("provision: dry-run: would mint %s route %s held %s by %s (app %q acct %q channel %q)", |
| 1663 | nr.Kind, rec.Host, status, op, rec.App, rec.Account, rec.Channel) |
| 1664 | return nil |
| 1665 | } |
| 1666 | if err := reg.Upsert(nr); err != nil { |
| 1667 | log.Printf("provision: mint held %s: %v", rec.Host, err) |
| 1668 | return nil |
| 1669 | } |
| 1670 | log.Printf("provision: minted %s route %s held %s by %s (app %q acct %q)", nr.Kind, rec.Host, status, op, rec.App, rec.Account) |
| 1671 | written = true |
| 1672 | return nil |
| 1673 | }); err != nil { |
| 1674 | log.Printf("provision: claim install held %s: %v", rec.Host, err) |
| 1675 | } |
| 1676 | if written && cfg.HoldWritten != nil { |
| 1677 | cfg.HoldWritten(rec.Host) |
| 1678 | } |
| 1679 | } |
| 1680 | |
| 1681 | // hostHoldLocks is one in-process mutex per host for the hold decision on |
| 1682 | // an existing row: see WithHostHoldLock. |
| 1683 | var hostHoldLocks struct { |
| 1684 | mu sync.Mutex |
| 1685 | hosts map[string]*sync.Mutex |
| 1686 | } |
| 1687 | |
| 1688 | // WithHostHoldLock runs fn holding host's hold lock, exclusively within this |
| 1689 | // process. Reconcile's hold decision for an existing row (holdExisting: the |
| 1690 | // record re-read, the hold asked again of it, the conditional write) runs |
| 1691 | // under it, so nothing that ends the hold can run between that read and |
| 1692 | // that write. The landing's open and its finalisation (moves Task 10, |
| 1693 | // movesland.go) MUST take this same lock around their record read and their |
| 1694 | // ReleaseHold or DeleteIfHold, and so must any later writer that ends a hold |
| 1695 | // reconcile could re-assert from the record: without it, an open that lands |
| 1696 | // in the window is re-held on the next line. Writers in other processes |
| 1697 | // (root) are met by SetHoldIf's own condition, not by this lock. |
| 1698 | // |
| 1699 | // A leaf: its holder reads one record, writes one row and takes no other |
| 1700 | // lock. Nothing takes it while holding the held-stage lock (G30), and |
| 1701 | // nothing holds it across a wait on another host. |
| 1702 | func WithHostHoldLock(host string, fn func()) { |
| 1703 | hostHoldLocks.mu.Lock() |
| 1704 | if hostHoldLocks.hosts == nil { |
| 1705 | hostHoldLocks.hosts = map[string]*sync.Mutex{} |
| 1706 | } |
| 1707 | l := hostHoldLocks.hosts[host] |
| 1708 | if l == nil { |
| 1709 | l = &sync.Mutex{} |
| 1710 | hostHoldLocks.hosts[host] = l |
| 1711 | } |
| 1712 | hostHoldLocks.mu.Unlock() |
| 1713 | l.Lock() |
| 1714 | defer l.Unlock() |
| 1715 | fn() |
| 1716 | } |
| 1717 | |
| 1718 | // holdExisting holds an existing row for a move (G10): Upsert never changes |
| 1719 | // a hold on conflict, so an unheld row of this pass's own is held with |
| 1720 | // SetHoldIf. A row already held by op is left as is; a row held by another |
| 1721 | // op is that op's, and is left alone; a row this pass did not mint (not |
| 1722 | // Provisioned, or another app's) is not taken into a move. |
| 1723 | // |
| 1724 | // Three rules keep a hold write from closing a row an operation has opened, |
| 1725 | // or from landing over a hold written since the pass's listing (fix round 1): |
| 1726 | // |
| 1727 | // - a recovery hold (restoring, releasing) is never written over an |
| 1728 | // unheld row. The recovering operation holds its own row as it enters |
| 1729 | // the recovery, and opens it while the record still names the |
| 1730 | // recovery (moves spec §7.4 reverting `probed`, §7.8 `opened`), so an |
| 1731 | // unheld row there is one it opened. Only a missing row is minted. |
| 1732 | // - the record is read again just before the write, and the hold is |
| 1733 | // asked again of the fresh record, all under the host's hold lock |
| 1734 | // (WithHostHoldLock), which the open and the finalisation take too: |
| 1735 | // this pass listed the records before the routes, and a target opens |
| 1736 | // its row only after its commit, so a fresh record still at landing |
| 1737 | // means the row has not been opened, and under the lock it cannot be |
| 1738 | // opened before the write lands. |
| 1739 | // - the write is SetHoldIf expecting the row unheld: a hold another op |
| 1740 | // wrote since the listing (or since this pass's own route read) is |
| 1741 | // never overwritten; the row is left to it and the next pass judges |
| 1742 | // it afresh. |
| 1743 | // |
| 1744 | // SetHoldIf can wait out root's activation guard and fail (G30): logged, |
| 1745 | // and the next pass tries again. |
| 1746 | func holdExisting(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig, rec Record, existing registry.Route, status, op string) { |
| 1747 | switch { |
| 1748 | case !registry.IsHold(status) || op == "": |
| 1749 | log.Printf("provision: %s: not holding: the move's hold %q by %q is not a hold", rec.Host, status, op) |
| 1750 | return |
| 1751 | case existing.HoldOp == op: |
| 1752 | return |
| 1753 | case existing.HoldOp != "": |
| 1754 | log.Printf("provision: %s: held %s by %s, not by %s; leaving it", rec.Host, existing.Status, existing.HoldOp, op) |
| 1755 | return |
| 1756 | case status == "restoring" || status == "releasing": |
| 1757 | return |
| 1758 | case !existing.Provisioned || existing.App != rec.App || existing.Account != rec.Account: |
| 1759 | log.Printf("provision: %s: the row is not this pass's (app %q acct %q); not holding it for %s", rec.Host, existing.App, existing.Account, op) |
| 1760 | return |
| 1761 | case cfg.DryRun: |
| 1762 | log.Printf("provision: dry-run: would hold %s %s by %s", rec.Host, status, op) |
| 1763 | return |
| 1764 | } |
| 1765 | WithHostHoldLock(rec.Host, func() { |
| 1766 | // From the bucket, not a record view snapshot: the listing may be one. |
| 1767 | fresh, err := GetRecord(ctx, freshStore(s), rec.Account, rec.App, rec.Host) |
| 1768 | if err != nil { |
| 1769 | log.Printf("provision: %s: not holding this pass: re-read the record: %v", rec.Host, err) |
| 1770 | return |
| 1771 | } |
| 1772 | if st, o, ok := cfg.Hold(*fresh); !ok || st != status || o != op { |
| 1773 | return // the move moved on since the listing; the next pass judges it afresh |
| 1774 | } |
| 1775 | if cfg.beforeHoldWrite != nil { |
| 1776 | cfg.beforeHoldWrite(rec.Host) |
| 1777 | } |
| 1778 | held, err := reg.SetHoldIf(rec.Host, status, op, "") |
| 1779 | if err != nil { |
| 1780 | log.Printf("provision: hold %s %s by %s: %v", rec.Host, status, op, err) |
| 1781 | return |
| 1782 | } |
| 1783 | if !held { |
| 1784 | log.Printf("provision: %s: not held %s by %s: the row's hold changed under this pass; the next pass judges it afresh", rec.Host, status, op) |
| 1785 | return |
| 1786 | } |
| 1787 | log.Printf("provision: held %s %s by %s (was %s)", rec.Host, status, op, existing.Status) |
| 1788 | if cfg.HoldWritten != nil { |
| 1789 | cfg.HoldWritten(rec.Host) |
| 1790 | } |
| 1791 | }) |
| 1792 | } |
| 1793 | |
| 1794 | // gateWord renders a gate column for a log line: "open" or "set", never the |
| 1795 | // hash — it has no business in a journal. |
| 1796 | func gateWord(hash string) string { |
| 1797 | if hash == "" { |
| 1798 | return "open" |
| 1799 | } |
| 1800 | return "set" |
| 1801 | } |
| 1802 | |
| 1803 | // knownAccounts lists accounts/ and materializes the sqid set this pass |
| 1804 | // trusts for ParseRecordKey's "known account" guard — the same three-way |
| 1805 | // sqid agreement domains.Reconcile enforces (reconcile.go:106): the key |
| 1806 | // path's sqid, the manifest body's own sqid, and the frozen encoding of the |
| 1807 | // body's id must all agree, or the manifest is a lie about identity and is |
| 1808 | // skipped, loudly. A tombstoned manifest (m.Deleted()) is skipped too — |
| 1809 | // retired accounts own no in-scope records. |
| 1810 | // |
| 1811 | // Listing failure returns an error (nothing here can proceed without it); a |
| 1812 | // per-manifest read or parse failure instead sets readFailed and is |
| 1813 | // otherwise skipped — never fatal to the pass, but disabling enough that the |
| 1814 | // removal phase in Reconcile refuses to run on it. |
| 1815 | func knownAccounts(ctx context.Context, s release.Store) (known map[string]bool, readFailed bool, err error) { |
| 1816 | known = map[string]bool{domains.OpsSqid: true} |
| 1817 | keys, err := s.List(ctx, "accounts/") |
| 1818 | if err != nil { |
| 1819 | return nil, false, fmt.Errorf("provision: list accounts: %w", err) |
| 1820 | } |
| 1821 | var manifestKeys []string |
| 1822 | for _, key := range keys { |
| 1823 | if _, ok := domains.AccountManifestSqid(key); ok { |
| 1824 | manifestKeys = append(manifestKeys, key) |
| 1825 | } |
| 1826 | } |
| 1827 | for _, part := range chunks(manifestKeys) { |
| 1828 | bodies := getEach(ctx, s, part) |
| 1829 | for _, key := range part { |
| 1830 | sqid, _ := domains.AccountManifestSqid(key) |
| 1831 | body, err := bodies[key].body, bodies[key].err |
| 1832 | if err != nil { |
| 1833 | log.Printf("provision: skipping %s: read failed: %v", key, err) |
| 1834 | readFailed = true |
| 1835 | continue |
| 1836 | } |
| 1837 | var m domains.AccountManifest |
| 1838 | if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" { |
| 1839 | log.Printf("provision: skipping %s: malformed account manifest: %v", key, err) |
| 1840 | readFailed = true |
| 1841 | continue |
| 1842 | } |
| 1843 | if m.Sqid != sqid || domains.Sqid(m.ID) != sqid { |
| 1844 | log.Printf("provision: skipping %s: sqid %q/%q is not Sqid(%d)=%q", key, sqid, m.Sqid, m.ID, domains.Sqid(m.ID)) |
| 1845 | readFailed = true |
| 1846 | continue |
| 1847 | } |
| 1848 | if m.Deleted() { |
| 1849 | continue |
| 1850 | } |
| 1851 | known[sqid] = true |
| 1852 | } |
| 1853 | } |
| 1854 | return known, readFailed, nil |
| 1855 | } |
| 1856 | |
| 1857 | // trashGate builds the per-(acct, app) "is this app in the trash" predicate |
| 1858 | // the SIDECAR pass gates MINTING on, caching each answer for the pass. what |
| 1859 | // names the route family in the log line and is all that ever varied between |
| 1860 | // callers. |
| 1861 | // |
| 1862 | // The instance pass had the same shape until #340 gave it a SECOND answer — |
| 1863 | // whether a trashed app's routes may also be RETIRED, which needs the |
| 1864 | // production-flag guard (trashFor/trashState above). Retirement is not a |
| 1865 | // question this predicate can answer, so the sidecar pass keeps the |
| 1866 | // mint-only one rather than inheriting an authority its own removal phase |
| 1867 | // was never designed around. A trashed app's sidecar rows therefore linger |
| 1868 | // until their records go, which is the same behaviour every pass had before |
| 1869 | // #340 and the safe direction — but it is a gap, recorded here and in the |
| 1870 | // design's follow-ups, not a decision this branch made. |
| 1871 | // |
| 1872 | // The predicate answers TRUE for an unreadable or malformed marker as well as |
| 1873 | // a valid one: not minting is the reversible direction, and a bucket writer |
| 1874 | // who could plant a marker to stall minting can already retire the record |
| 1875 | // outright. |
| 1876 | func trashGate(ctx context.Context, s release.Store, what string) func(acct, app string) bool { |
| 1877 | cache := map[acctApp]bool{} |
| 1878 | seen := map[acctApp]bool{} |
| 1879 | return func(acct, app string) bool { |
| 1880 | k := acctApp{acct, app} |
| 1881 | if seen[k] { |
| 1882 | return cache[k] |
| 1883 | } |
| 1884 | seen[k] = true |
| 1885 | tomb, err := release.ReadAppTombstone(ctx, s, acct, app) |
| 1886 | switch { |
| 1887 | case err != nil: |
| 1888 | log.Printf("provision: trash marker for %s/%s unreadable: %v — not minting for this app", acct, app, err) |
| 1889 | cache[k] = true |
| 1890 | case tomb != nil: |
| 1891 | log.Printf("provision: %s/%s is in the trash — not minting %s routes for it", acct, app, what) |
| 1892 | cache[k] = true |
| 1893 | } |
| 1894 | return cache[k] |
| 1895 | } |
| 1896 | } |
| 1897 | |
| 1898 | // ListAllRecords enumerates EVERY instance record in the deployment — the |
| 1899 | // same account-qualified walk Reconcile does (list accounts/ for the trusted |
| 1900 | // sqid set, list apps/, ParseRecordKey, Get, decode), exported for callers |
| 1901 | // that need deployment-GLOBAL route facts rather than one app's. |
| 1902 | // |
| 1903 | // It exists because the registry is not deployment-global truth. Routes are |
| 1904 | // provisioned per box: an everywhereClass record (exec/unit/socket kinds) is |
| 1905 | // homed to ONE box, so that host is simply absent from every other box's |
| 1906 | // registry. Any deployment-wide decision derived from one box's registry |
| 1907 | // therefore changes meaning when the work moves to a different box — which, |
| 1908 | // for DNS steering (internal/steering), meant a housekeeper lease migration |
| 1909 | // could delete a live steered host's records and NXDOMAIN it until the lease |
| 1910 | // came back. The records ARE the deployment's truth; this is how to read all |
| 1911 | // of them. |
| 1912 | // |
| 1913 | // FAIL-CLOSED, and that is the deliberate difference from Reconcile's own |
| 1914 | // walk. Reconcile tolerates a partial read (it logs, sets readFailed, and |
| 1915 | // vetoes only its removal phase) because it must still MINT on a degraded |
| 1916 | // pass. A caller deriving global state has no such split: a short list is |
| 1917 | // indistinguishable from records having been deleted, so every read failure |
| 1918 | // is an error and no records are returned — |
| 1919 | // |
| 1920 | // - accounts/ listing fails, or ANY account manifest is unreadable, |
| 1921 | // malformed, or disagrees with its own sqid: that account's records all |
| 1922 | // fail ParseRecordKey's known-account guard and would vanish silently |
| 1923 | // from the result. Fatal. |
| 1924 | // - apps/ listing fails, or ANY record object will not Get. Fatal. |
| 1925 | // |
| 1926 | // A record that READS but does not decode — or whose body disagrees with its |
| 1927 | // own path — is the other class: a content problem in one object, which is |
| 1928 | // skipped and logged (Reconcile's `protect` treatment, minus the registry |
| 1929 | // bookkeeping). One malformed record must not blind a caller to every other. |
| 1930 | func ListAllRecords(ctx context.Context, s release.Store) ([]Record, error) { |
| 1931 | recs, _, _, err := readAllRecords(ctx, s, true) |
| 1932 | return recs, err |
| 1933 | } |
| 1934 | |
| 1935 | // ListAllRecordsStrict is ListAllRecords with one more answer: the hosts it |
| 1936 | // skipped. ListAllRecords is already fail-closed on every READ failure (a |
| 1937 | // listing that fails, a manifest or a record that will not Get is an error |
| 1938 | // and no records come back), but it logs and drops a record that reads and |
| 1939 | // then does not decode, or whose body disagrees with its own path, so the |
| 1940 | // set it returns is silently short of those records. A caller that draws a |
| 1941 | // conclusion from a record's absence (the seeds sweep deciding no record |
| 1942 | // names a snapshot) cannot tell that gap from a record that is not there. |
| 1943 | // ListAllRecordsStrict names the gap: unreadable holds the host of each |
| 1944 | // record key whose body did not decode or disagreed with its path (keyed by |
| 1945 | // the host in the path, never the body's). unreadable is non-nil on |
| 1946 | // success. The records returned are exactly ListAllRecords' set, and every |
| 1947 | // error is ListAllRecords' error; a caller deleting on absence must keep |
| 1948 | // everything that belongs to a host in unreadable. |
| 1949 | // |
| 1950 | // One more blind spot, shared with ListAllRecords and Reconcile: records |
| 1951 | // under an account with no manifest, or whose manifest is soft-deleted, are |
| 1952 | // not in recs and not in unreadable. They are invisible, not unreadable. |
| 1953 | func ListAllRecordsStrict(ctx context.Context, s release.Store) (recs []Record, unreadable map[string]bool, err error) { |
| 1954 | recs, _, unreadable, err = readAllRecords(ctx, s, true) |
| 1955 | if err != nil { |
| 1956 | return nil, nil, err |
| 1957 | } |
| 1958 | return recs, unreadable, nil |
| 1959 | } |
| 1960 | |
| 1961 | // ListAllRecordsTolerant is ListAllRecords for a caller that acts on each |
| 1962 | // record it reads, one at a time, and draws nothing from a record's absence |
| 1963 | // (the run sweep's expiry and lost-run phases). An account manifest that |
| 1964 | // will not read cleanly, an app whose records will not list, or a record |
| 1965 | // that will not Get is logged and skipped, and complete comes back false; |
| 1966 | // every other record is returned. err is only the top-level listing failing, |
| 1967 | // when nothing could be read. A caller that also draws conclusions from |
| 1968 | // absence must act on that only when complete is true. |
| 1969 | func ListAllRecordsTolerant(ctx context.Context, s release.Store) (recs []Record, complete bool, err error) { |
| 1970 | recs, complete, _, err = readAllRecords(ctx, s, false) |
| 1971 | return recs, complete, err |
| 1972 | } |
| 1973 | |
| 1974 | // readAllRecords is both: strict fails on any read failure, as |
| 1975 | // ListAllRecords documents; tolerant logs it, skips what it could not read |
| 1976 | // and reports complete false. In both modes unreadable holds the path host |
| 1977 | // of every record that read but did not decode or disagreed with its path |
| 1978 | // (skipped, logged); it is non-nil whenever err is nil. |
| 1979 | func readAllRecords(ctx context.Context, s release.Store, strict bool) (recs []Record, complete bool, unreadable map[string]bool, err error) { |
| 1980 | complete = true |
| 1981 | unreadable = map[string]bool{} |
| 1982 | known, readFailed, err := knownAccounts(ctx, s) |
| 1983 | if err != nil { |
| 1984 | return nil, false, nil, err |
| 1985 | } |
| 1986 | if readFailed { |
| 1987 | if strict { |
| 1988 | return nil, false, nil, errors.New("provision: an account manifest could not be read cleanly — refusing to report a partial record set") |
| 1989 | } |
| 1990 | log.Printf("provision: an account manifest could not be read cleanly: that account's records are skipped") |
| 1991 | complete = false |
| 1992 | } |
| 1993 | // Sorted (listAppSubtree sorts) so the result — and anything folded |
| 1994 | // from it — is stable call to call, whatever order the store lists in. |
| 1995 | keys, partial, err := listAppSubtree(ctx, s, known, "instances/") |
| 1996 | if err != nil { |
| 1997 | return nil, false, nil, fmt.Errorf("provision: list apps: %w", err) |
| 1998 | } |
| 1999 | if partial { |
| 2000 | if strict { |
| 2001 | return nil, false, nil, errors.New("provision: an app's records could not be listed — refusing to report a partial record set") |
| 2002 | } |
| 2003 | log.Printf("provision: an app's records could not be listed: they are skipped") |
| 2004 | complete = false |
| 2005 | } |
| 2006 | var recordKeys []string |
| 2007 | for _, key := range keys { |
| 2008 | if _, _, _, ok := ParseRecordKey(key, known); ok { |
| 2009 | recordKeys = append(recordKeys, key) |
| 2010 | } |
| 2011 | } |
| 2012 | for _, part := range chunks(recordKeys) { |
| 2013 | bodies := getEach(ctx, s, part) |
| 2014 | for _, key := range part { |
| 2015 | acct, app, host, _ := ParseRecordKey(key, known) |
| 2016 | raw, err := bodies[key].body, bodies[key].err |
| 2017 | if err != nil { |
| 2018 | if strict { |
| 2019 | return nil, false, nil, fmt.Errorf("provision: read record %s: %w", key, err) |
| 2020 | } |
| 2021 | log.Printf("provision: skipping record %s: unreadable: %v", key, err) |
| 2022 | complete = false |
| 2023 | continue |
| 2024 | } |
| 2025 | rec, err := DecodeRecord(raw) |
| 2026 | if err != nil { |
| 2027 | log.Printf("provision: skipping record %s: malformed: %v", key, err) |
| 2028 | unreadable[host] = true |
| 2029 | continue |
| 2030 | } |
| 2031 | if rec.Host != host || rec.Account != acct || rec.App != app { |
| 2032 | log.Printf("provision: skipping record %s: disagrees with its own path (host=%q account=%q app=%q)", |
| 2033 | key, rec.Host, rec.Account, rec.App) |
| 2034 | unreadable[host] = true |
| 2035 | continue |
| 2036 | } |
| 2037 | recs = append(recs, rec) |
| 2038 | } |
| 2039 | } |
| 2040 | return recs, complete, unreadable, nil |
| 2041 | } |
| 2042 | |
| 2043 | // listAppSubtree returns the keys a record reader needs to see: every key |
| 2044 | // under apps/<acct>/<app>/<sub> for each known account and each of its apps, |
| 2045 | // sorted — or, on a store that cannot be walked that way, every key under |
| 2046 | // apps/, as the readers always listed. The callers parse every key exactly as |
| 2047 | // before (ParseRecordKey, ParseSidecarKey); this changes only which keys they |
| 2048 | // are shown. |
| 2049 | // |
| 2050 | // It exists because List(ctx, "apps/") returns every object under every app — |
| 2051 | // every release artifact and every log chunk — to find the records among |
| 2052 | // them. On the flagship on 2026-10-05 apps/ held 59,288 objects, 50,695 of |
| 2053 | // them logs, for 152 instance records: sixty list pages per reader, three |
| 2054 | // readers on every serial bump, and most of why a box took 16 to 43 seconds |
| 2055 | // to notice a promote (#231). The walk is one ListDirs of apps/, one per |
| 2056 | // known account, and one small List per app. |
| 2057 | // |
| 2058 | // It finds the same records the full listing did. A record's account is |
| 2059 | // always one of known (the parsers refuse any other), and its app is a |
| 2060 | // directory directly under the account, so no record lives anywhere the walk |
| 2061 | // does not look — PROVIDED ListDirs answers in the bucket's shape. Only a |
| 2062 | // store that says so (release.HasBucketListDirs: the S3 store, and a prefix |
| 2063 | // store over one) is walked; the fleet door, whose ListDirs("apps/") is its |
| 2064 | // own account's app directories, and DirStore, which lists by walking its |
| 2065 | // whole root, are listed the old way. Assuming the shape on the fleet door |
| 2066 | // found no records at all, and the reconcile pass would have read that as |
| 2067 | // every route retired. |
| 2068 | // |
| 2069 | // Failure comes in two sizes. The top-level listing failing is an error, as |
| 2070 | // the single List failing was. One account's or one app's listing failing is |
| 2071 | // PARTIAL: the keys that were listed come back, and partial reports that some |
| 2072 | // were not. The callers treat that as they treat a record they could not |
| 2073 | // read — removal is vetoed for the whole pass, everything that was seen still |
| 2074 | // converges — because a partial record set must never read as "the bucket |
| 2075 | // retired the rest", and one app's bad minute must not stop every other app's |
| 2076 | // routes from being minted. |
| 2077 | func listAppSubtree(ctx context.Context, s release.Store, known map[string]bool, sub string) (keys []string, partial bool, err error) { |
| 2078 | if !release.HasBucketListDirs(s) { |
| 2079 | keys, err := s.List(ctx, "apps/") |
| 2080 | slices.Sort(keys) // Store.List promises no order; the walk's is sorted |
| 2081 | return keys, false, err |
| 2082 | } |
| 2083 | accts, err := s.ListDirs(ctx, "apps/") |
| 2084 | if err != nil { |
| 2085 | return nil, false, err |
| 2086 | } |
| 2087 | var acctPrefixes []string |
| 2088 | for _, ap := range accts { |
| 2089 | if known[strings.TrimSuffix(strings.TrimPrefix(ap, "apps/"), "/")] { |
| 2090 | acctPrefixes = append(acctPrefixes, ap) |
| 2091 | } |
| 2092 | } |
| 2093 | |
| 2094 | var ( |
| 2095 | mu sync.Mutex |
| 2096 | prefixes []string |
| 2097 | out []string |
| 2098 | ) |
| 2099 | fail := func(what string, err error) { |
| 2100 | log.Printf("provision: listing %s: %v — its records are unseen this pass", what, err) |
| 2101 | partial = true |
| 2102 | } |
| 2103 | // A handful at a time, accounts and apps alike: each is one small round |
| 2104 | // trip, and a box walks a few dozen accounts and a few hundred apps. |
| 2105 | forEach(acctPrefixes, func(ap string) { |
| 2106 | apps, err := s.ListDirs(ctx, ap) |
| 2107 | mu.Lock() |
| 2108 | defer mu.Unlock() |
| 2109 | if err != nil { |
| 2110 | fail(ap, err) |
| 2111 | return |
| 2112 | } |
| 2113 | for _, p := range apps { |
| 2114 | prefixes = append(prefixes, p+sub) |
| 2115 | } |
| 2116 | }) |
| 2117 | forEach(prefixes, func(p string) { |
| 2118 | keys, err := s.List(ctx, p) |
| 2119 | mu.Lock() |
| 2120 | defer mu.Unlock() |
| 2121 | if err != nil { |
| 2122 | fail(p, err) |
| 2123 | return |
| 2124 | } |
| 2125 | out = append(out, keys...) |
| 2126 | }) |
| 2127 | slices.Sort(out) |
| 2128 | return out, partial, nil |
| 2129 | } |
| 2130 | |
| 2131 | // readChunk is how many keys a judging loop prefetches at once: enough to keep |
| 2132 | // listWalkers busy, few enough that a large record set is never held in |
| 2133 | // memory whole, and that a loop stopping at its first bad read has read only |
| 2134 | // a chunk ahead of it. |
| 2135 | const readChunk = 64 |
| 2136 | |
| 2137 | // chunks splits keys into runs of at most readChunk, in order. |
| 2138 | func chunks(keys []string) [][]string { |
| 2139 | var out [][]string |
| 2140 | for len(keys) > readChunk { |
| 2141 | out = append(out, keys[:readChunk]) |
| 2142 | keys = keys[readChunk:] |
| 2143 | } |
| 2144 | if len(keys) > 0 { |
| 2145 | out = append(out, keys) |
| 2146 | } |
| 2147 | return out |
| 2148 | } |
| 2149 | |
| 2150 | // fetched is one prefetched object: its body, or why it could not be read. |
| 2151 | type fetched struct { |
| 2152 | body []byte |
| 2153 | err error |
| 2154 | } |
| 2155 | |
| 2156 | // getEach reads every key in keys, listWalkers at a time, and returns each |
| 2157 | // one's body or error by key. |
| 2158 | // |
| 2159 | // The record readers used to Get each record in turn inside the loop that |
| 2160 | // judged it. With 152 records on the flagship that was the bulk of a 12.5s |
| 2161 | // reconcile and a 6.3s moves pass on every serial bump (#231, measured on the |
| 2162 | // box with the fast loop's stage timing). The loops are unchanged: they still |
| 2163 | // visit keys in the same order and judge each one exactly as before, only |
| 2164 | // reading the body from here rather than from the store. A key's read error |
| 2165 | // is that key's, as it was. |
| 2166 | // |
| 2167 | // Callers prefetch one chunk (chunks) at a time and judge it before the next. |
| 2168 | // A context already cancelled reads nothing: every key reports ctx.Err(), so |
| 2169 | // a judging loop sees a read failure for it, never a missing body. |
| 2170 | func getEach(ctx context.Context, s release.Store, keys []string) map[string]fetched { |
| 2171 | out := make(map[string]fetched, len(keys)) |
| 2172 | if err := ctx.Err(); err != nil { |
| 2173 | for _, k := range keys { |
| 2174 | out[k] = fetched{err: err} |
| 2175 | } |
| 2176 | return out |
| 2177 | } |
| 2178 | var mu sync.Mutex |
| 2179 | forEach(keys, func(k string) { |
| 2180 | body, _, err := s.Get(ctx, k) |
| 2181 | mu.Lock() |
| 2182 | out[k] = fetched{body: body, err: err} |
| 2183 | mu.Unlock() |
| 2184 | }) |
| 2185 | return out |
| 2186 | } |
| 2187 | |
| 2188 | // forEach runs f over items, listWalkers at a time, and returns when all |
| 2189 | // have finished. |
| 2190 | func forEach(items []string, f func(string)) { |
| 2191 | const listWalkers = 8 |
| 2192 | var wg sync.WaitGroup |
| 2193 | sem := make(chan struct{}, listWalkers) |
| 2194 | for _, it := range items { |
| 2195 | wg.Add(1) |
| 2196 | sem <- struct{}{} |
| 2197 | go func(it string) { |
| 2198 | defer wg.Done() |
| 2199 | defer func() { <-sem }() |
| 2200 | f(it) |
| 2201 | }(it) |
| 2202 | } |
| 2203 | wg.Wait() |
| 2204 | } |
| 2205 | |
| 2206 | // poolServed reports whether pool is one this box serves. An empty |
| 2207 | // cfg.Pools means the shared "" pool ONLY, never "every pool" — the same |
| 2208 | // convention domains.Config's AppsDomain=="" carries: unconfigured means the |
| 2209 | // default, not "all". |
| 2210 | func poolServed(pools []string, pool string) bool { |
| 2211 | if len(pools) == 0 { |
| 2212 | return pool == "" |
| 2213 | } |
| 2214 | return slices.Contains(pools, pool) |
| 2215 | } |
| 2216 | |
| 2217 | // SharedInstanceEligibility answers, from this box's own reconcile config, |
| 2218 | // whether it mints an ORDINARY instance record: one in the shared "" |
| 2219 | // pool, needing inbound service. The edges file publishes both answers |
| 2220 | // (serves_shared_pool, accepts_inbound) so the console never offers an |
| 2221 | // automatic choice of a region whose boxes would refuse the record |
| 2222 | // (nearest-region spec §6.1). It is the reconciler's own predicates — |
| 2223 | // poolServed and the CARLOS_INBOUND=none refusal in the reconcile pass — so |
| 2224 | // publication and minting cannot disagree. |
| 2225 | func (cfg ReconcileConfig) SharedInstanceEligibility() (sharedPool, inbound bool) { |
| 2226 | return poolServed(cfg.Pools, ""), cfg.Inbound != InboundNone |
| 2227 | } |
| 2228 | |
| 2229 | // seedJSON renders a manifest's seed argv as the wire text SetSeed and a |
| 2230 | // minted route's Seed field carry: "" for nil/empty (no seed), else the |
| 2231 | // JSON array of the argv strings. |
| 2232 | func seedJSON(m *Manifest) string { |
| 2233 | if m == nil || len(m.Seed) == 0 { |
| 2234 | return "" |
| 2235 | } |
| 2236 | b, err := json.Marshal(m.Seed) // []string always marshals; err is unreachable |
| 2237 | if err != nil { |
| 2238 | return "" |
| 2239 | } |
| 2240 | return string(b) |
| 2241 | } |
| 2242 | |
amadan