| 1 | package provision |
| 2 | |
| 3 | // ReconcileSidecars turns bucket SIDECAR records into registry routes, for |
| 4 | // THIS agent's own region, pool set and inbound posture. Spec: |
| 5 | // docs/superpowers/specs/2026-08-17-sidecar-members-design.md §4, §8. |
| 6 | // |
| 7 | // It is Reconcile's twin, one record family over, and it is deliberately a |
| 8 | // separate pass rather than a widening of that one: the two families answer |
| 9 | // different questions about a box (can it SERVE vs may it RUN), and a single |
| 10 | // loop would have had to carry a Kind branch through every phase — including |
| 11 | // the removal phase, where a missed branch deletes production rows. |
| 12 | // |
| 13 | // Which makes the collision this file exists to close worth stating plainly. |
| 14 | // Reconcile's removal phase is scoped by Provisioned, region, pool and |
| 15 | // absence-from-records. A sidecar row is Provisioned too (that flag is what |
| 16 | // grants a pass removal authority over a row), and it can never appear in the |
| 17 | // INSTANCE record set — so, unguarded, that pass deletes every sidecar row on |
| 18 | // every tick: minted by this pass, destroyed by that one, forever, with no |
| 19 | // error anywhere and nothing in the logs that reads as a failure. Each pass's |
| 20 | // removal authority is therefore scoped to its OWN Kind, asserted from both |
| 21 | // sides (TestInstanceReconcileNeverRemovesSidecarRows, |
| 22 | // TestSidecarReconcileRemovesOnlyItsOwnKind). |
| 23 | // |
| 24 | // ONE EXCEPTION, and it is deliberate: domains.Reconcile drops EVERY route of |
| 25 | // a trashed app (internal/domains/reconcile.go, the trashed-apps loop) with no |
| 26 | // Kind filter at all, sidecar rows included. So removal authority over a |
| 27 | // sidecar row is held by TWO passes, not one. That is not a leak — it is the |
| 28 | // only path by which a trashed app's sidecars stop running, it carries its own |
| 29 | // production-flag guard, and it is precisely what makes THIS pass's |
| 30 | // decline-but-keep response to a trashed app correct rather than a way for |
| 31 | // rows to outlive their app. Anyone tightening the rule stated above should |
| 32 | // know where its exception lives before assuming a sidecar row can only be |
| 33 | // deleted here. |
| 34 | |
| 35 | import ( |
| 36 | "context" |
| 37 | "encoding/json" |
| 38 | "errors" |
| 39 | "fmt" |
| 40 | "log" |
| 41 | "os" |
| 42 | "path/filepath" |
| 43 | "time" |
| 44 | |
| 45 | "github.com/carlosframework/platform/internal/registry" |
| 46 | "github.com/carlosframework/platform/internal/release" |
| 47 | ) |
| 48 | |
| 49 | // SidecarKind is the route Kind every sidecar row carries — the whole basis |
| 50 | // of each pass's removal authority, and the value the edge refuses outright. |
| 51 | // A literal in two passes' removal conditions is a typo away from a fleet |
| 52 | // deletion, so both read this constant. |
| 53 | const SidecarKind = "sidecar" |
| 54 | |
| 55 | // sidecarCoSpawnSuffix names the LEGACY, file-staged co-spawn: a sidecar |
| 56 | // attached to a host by dropping <DataDir>/<host>-sidecar.env |
| 57 | // (internal/activator/backend_exec.go). It is legacy but LIVE — Eleven runs |
| 58 | // on it today — so a host that already has one is refused a native sidecar |
| 59 | // row. One host, one mechanism. |
| 60 | const sidecarCoSpawnSuffix = "-sidecar.env" |
| 61 | |
| 62 | // sidecarArgsJSON renders a manifest's argv as the wire text registry.Route's |
| 63 | // Args column carries: "" for nil/empty (no argv), else the JSON array. Same |
| 64 | // shape and same reason as seedJSON — json.Marshal of a nil slice is "null", |
| 65 | // which is neither empty nor a usable argv. |
| 66 | func sidecarArgsJSON(m SidecarManifest) string { |
| 67 | if len(m.Args) == 0 { |
| 68 | return "" |
| 69 | } |
| 70 | b, err := json.Marshal(m.Args) // []string always marshals; err is unreachable |
| 71 | if err != nil { |
| 72 | return "" |
| 73 | } |
| 74 | return string(b) |
| 75 | } |
| 76 | |
| 77 | // sidecarPlacement resolves the placement policy a record is placed under |
| 78 | // (design §8): MOST RESTRICTIVE WINS. Either voice asking for no-inbound gets |
| 79 | // no-inbound, and only when both are silent (or both say "any") does the |
| 80 | // workload place anywhere. |
| 81 | // |
| 82 | // The record can therefore still NARROW an app-wide "any" — one bot that must |
| 83 | // sit somewhere with no inbound path, under an app with no such rule — which |
| 84 | // is the direction that reads as a requirement. What it can no longer do is |
| 85 | // WIDEN: a record stating "any" under a no-inbound manifest used to place that |
| 86 | // bot on inbound boxes, turning app-wide policy into a suggestion. The console |
| 87 | // door already refuses to write such a record (it never accepts a placement |
| 88 | // field at all, exactly so the manifest stays authoritative); a direct bucket |
| 89 | // write, an older writer, or anything else holding the prefix's keys does not |
| 90 | // go through that door, and both doors hold the invariant rather than trusting |
| 91 | // each other. |
| 92 | func sidecarPlacement(rec SidecarRecord, m SidecarManifest) string { |
| 93 | if rec.Placement == PlacementNoInbound || m.PlacementOrDefault() == PlacementNoInbound { |
| 94 | return PlacementNoInbound |
| 95 | } |
| 96 | if rec.Placement != "" { |
| 97 | return rec.Placement |
| 98 | } |
| 99 | return m.PlacementOrDefault() |
| 100 | } |
| 101 | |
| 102 | // placeable reports whether this box satisfies a record's placement — the |
| 103 | // BOX half of design §8's two independent halves. It reads as a SCOPE test, |
| 104 | // beside region and pool, and is used as one: a record this box may not run |
| 105 | // is not this box's business at all, so it never enters the record set and |
| 106 | // the removal phase takes any row it left behind. That is how a placement |
| 107 | // change takes effect — the same way a region change does. |
| 108 | func placeable(rec SidecarRecord, m SidecarManifest, cfg ReconcileConfig) bool { |
| 109 | return sidecarPlacement(rec, m) != PlacementNoInbound || cfg.Inbound == InboundNone |
| 110 | } |
| 111 | |
| 112 | // sidecarRouteFor is the one answer to "what route does this sidecar record |
| 113 | // describe" — the mint builds from it and the converge branch compares |
| 114 | // against it, so a shape this pass would mint and a shape it calls converged |
| 115 | // can never drift apart (routeFor's rule, same reason). |
| 116 | // |
| 117 | // Socket and Addr are deliberately left EMPTY, and that is the whole |
| 118 | // reachability story: a sidecar binds nothing and nothing dials it. The edge |
| 119 | // refuses Kind == "sidecar" outright (design §6.2) rather than relying on |
| 120 | // this omission, but the row must not carry an address for any future arm to |
| 121 | // find either. |
| 122 | // |
| 123 | // DBPath, by contrast, IS derived: a sidecar gets a database like any other |
| 124 | // tenant, and a hibernating route whose row carried no path would be swept up |
| 125 | // by ops.FleetDBs' dataDir glob and replicated box-wide as if it never slept |
| 126 | // (routeFor's comment records what that costs). |
| 127 | // |
| 128 | // The ceilings are materialized onto the row from the manifest precisely so |
| 129 | // the activator's sweeper never needs a bucket read to decide a worker has |
| 130 | // outrun its limits. |
| 131 | func sidecarRouteFor(rec SidecarRecord, m SidecarManifest, dataDir string) registry.Route { |
| 132 | return registry.Route{ |
| 133 | Host: rec.Host, Slug: rec.Name, Kind: SidecarKind, |
| 134 | App: rec.App, Account: rec.Account, Pool: rec.Pool, Region: rec.Region, |
| 135 | // The RECORD's channel, never the manifest's: a record that reached |
| 136 | // this pass without one was refused above (a route with no channel |
| 137 | // resolves no release, so it has no build to exec), so there is |
| 138 | // nothing here for a manifest-side fallback to rescue. The manifest's |
| 139 | // Channel is the console door's default at CREATE time. |
| 140 | Channel: rec.Channel, Environment: rec.Environment, |
| 141 | DBPath: DefaultDBPath(dataDir, rec.Host), |
| 142 | Hibernate: m.HibernateOrDefault(), |
| 143 | Args: sidecarArgsJSON(m), |
| 144 | IdleSeconds: int(m.IdleOrDefault() / time.Second), |
| 145 | MaxRunSeconds: int(m.MaxRunOrDefault() / time.Second), |
| 146 | Provisioned: true, |
| 147 | Status: "ready", |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | // coSpawnStaged reports whether host already has a legacy file-staged |
| 152 | // co-spawn in dataDir. A read error other than "not there" answers TRUE: not |
| 153 | // minting is the reversible direction, and a host whose staging state cannot |
| 154 | // be established is exactly the host a second mechanism must not be attached |
| 155 | // to. |
| 156 | func coSpawnStaged(dataDir, host string) bool { |
| 157 | _, err := os.Stat(filepath.Join(dataDir, host+sidecarCoSpawnSuffix)) |
| 158 | return err == nil || !errors.Is(err, os.ErrNotExist) |
| 159 | } |
| 160 | |
| 161 | // ReconcileSidecars materializes bucket sidecar records into registry routes |
| 162 | // for this box's region, pools and inbound posture: mint on appearance, |
| 163 | // converge the manifest-owned columns, remove on record deletion — removal |
| 164 | // only for Kind "sidecar" routes this pass (or an earlier one running the |
| 165 | // same scope) minted, marked by Provisioned. |
| 166 | // |
| 167 | // Per-item failures log and skip. A failed account-manifest or record read |
| 168 | // additionally disables the removal phase for the WHOLE pass: a transient |
| 169 | // read failure must never mass-retire a tenant's fleet. Minting is additive |
| 170 | // and always safe, so it proceeds regardless. |
| 171 | // |
| 172 | // SCOPE — region, pool and PLACEMENT (design §8) — decides what enters the |
| 173 | // record set at all, and so decides removal: a record this box may not run is |
| 174 | // not this box's business, and any row it left behind is retired by the |
| 175 | // ordinary removal phase. That is how a placement change takes effect, exactly |
| 176 | // as a region change does through the instance pass; a member who asks that a |
| 177 | // bot only ever run where there is no inbound path must not find it still |
| 178 | // running on an inbound box after flipping the manifest. Evacuation is |
| 179 | // governed by the same fail-safe as every other removal: if any read failed |
| 180 | // this pass, nothing is evacuated. |
| 181 | // |
| 182 | // Two states instead decline to MINT while leaving the record in the record |
| 183 | // set — so removal never fires for them — because neither is a statement |
| 184 | // about where the workload belongs: |
| 185 | // |
| 186 | // - the app is in the trash (domains.Reconcile drops its routes on its own |
| 187 | // authority; minting here would re-raise them every tick), |
| 188 | // - the app's sidecars manifest is absent or unreadable (no argv to run, |
| 189 | // and no ceilings to stamp). |
| 190 | func ReconcileSidecars(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig) error { |
| 191 | if cfg.Now == nil { |
| 192 | cfg.Now = time.Now |
| 193 | } |
| 194 | |
| 195 | // scan is the record-collection phase over st: the placeable in-scope |
| 196 | // records by host, the hosts whose record is there but cannot be used |
| 197 | // (protect), the per-app sidecars manifest lookup the pass goes on using, |
| 198 | // and whether anything failed to read (readFailed, which disables removal |
| 199 | // for the whole pass; a manifest read that fails later in the pass sets |
| 200 | // it too). The pass scans s; a pass over a record view snapshot that has |
| 201 | // a removal to decide scans the bucket too (freshStore), so both judge |
| 202 | // every record by exactly these rules, placement against the record and |
| 203 | // the app's sidecars manifest included (S3 change-index design §4.4). |
| 204 | type sidecarScan struct { |
| 205 | recs map[string]SidecarRecord |
| 206 | protect map[string]bool |
| 207 | readFailed bool |
| 208 | manifestFor func(acct, app string) (SidecarManifest, bool) |
| 209 | } |
| 210 | scan := func(st release.Store) (*sidecarScan, error) { |
| 211 | sc := &sidecarScan{} |
| 212 | known, readFailed, err := knownAccounts(ctx, st) |
| 213 | if err != nil { |
| 214 | return nil, err |
| 215 | } |
| 216 | sc.readFailed = readFailed |
| 217 | |
| 218 | // --- manifest cache, per (acct, app), lazily ------------------------------ |
| 219 | // known=false covers BOTH "no manifest" and "the read failed", and in both |
| 220 | // cases this pass declines to mint or converge: every manifest-owned |
| 221 | // column (argv, the ceilings, hibernation) would otherwise be stamped from |
| 222 | // a zero value — an empty argv over a working one, on a blip. This is |
| 223 | // seedFor's seedKnown discipline, applied to a family where the manifest |
| 224 | // carries the RUN instruction rather than a seed. |
| 225 | // |
| 226 | // Declared BEFORE the record scan because placement is a scope axis and |
| 227 | // placement can be stated app-wide: deciding whether a record is in scope |
| 228 | // needs the manifest in hand. An unknown manifest resolves placement from |
| 229 | // the record alone (sidecarPlacement over a zero manifest) — and a manifest |
| 230 | // whose read FAILED has already disabled the removal phase, so an |
| 231 | // unknowable placement can never evacuate anything. |
| 232 | manCache := map[acctApp]SidecarManifest{} |
| 233 | manKnown := map[acctApp]bool{} |
| 234 | manSeen := map[acctApp]bool{} |
| 235 | manifestFor := func(acct, app string) (SidecarManifest, bool) { |
| 236 | k := acctApp{acct, app} |
| 237 | if manSeen[k] { |
| 238 | return manCache[k], manKnown[k] |
| 239 | } |
| 240 | manSeen[k] = true |
| 241 | m, err := GetSidecarManifest(ctx, st, acct, app) |
| 242 | if err != nil { |
| 243 | if errors.Is(err, os.ErrNotExist) { |
| 244 | // Absence IS the answer (design §3.1): sidecars are not |
| 245 | // enabled for this app. Not a read failure — removal stays |
| 246 | // armed for the pass — but the records themselves stay in |
| 247 | // recs, so nothing of theirs is retired on it. |
| 248 | log.Printf("provision: %s/%s has sidecar records but no sidecars manifest — not minting", acct, app) |
| 249 | return SidecarManifest{}, false |
| 250 | } |
| 251 | log.Printf("provision: sidecar manifest %s/%s: read failed: %v", acct, app, err) |
| 252 | sc.readFailed = true |
| 253 | return SidecarManifest{}, false |
| 254 | } |
| 255 | manCache[k], manKnown[k] = *m, true |
| 256 | return manCache[k], manKnown[k] |
| 257 | } |
| 258 | |
| 259 | // --- collect in-scope records -------------------------------------------- |
| 260 | keys, partial, err := listAppSubtree(ctx, st, known, "sidecars/") |
| 261 | if err != nil { |
| 262 | return nil, fmt.Errorf("provision: list apps: %w", err) |
| 263 | } |
| 264 | if partial { |
| 265 | sc.readFailed = true // unseen records must not read as retired |
| 266 | } |
| 267 | recs := map[string]SidecarRecord{} // host -> record, in-scope only |
| 268 | // protect holds hosts whose record KEY parsed but whose body could not be |
| 269 | // trusted — unparseable JSON, a body disagreeing with its own path, or a |
| 270 | // shape this binary refuses. Such a host must NEVER read as "the bucket |
| 271 | // retired it": a non-additive record encoding change reaching an older |
| 272 | // binary would make EVERY record fail exactly this way, and one tick would |
| 273 | // then delete every sidecar in scope. Narrower than readFailed (which |
| 274 | // vetoes removal for the whole pass): only the unusable host is held back. |
| 275 | protect := map[string]bool{} |
| 276 | var recordKeys []string |
| 277 | for _, key := range keys { |
| 278 | if _, _, _, ok := ParseSidecarKey(key, known); ok { |
| 279 | recordKeys = append(recordKeys, key) |
| 280 | } |
| 281 | } |
| 282 | for _, part := range chunks(recordKeys) { |
| 283 | bodies := getEach(ctx, st, part) |
| 284 | for _, key := range part { |
| 285 | acct, app, name, _ := ParseSidecarKey(key, known) |
| 286 | // Derived from the KEY, never from the body: the body is what is |
| 287 | // under suspicion in every branch below. |
| 288 | host := SidecarHost(name, app, acct) |
| 289 | raw, err := bodies[key].body, bodies[key].err |
| 290 | if err != nil { |
| 291 | log.Printf("provision: skipping sidecar record %s: read failed: %v", key, err) |
| 292 | // Its absence from recs below must not read as "the bucket retired |
| 293 | // it" — a transient Get failure disables removal for the whole |
| 294 | // pass, same as an unreadable account manifest. |
| 295 | sc.readFailed = true |
| 296 | continue |
| 297 | } |
| 298 | var rec SidecarRecord |
| 299 | if err := json.Unmarshal(raw, &rec); err != nil { |
| 300 | log.Printf("provision: skipping sidecar record %s: malformed: %v", key, err) |
| 301 | protect[host] = true |
| 302 | continue |
| 303 | } |
| 304 | if rec.Name != name || rec.Account != acct || rec.App != app { |
| 305 | log.Printf("provision: skipping sidecar record %s: disagrees with its own path (name=%q account=%q app=%q)", |
| 306 | key, rec.Name, rec.Account, rec.App) |
| 307 | protect[host] = true |
| 308 | continue |
| 309 | } |
| 310 | // ValidateSidecarShape is the console create door's policy check — but |
| 311 | // a door is not a guarantee about what is already in the bucket: |
| 312 | // anything with bucket write access, and any record written by a |
| 313 | // different (older, or future) writer, arrives here unexamined. Both |
| 314 | // doors hold the invariant, neither trusts the other. Protected like a |
| 315 | // corrupted body, and for the same reason: a record that cannot be |
| 316 | // used is not a record that was retired. |
| 317 | if err := ValidateSidecarShape(rec); err != nil { |
| 318 | log.Printf("provision: skipping sidecar record %s: invalid shape: %v", key, err) |
| 319 | protect[host] = true |
| 320 | continue |
| 321 | } |
| 322 | // The host is DERIVED from the identity, never taken on trust: it |
| 323 | // becomes a database path and a registry primary key, and a record |
| 324 | // free to state its own host could name any row on the box. |
| 325 | if rec.Host != host { |
| 326 | log.Printf("provision: skipping sidecar record %s: host %q is not %q", key, rec.Host, host) |
| 327 | protect[host] = true |
| 328 | continue |
| 329 | } |
| 330 | // ValidateSidecarShape deliberately does not inspect Channel (it |
| 331 | // validates shape fields, and Channel is not one), but a channel-less |
| 332 | // record must not become — or converge — a route pinned to no channel: |
| 333 | // such a row resolves no release, so it has no build to exec, and the |
| 334 | // worker boot pass would try to run it against a symlink that does not |
| 335 | // exist. The console stamps a channel at create, so a record without |
| 336 | // one is malformed. Protected like a corrupted body: the record is |
| 337 | // there, it just cannot be used. |
| 338 | if rec.Channel == "" { |
| 339 | log.Printf("provision: skipping sidecar record %s: empty channel", key) |
| 340 | protect[host] = true |
| 341 | continue |
| 342 | } |
| 343 | if rec.Region != cfg.Region || !poolServed(cfg.Pools, rec.Pool) { |
| 344 | continue // another agent's jurisdiction, not this pass's business |
| 345 | } |
| 346 | // Placement is the third scope axis (design §8), and it is applied |
| 347 | // HERE, beside region and pool, precisely so it behaves like them: a |
| 348 | // record this box may not run never enters recs, and the removal phase |
| 349 | // below therefore evacuates any row it left behind. "none" is a |
| 350 | // property of the BOX's network; "no-inbound" is a requirement of the |
| 351 | // WORKLOAD — the point of the two halves being independent is that a |
| 352 | // member gets a mechanical guarantee rather than an operator's promise, |
| 353 | // which a row left running on an inbound box would not be. |
| 354 | if m, _ := manifestFor(acct, app); !placeable(rec, m, cfg) { |
| 355 | log.Printf("provision: sidecar %s is not placeable here: it asks for %q placement and this box accepts inbound work (CARLOS_INBOUND=%q)", |
| 356 | rec.Host, sidecarPlacement(rec, m), cfg.Inbound) |
| 357 | continue |
| 358 | } |
| 359 | // Two records claiming one host is a collision this pass cannot |
| 360 | // arbitrate silently: whichever this listing reaches FIRST wins, and |
| 361 | // the loser is skipped loudly rather than overwriting the winner in |
| 362 | // the map with no trace of the conflict. |
| 363 | if prev, dup := recs[rec.Host]; dup { |
| 364 | log.Printf("provision: duplicate sidecar host %s: keeping %s, skipping %s", rec.Host, |
| 365 | SidecarRecordKey(prev.Account, prev.App, prev.Name), key) |
| 366 | continue |
| 367 | } |
| 368 | recs[rec.Host] = rec |
| 369 | } |
| 370 | } |
| 371 | sc.recs, sc.protect, sc.manifestFor = recs, protect, manifestFor |
| 372 | return sc, nil |
| 373 | } |
| 374 | sc, err := scan(s) |
| 375 | if err != nil { |
| 376 | return err |
| 377 | } |
| 378 | recs, protect, manifestFor := sc.recs, sc.protect, sc.manifestFor |
| 379 | |
| 380 | trashed := trashGate(ctx, s, "sidecar") |
| 381 | |
| 382 | // --- mint / converge ------------------------------------------------------- |
| 383 | routes, err := reg.List() |
| 384 | if err != nil { |
| 385 | return fmt.Errorf("provision: list routes: %w", err) |
| 386 | } |
| 387 | byHost := map[string]registry.Route{} |
| 388 | for _, rt := range routes { |
| 389 | byHost[rt.Host] = rt |
| 390 | } |
| 391 | |
| 392 | for _, rec := range recs { |
| 393 | if trashed(rec.Account, rec.App) { |
| 394 | continue |
| 395 | } |
| 396 | m, mKnown := manifestFor(rec.Account, rec.App) |
| 397 | if !mKnown { |
| 398 | continue |
| 399 | } |
| 400 | existing, ok := byHost[rec.Host] |
| 401 | |
| 402 | // Refusals that need no claim are decided BEFORE cfg.claimInstall: the |
| 403 | // hook activates the host guard and writes inventory, effects that a |
| 404 | // record this pass will not act on must never cause. Only work that |
| 405 | // is going to touch the registry reaches it. |
| 406 | var prior *registry.Route |
| 407 | nr := sidecarRouteFor(rec, m, cfg.DataDir) |
| 408 | if ok { |
| 409 | prior = &existing |
| 410 | } else { |
| 411 | // One host, one mechanism: a host already carrying the legacy |
| 412 | // file-staged co-spawn must not ALSO get a native sidecar row, or |
| 413 | // the same workload runs twice under two owners that know nothing |
| 414 | // of each other. |
| 415 | // |
| 416 | // This is a MINT-TIME check and nothing more. A -sidecar.env |
| 417 | // staged AFTER this row exists is not detected, and that host then |
| 418 | // carries both mechanisms with no complaint from this pass. It is |
| 419 | // left that way deliberately: the reverse — a converge-time |
| 420 | // refusal — would mean a file dropped by hand could stop an |
| 421 | // already-provisioned sidecar converging, which hands anything |
| 422 | // that can write the data dir a way to freeze a member's row. The |
| 423 | // direction that matters is the one guarded: the file came first, |
| 424 | // so the file keeps the host. |
| 425 | if coSpawnStaged(cfg.DataDir, rec.Host) { |
| 426 | log.Printf("provision: REFUSING to mint sidecar %s: %s%s is staged on this box — "+ |
| 427 | "the legacy file co-spawn and a native sidecar row are two mechanisms for one host; "+ |
| 428 | "remove the file to migrate this host onto sidecar records", |
| 429 | rec.Host, rec.Host, sidecarCoSpawnSuffix) |
| 430 | continue |
| 431 | } |
| 432 | // Same admission rule as the instance pass (amadan #213): a |
| 433 | // sidecar that would start without its sealed config is not |
| 434 | // minted here. See ReconcileConfig.ConfigGate. |
| 435 | if cfg.ConfigGate != nil { |
| 436 | if err := cfg.ConfigGate(ctx, nr); err != nil { |
| 437 | continue |
| 438 | } |
| 439 | } |
| 440 | } |
| 441 | // A row at this host that this pass does not own is left alone before |
| 442 | // any claim work, so the hook never activates the guard or writes |
| 443 | // inventory for it. (Flag off, the same decision is made inside the |
| 444 | // write below.) |
| 445 | if ok && cfg.ClaimEnabled && !(existing.Kind == SidecarKind && existing.Provisioned && |
| 446 | existing.App == rec.App && existing.Account == rec.Account) { |
| 447 | log.Printf("provision: %s exists but is not a sidecar row this pass owns (kind %q provisioned %v app %q acct %q) — leaving it alone", |
| 448 | rec.Host, existing.Kind, existing.Provisioned, existing.App, existing.Account) |
| 449 | continue |
| 450 | } |
| 451 | selected := ClaimRecord{Host: rec.Host, Key: SidecarRecordKey(rec.Account, rec.App, rec.Name), Family: "sidecar", Account: rec.Account, App: rec.App, CreatedAt: rec.CreatedAt, Value: rec, Route: nr, DryRun: cfg.DryRun} |
| 452 | if err := cfg.claimInstall(ctx, selected, prior, func(owner registry.ClaimOwnership) error { |
| 453 | switch { |
| 454 | case !ok: |
| 455 | nr.ClaimOwnership = owner |
| 456 | if cfg.DryRun { |
| 457 | log.Printf("provision: dry-run: would mint sidecar route %s (app %q acct %q channel %q args %s)", |
| 458 | rec.Host, rec.App, rec.Account, nr.Channel, nr.Args) |
| 459 | return nil |
| 460 | } |
| 461 | if err := reg.Upsert(nr); err != nil { |
| 462 | log.Printf("provision: mint sidecar %s: %v", rec.Host, err) |
| 463 | return nil |
| 464 | } |
| 465 | log.Printf("provision: minted sidecar route %s (app %q acct %q args %s)", rec.Host, rec.App, rec.Account, nr.Args) |
| 466 | case existing.Kind == SidecarKind && existing.Provisioned && |
| 467 | existing.App == rec.App && existing.Account == rec.Account: |
| 468 | convergeSidecar(reg, existing, rec, m, cfg) |
| 469 | default: |
| 470 | // A row exists at this host that this pass does not own. There is |
| 471 | // deliberately NO adoption path here, unlike the instance pass: |
| 472 | // the .sidecar.internal suffix is reserved and refused at every |
| 473 | // door a name could enter through, so no hand-added row to |
| 474 | // backfill can legitimately exist — and adoption is an authority |
| 475 | // TRANSFER (it is what grants the removal phase power over a row), |
| 476 | // never a shape coincidence. |
| 477 | log.Printf("provision: %s exists but is not a sidecar row this pass owns (kind %q provisioned %v app %q acct %q) — leaving it alone", |
| 478 | rec.Host, existing.Kind, existing.Provisioned, existing.App, existing.Account) |
| 479 | } |
| 480 | return nil |
| 481 | }); err != nil { |
| 482 | log.Printf("provision: claim install %s: %v", rec.Host, err) |
| 483 | } |
| 484 | } |
| 485 | |
| 486 | // --- removal, only when nothing failed to read this pass -------------------- |
| 487 | // Four conditions, each load-bearing: this pass's OWN Kind (the guard at |
| 488 | // the top of this file — the instance pass carries its mirror), Provisioned |
| 489 | // (the flag, not the app and not the channel, is what grants removal |
| 490 | // authority; a hand-added row never has it), scope (this box's own |
| 491 | // region/pool jurisdiction), and absence from the record set this pass just |
| 492 | // read (the bucket's own decision to retire the sidecar, not a guess). |
| 493 | // |
| 494 | // retire runs that phase over one set of records and protected hosts. |
| 495 | // With stopAtFirst it removes nothing and reports whether any row would |
| 496 | // go: a record view snapshot's absence is not the bucket's, so a pass |
| 497 | // over a snapshot that would remove anything decides every removal again |
| 498 | // from the bucket's own walk, below (S3 change-index design §4.4). |
| 499 | retire := func(recs map[string]SidecarRecord, protect map[string]bool, stopAtFirst bool) (wouldRemove bool) { |
| 500 | for _, rt := range routes { |
| 501 | if rt.Kind != SidecarKind || !rt.Provisioned { |
| 502 | continue |
| 503 | } |
| 504 | if rt.Region != cfg.Region || !poolServed(cfg.Pools, rt.Pool) { |
| 505 | continue |
| 506 | } |
| 507 | if _, ok := recs[rt.Host]; ok { |
| 508 | continue |
| 509 | } |
| 510 | if protect[rt.Host] { |
| 511 | log.Printf("provision: not removing sidecar %s: its record exists but could not be read cleanly this pass", rt.Host) |
| 512 | continue |
| 513 | } |
| 514 | if stopAtFirst { |
| 515 | return true |
| 516 | } |
| 517 | if cfg.DryRun { |
| 518 | log.Printf("provision: dry-run: would remove retired sidecar route %s (record gone)", rt.Host) |
| 519 | continue |
| 520 | } |
| 521 | if err := cfg.claimRetire(ctx, reg, rt); err != nil { |
| 522 | log.Printf("provision: remove retired sidecar route %s: %v", rt.Host, err) |
| 523 | continue |
| 524 | } |
| 525 | log.Printf("provision: removed retired sidecar route %s (record gone)", rt.Host) |
| 526 | } |
| 527 | return false |
| 528 | } |
| 529 | if sc.readFailed || !retire(recs, protect, onSnapshot(s)) { |
| 530 | return nil |
| 531 | } |
| 532 | // A snapshot pass with a removal to make: the record walk again, over |
| 533 | // the bucket, and the removal phase over what it read, by the walk's own |
| 534 | // rules (unusable records protected before scope, hosts matched across |
| 535 | // every key, placement from the bucket's record and sidecars manifest |
| 536 | // against this box's inbound). Anything it cannot read keeps every row |
| 537 | // this pass, as in the walk. |
| 538 | lsc, err := scan(freshStore(s)) |
| 539 | if err != nil || lsc.readFailed { |
| 540 | log.Printf("provision: not removing any sidecar route this pass: the record view lacks a sidecar this box would run, and the bucket's own walk could not be read cleanly%s", walkErrSuffix(err)) |
| 541 | return nil |
| 542 | } |
| 543 | retire(lsc.recs, lsc.protect, false) |
| 544 | return nil |
| 545 | } |
| 546 | |
| 547 | // convergeSidecar applies the columns the record and the manifest are |
| 548 | // authoritative for onto a row this pass owns. Narrow UPDATEs throughout, |
| 549 | // never a re-Upsert: a read-modify-write racing the activator's own writes |
| 550 | // (a worker's reported version, its next-due stamp) would clobber them. |
| 551 | // |
| 552 | // The split of authority is the same one the design states: the RECORD owns |
| 553 | // channel and environment (what this sidecar runs and which config it gets — |
| 554 | // the #130 discipline), the MANIFEST owns argv, the ceilings and hibernation |
| 555 | // (app-wide policy). Both are re-stamped every pass, exactly as an instance's |
| 556 | // seed is, because truth lives in the bucket and a box-local edit to one of |
| 557 | // these columns is drift, not a decision. |
| 558 | // |
| 559 | // Every write logs on success as well as on failure: these are the writes |
| 560 | // that REVERT an operator's own `carlos route` edit, and a row that moved with |
| 561 | // nothing saying so is the failure class behind #130 seen from the outside. |
| 562 | func convergeSidecar(reg *registry.Registry, existing registry.Route, rec SidecarRecord, m SidecarManifest, cfg ReconcileConfig) { |
| 563 | want := sidecarRouteFor(rec, m, cfg.DataDir) |
| 564 | |
| 565 | if want.Channel != "" && existing.Channel != want.Channel { |
| 566 | if cfg.DryRun { |
| 567 | log.Printf("provision: dry-run: would converge sidecar %s channel %s -> %s", rec.Host, existing.Channel, want.Channel) |
| 568 | } else if err := reg.SetChannel(rec.Host, want.Channel); err != nil { |
| 569 | log.Printf("provision: converge channel on sidecar %s: %v", rec.Host, err) |
| 570 | } else { |
| 571 | log.Printf("provision: converged sidecar %s channel %s -> %s", rec.Host, existing.Channel, want.Channel) |
| 572 | } |
| 573 | } |
| 574 | if existing.Environment != want.Environment { |
| 575 | if cfg.DryRun { |
| 576 | log.Printf("provision: dry-run: would converge sidecar %s environment %q -> %q", rec.Host, existing.Environment, want.Environment) |
| 577 | } else if err := reg.SetEnvironment(rec.Host, want.Environment); err != nil { |
| 578 | log.Printf("provision: converge environment on sidecar %s: %v", rec.Host, err) |
| 579 | } else { |
| 580 | log.Printf("provision: converged sidecar %s environment %q -> %q", rec.Host, existing.Environment, want.Environment) |
| 581 | } |
| 582 | } |
| 583 | if existing.Hibernate != want.Hibernate { |
| 584 | if cfg.DryRun { |
| 585 | log.Printf("provision: dry-run: would converge sidecar %s hibernate %v -> %v", rec.Host, existing.Hibernate, want.Hibernate) |
| 586 | } else if err := reg.SetHibernate(rec.Host, want.Hibernate); err != nil { |
| 587 | log.Printf("provision: converge hibernate on sidecar %s: %v", rec.Host, err) |
| 588 | } else { |
| 589 | log.Printf("provision: converged sidecar %s hibernate %v -> %v", rec.Host, existing.Hibernate, want.Hibernate) |
| 590 | } |
| 591 | } |
| 592 | if existing.Args != want.Args { |
| 593 | if cfg.DryRun { |
| 594 | log.Printf("provision: dry-run: would converge sidecar %s args %s -> %s", rec.Host, existing.Args, want.Args) |
| 595 | } else if err := reg.SetArgs(rec.Host, want.Args); err != nil { |
| 596 | log.Printf("provision: converge args on sidecar %s: %v", rec.Host, err) |
| 597 | } else { |
| 598 | log.Printf("provision: converged sidecar %s args %s -> %s", rec.Host, existing.Args, want.Args) |
| 599 | } |
| 600 | } |
| 601 | // One UPDATE for both ceilings, because they are one policy: the idle |
| 602 | // window and the run ceiling together decide when the sweeper stops a |
| 603 | // worker, and a row carrying the new idle under the old maximum (or the |
| 604 | // reverse) is a combination the manifest never stated. |
| 605 | if existing.IdleSeconds != want.IdleSeconds || existing.MaxRunSeconds != want.MaxRunSeconds { |
| 606 | if cfg.DryRun { |
| 607 | log.Printf("provision: dry-run: would converge sidecar %s ceilings %d/%d -> %d/%d", rec.Host, |
| 608 | existing.IdleSeconds, existing.MaxRunSeconds, want.IdleSeconds, want.MaxRunSeconds) |
| 609 | } else if err := reg.SetWorkerLimits(rec.Host, want.IdleSeconds, want.MaxRunSeconds); err != nil { |
| 610 | log.Printf("provision: converge ceilings on sidecar %s: %v", rec.Host, err) |
| 611 | } else { |
| 612 | log.Printf("provision: converged sidecar %s ceilings %d/%d -> %d/%d", rec.Host, |
| 613 | existing.IdleSeconds, existing.MaxRunSeconds, want.IdleSeconds, want.MaxRunSeconds) |
| 614 | } |
| 615 | } |
| 616 | // The columns nothing converges in place: the database path (moving it |
| 617 | // under a running process would point it at a second copy of its own |
| 618 | // data) and the two reachability columns a sidecar row must never carry. |
| 619 | // Reported, never rewritten — `carlos remove` purges a route's capability |
| 620 | // grants, so it is a last resort, not a routine fix. |
| 621 | if existing.DBPath != want.DBPath || existing.Socket != "" || existing.Addr != "" { |
| 622 | log.Printf("provision: sidecar %s: row disagrees with the shape this record describes "+ |
| 623 | "(db %q/%q socket %q addr %q) — not converged in place; a sidecar binds nothing, "+ |
| 624 | "so a socket or addr on this row is a row that was not minted by this pass", |
| 625 | rec.Host, existing.DBPath, want.DBPath, existing.Socket, existing.Addr) |
| 626 | } |
| 627 | } |
| 628 | |
amadan