amadan

carlos / platform Public

Scope each reconcile pass's removal authority to its own kind

Paul Campbell pushed by paul@keymail.dev 509db85bedea9f1d992cb90a15a278449bf19f8c parent a22c66e
5 files changed, +1016 −23
  • internal/host/agent.go +20 −4
  • internal/host/agent_test.go +38 −0
  • internal/provision/reconcile.go +61 −19
  • internal/provision/sidecar_reconcile.go +462 −0
  • internal/provision/sidecar_reconcile_test.go +435 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index a3b837067..4cfbaf395 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -694,14 +694,30 @@ func syncExecSerialized(ctx context.Context, reg *registry.Registry, store relea
// no Reconcile and is not this mutex's problem.
var reconcileMu sync.Mutex
-// reconcileSerialized is the one way any agent loop runs provision.Reconcile
-// — the mutex lives in a shared wrapper, not at the call sites, so a future
-// third caller cannot forget it. Pinned by
+// reconcileSerialized is the one way any agent loop runs the record-family
+// reconcile passes — the mutex lives in a shared wrapper, not at the call
+// sites, so a future third caller cannot forget it. Pinned by
// TestReconcileSerializedHoldsTheMutex.
+//
+// BOTH families run here, under ONE hold of the mutex: the instance pass and
+// the sidecar pass. Each pass's removal phase is scoped to its own route Kind
+// (provision.ReconcileSidecars' doc comment), so they cannot delete each
+// other's rows — but they read and write the same registry, and the
+// overlapping-pass hazard the mutex exists for (a route minted by a newer
+// pass looking exactly like a retired record to an older one) applies within
+// each family just as much. Running them as one serialized unit also means a
+// caller cannot converge instances while leaving sidecars a tick behind.
+//
+// Errors are JOINED rather than short-circuited: a sidecar-side failure must
+// not hide an instance-side one, and neither must stop the other running —
+// each pass is independently safe to run and independently valuable.
func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig) error {
reconcileMu.Lock()
defer reconcileMu.Unlock()
- return provision.Reconcile(ctx, s, reg, cfg)
+ return errors.Join(
+ provision.Reconcile(ctx, s, reg, cfg),
+ provision.ReconcileSidecars(ctx, s, reg, cfg),
+ )
}
// bounce builds the restart callback SyncExec calls for a route whose
diff --git a/internal/host/agent_test.go b/internal/host/agent_test.go
index a21ecdabe..4501821dc 100644
--- a/internal/host/agent_test.go
+++ b/internal/host/agent_test.go
@@ -22,6 +22,7 @@ import (
"time"
"github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/domains"
"github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
@@ -1054,6 +1055,43 @@ func TestReconcileSerializedHoldsTheMutex(t *testing.T) {
}
}
+// TestReconcileSerializedRunsBothRecordFamilies pins the wiring: the sidecar
+// pass runs from the SAME serialized entry point as the instance pass, under
+// the same mutex hold. Wired anywhere else it would either race the instance
+// pass over one registry, or (worse) be forgotten by one of the two loops and
+// converge only every five minutes — or never.
+func TestReconcileSerializedRunsBothRecordFamilies(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ sqid := domains.Sqid(2)
+ manifest := []byte(`{"id":2,"sqid":"` + sqid + `","name":"Eleven","created_at":"2026-08-02T09:00:00Z"}`)
+ if _, err := s.Put(ctx, "accounts/"+sqid+"/account.json", manifest, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := provision.PutSidecarManifest(ctx, s, sqid, "eleven", provision.SidecarManifest{
+ Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}, Channel: "stable"}); err != nil {
+ t.Fatal(err)
+ }
+ host := provision.SidecarHost("bot", "eleven", sqid)
+ if err := provision.CreateSidecarRecord(ctx, s, provision.SidecarRecord{
+ Schema: 1, Name: "bot", Host: host, App: "eleven", Account: sqid, Channel: "stable"}); err != nil {
+ t.Fatal(err)
+ }
+
+ reg := restartTestRegistry(t)
+ if err := reconcileSerialized(ctx, s, reg, provision.ReconcileConfig{DataDir: t.TempDir()}); err != nil {
+ t.Fatalf("reconcileSerialized: %v", err)
+ }
+
+ rt, err := reg.Get(host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rt == nil {
+ t.Fatal("no sidecar route minted — the sidecar pass must run from reconcileSerialized, beside the instance pass and inside its mutex")
+ }
+}
+
// TestFastTickActsOnSerialChangeNotBoot pins seed-don't-replay, in both of
// its halves. An ABSENT serial object seeds at zero — absence is not
// history, there is nothing to replay, and every production bucket starts
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index 71ee5a3b7..fee9be573 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -39,6 +39,13 @@ type ReconcileConfig struct {
DataDir string
// Now is the clock seam; nil = time.Now.
Now func() time.Time
+ // Inbound is this box's inbound-work declaration (CARLOS_INBOUND, design
+ // §8). "" and InboundAny are today's behaviour exactly — an inert
+ // default, the apps-domain-style gate this repo already uses — and
+ // InboundNone means the box serves sidecar work only: no certs, no
+ // inbound ports, possibly a private subnet with egress-only networking.
+ // Read by ReconcileSidecars' placement gate.
+ Inbound string
// DryRun reports what the pass would do and writes nothing — not to the
// registry, not anywhere. The backfill mitigation (design §4.3): a
// production route is adopted only after someone has read what adopting
@@ -47,6 +54,16 @@ type ReconcileConfig struct {
DryRun bool
}
+// InboundAny and InboundNone are the closed vocabulary ReconcileConfig.Inbound
+// is read against — the BOX's half of design §8's two independent halves (the
+// record's half is PlacementAny/PlacementNoInbound). "" reads as InboundAny:
+// a box that has not been told is a box that serves, which is every box that
+// exists today.
+const (
+ InboundAny = "any"
+ InboundNone = "none"
+)
+
// acctApp identifies one app within one account — the unit both the record
// scan and the manifest/seed cache below key on, since app names are unique
// only within an account, never globally.
@@ -478,25 +495,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// minting is the reversible direction, and a bucket writer who could
// plant a marker to stall minting can already retire the instance
// outright by deleting its record.
- trashCache := map[acctApp]bool{}
- trashSeen := map[acctApp]bool{}
- trashedFor := func(acct, app string) bool {
- k := acctApp{acct, app}
- if trashSeen[k] {
- return trashCache[k]
- }
- trashSeen[k] = true
- tomb, err := release.ReadAppTombstone(ctx, s, acct, app)
- switch {
- case err != nil:
- log.Printf("provision: trash marker for %s/%s unreadable: %v — not minting for this app", acct, app, err)
- trashCache[k] = true
- case tomb != nil:
- log.Printf("provision: %s/%s is in the trash — not minting instance routes for it", acct, app)
- trashCache[k] = true
- }
- return trashCache[k]
- }
+ trashedFor := trashGate(ctx, s, "instance")
// --- mint / converge / conflict -------------------------------------------
routes, err := reg.List()
@@ -650,6 +649,17 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// the adoption guard is not mistaken for a standing guarantee.
if !readFailed {
for _, rt := range routes {
+ // Removal authority is scoped to this pass's OWN kind. A sidecar
+ // row is Provisioned too — that flag is what grants a pass removal
+ // authority — and it can never appear in the INSTANCE record set,
+ // so without this guard the two passes would delete each other's
+ // rows on every tick, forever, with no error anywhere and nothing
+ // in the logs that reads as a failure. ReconcileSidecars carries
+ // the mirror of this guard; both directions are asserted
+ // (TestInstanceReconcileNeverRemovesSidecarRows).
+ if rt.Kind == SidecarKind {
+ continue
+ }
if !rt.Provisioned || rt.Region != cfg.Region || !poolServed(cfg.Pools, rt.Pool) {
continue
}
@@ -722,6 +732,38 @@ func knownAccounts(ctx context.Context, s release.Store) (known map[string]bool,
return known, readFailed, nil
}
+// trashGate builds the per-(acct, app) "is this app in the trash" predicate
+// both record passes gate MINTING on, caching each answer for the pass. what
+// names the route family in the log line ("instance", "sidecar") and is the
+// only thing that differs between the two callers — one implementation, so
+// the two families cannot come to disagree about what a trash marker means.
+//
+// The predicate answers TRUE for an unreadable or malformed marker as well as
+// a valid one: not minting is the reversible direction, and a bucket writer
+// who could plant a marker to stall minting can already retire the record
+// outright.
+func trashGate(ctx context.Context, s release.Store, what string) func(acct, app string) bool {
+ cache := map[acctApp]bool{}
+ seen := map[acctApp]bool{}
+ return func(acct, app string) bool {
+ k := acctApp{acct, app}
+ if seen[k] {
+ return cache[k]
+ }
+ seen[k] = true
+ tomb, err := release.ReadAppTombstone(ctx, s, acct, app)
+ switch {
+ case err != nil:
+ log.Printf("provision: trash marker for %s/%s unreadable: %v — not minting for this app", acct, app, err)
+ cache[k] = true
+ case tomb != nil:
+ log.Printf("provision: %s/%s is in the trash — not minting %s routes for it", acct, app, what)
+ cache[k] = true
+ }
+ return cache[k]
+ }
+}
+
// poolServed reports whether pool is one this box serves. An empty
// cfg.Pools means the shared "" pool ONLY, never "every pool" — the same
// convention domains.Config's AppsDomain=="" carries: unconfigured means the
diff --git a/internal/provision/sidecar_reconcile.go b/internal/provision/sidecar_reconcile.go
new file mode 100644
index 000000000..c833986a0
--- /dev/null
+++ b/internal/provision/sidecar_reconcile.go
@@ -0,0 +1,462 @@
+package provision
+
+// ReconcileSidecars turns bucket SIDECAR records into registry routes, for
+// THIS agent's own region, pool set and inbound posture. Spec:
+// docs/superpowers/specs/2026-08-17-sidecar-members-design.md §4, §8.
+//
+// It is Reconcile's twin, one record family over, and it is deliberately a
+// separate pass rather than a widening of that one: the two families answer
+// different questions about a box (can it SERVE vs may it RUN), and a single
+// loop would have had to carry a Kind branch through every phase — including
+// the removal phase, where a missed branch deletes production rows.
+//
+// Which makes the collision this file exists to close worth stating plainly.
+// Reconcile's removal phase is scoped by Provisioned, region, pool and
+// absence-from-records. A sidecar row is Provisioned too (that flag is what
+// grants a pass removal authority over a row), and it can never appear in the
+// INSTANCE record set — so, unguarded, that pass deletes every sidecar row on
+// every tick: minted by this pass, destroyed by that one, forever, with no
+// error anywhere and nothing in the logs that reads as a failure. Each pass's
+// removal authority is therefore scoped to its OWN Kind, asserted from both
+// sides (TestInstanceReconcileNeverRemovesSidecarRows,
+// TestSidecarReconcileRemovesOnlyItsOwnKind).
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// SidecarKind is the route Kind every sidecar row carries — the whole basis
+// of each pass's removal authority, and the value the edge refuses outright.
+// A literal in two passes' removal conditions is a typo away from a fleet
+// deletion, so both read this constant.
+const SidecarKind = "sidecar"
+
+// sidecarCoSpawnSuffix names the LEGACY, file-staged co-spawn: a sidecar
+// attached to a host by dropping <DataDir>/<host>-sidecar.env
+// (internal/activator/backend_exec.go). It is legacy but LIVE — Eleven runs
+// on it today — so a host that already has one is refused a native sidecar
+// row. One host, one mechanism.
+const sidecarCoSpawnSuffix = "-sidecar.env"
+
+// sidecarArgsJSON renders a manifest's argv as the wire text registry.Route's
+// Args column carries: "" for nil/empty (no argv), else the JSON array. Same
+// shape and same reason as seedJSON — json.Marshal of a nil slice is "null",
+// which is neither empty nor a usable argv.
+func sidecarArgsJSON(m SidecarManifest) string {
+ if len(m.Args) == 0 {
+ return ""
+ }
+ b, err := json.Marshal(m.Args) // []string always marshals; err is unreachable
+ if err != nil {
+ return ""
+ }
+ return string(b)
+}
+
+// sidecarChannel resolves the channel a sidecar's row is pinned to: the
+// record's own if it states one, else the app manifest's. The record wins
+// because it is the per-sidecar authorization; the manifest is the app-wide
+// default a record may simply not repeat.
+func sidecarChannel(rec SidecarRecord, m SidecarManifest) string {
+ if rec.Channel != "" {
+ return rec.Channel
+ }
+ return m.Channel
+}
+
+// sidecarPlacement resolves the placement policy a record is placed under
+// (design §8). The record wins when it states one — it is the workload's own
+// requirement — and the manifest's PlacementOrDefault answers otherwise, so
+// an app-wide "everything I run must sit somewhere with no inbound path"
+// holds for records that never mention placement.
+func sidecarPlacement(rec SidecarRecord, m SidecarManifest) string {
+ if rec.Placement != "" {
+ return rec.Placement
+ }
+ return m.PlacementOrDefault()
+}
+
+// sidecarRouteFor is the one answer to "what route does this sidecar record
+// describe" — the mint builds from it and the converge branch compares
+// against it, so a shape this pass would mint and a shape it calls converged
+// can never drift apart (routeFor's rule, same reason).
+//
+// Socket and Addr are deliberately left EMPTY, and that is the whole
+// reachability story: a sidecar binds nothing and nothing dials it. The edge
+// refuses Kind == "sidecar" outright (design §6.2) rather than relying on
+// this omission, but the row must not carry an address for any future arm to
+// find either.
+//
+// DBPath, by contrast, IS derived: a sidecar gets a database like any other
+// tenant, and a hibernating route whose row carried no path would be swept up
+// by ops.FleetDBs' dataDir glob and replicated box-wide as if it never slept
+// (routeFor's comment records what that costs).
+//
+// The ceilings are materialized onto the row from the manifest precisely so
+// the activator's sweeper never needs a bucket read to decide a worker has
+// outrun its limits.
+func sidecarRouteFor(rec SidecarRecord, m SidecarManifest, dataDir string) registry.Route {
+ return registry.Route{
+ Host: rec.Host, Slug: rec.Name, Kind: SidecarKind,
+ App: rec.App, Account: rec.Account, Pool: rec.Pool, Region: rec.Region,
+ Channel: sidecarChannel(rec, m), Environment: rec.Environment,
+ DBPath: DefaultDBPath(dataDir, rec.Host),
+ Hibernate: m.HibernateOrDefault(),
+ Args: sidecarArgsJSON(m),
+ IdleSeconds: int(m.IdleOrDefault() / time.Second),
+ MaxRunSeconds: int(m.MaxRunOrDefault() / time.Second),
+ Provisioned: true,
+ Status: "ready",
+ }
+}
+
+// coSpawnStaged reports whether host already has a legacy file-staged
+// co-spawn in dataDir. A read error other than "not there" answers TRUE: not
+// minting is the reversible direction, and a host whose staging state cannot
+// be established is exactly the host a second mechanism must not be attached
+// to.
+func coSpawnStaged(dataDir, host string) bool {
+ _, err := os.Stat(filepath.Join(dataDir, host+sidecarCoSpawnSuffix))
+ return err == nil || !errors.Is(err, os.ErrNotExist)
+}
+
+// ReconcileSidecars materializes bucket sidecar records into registry routes
+// for this box's region, pools and inbound posture: mint on appearance,
+// converge the manifest-owned columns, remove on record deletion — removal
+// only for Kind "sidecar" routes this pass (or an earlier one running the
+// same scope) minted, marked by Provisioned.
+//
+// Per-item failures log and skip. A failed account-manifest or record read
+// additionally disables the removal phase for the WHOLE pass: a transient
+// read failure must never mass-retire a tenant's fleet. Minting is additive
+// and always safe, so it proceeds regardless.
+//
+// Three states decline to MINT while leaving the record in the pass's record
+// set — so removal never fires for them either, because declining to create
+// is reversible and deleting a row is not (registry.Delete purges the route's
+// capability grants, and nothing else would put the row back):
+//
+// - the app is in the trash (domains.Reconcile drops its routes on its own
+// authority; minting here would re-raise them every tick),
+// - the app's sidecars manifest is absent or unreadable (no argv to run,
+// and no ceilings to stamp),
+// - the record's placement is not one this box satisfies (design §8).
+func ReconcileSidecars(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig) error {
+ if cfg.Now == nil {
+ cfg.Now = time.Now
+ }
+
+ known, readFailed, err := knownAccounts(ctx, s)
+ if err != nil {
+ return err
+ }
+
+ // --- collect in-scope records --------------------------------------------
+ keys, err := s.List(ctx, "apps/")
+ if err != nil {
+ return fmt.Errorf("provision: list apps: %w", err)
+ }
+ recs := map[string]SidecarRecord{} // host -> record, in-scope only
+ // protect holds hosts whose record KEY parsed but whose body could not be
+ // trusted — unparseable JSON, a body disagreeing with its own path, or a
+ // shape this binary refuses. Such a host must NEVER read as "the bucket
+ // retired it": a non-additive record encoding change reaching an older
+ // binary would make EVERY record fail exactly this way, and one tick would
+ // then delete every sidecar in scope. Narrower than readFailed (which
+ // vetoes removal for the whole pass): only the unusable host is held back.
+ protect := map[string]bool{}
+ for _, key := range keys {
+ acct, app, name, ok := ParseSidecarKey(key, known)
+ if !ok {
+ continue
+ }
+ // Derived from the KEY, never from the body: the body is what is
+ // under suspicion in every branch below.
+ host := SidecarHost(name, app, acct)
+ raw, _, err := s.Get(ctx, key)
+ if err != nil {
+ log.Printf("provision: skipping sidecar record %s: read failed: %v", key, err)
+ // Its absence from recs below must not read as "the bucket retired
+ // it" — a transient Get failure disables removal for the whole
+ // pass, same as an unreadable account manifest.
+ readFailed = true
+ continue
+ }
+ var rec SidecarRecord
+ if err := json.Unmarshal(raw, &rec); err != nil {
+ log.Printf("provision: skipping sidecar record %s: malformed: %v", key, err)
+ protect[host] = true
+ continue
+ }
+ if rec.Name != name || rec.Account != acct || rec.App != app {
+ log.Printf("provision: skipping sidecar record %s: disagrees with its own path (name=%q account=%q app=%q)",
+ key, rec.Name, rec.Account, rec.App)
+ protect[host] = true
+ continue
+ }
+ // ValidateSidecarShape is the console create door's policy check — but
+ // a door is not a guarantee about what is already in the bucket:
+ // anything with bucket write access, and any record written by a
+ // different (older, or future) writer, arrives here unexamined. Both
+ // doors hold the invariant, neither trusts the other. Protected like a
+ // corrupted body, and for the same reason: a record that cannot be
+ // used is not a record that was retired.
+ if err := ValidateSidecarShape(rec); err != nil {
+ log.Printf("provision: skipping sidecar record %s: invalid shape: %v", key, err)
+ protect[host] = true
+ continue
+ }
+ // The host is DERIVED from the identity, never taken on trust: it
+ // becomes a database path and a registry primary key, and a record
+ // free to state its own host could name any row on the box.
+ if rec.Host != host {
+ log.Printf("provision: skipping sidecar record %s: host %q is not %q", key, rec.Host, host)
+ protect[host] = true
+ continue
+ }
+ if rec.Region != cfg.Region || !poolServed(cfg.Pools, rec.Pool) {
+ continue // another agent's jurisdiction, not this pass's business
+ }
+ // Two records claiming one host is a collision this pass cannot
+ // arbitrate silently: whichever this listing reaches FIRST wins, and
+ // the loser is skipped loudly rather than overwriting the winner in
+ // the map with no trace of the conflict.
+ if prev, dup := recs[rec.Host]; dup {
+ log.Printf("provision: duplicate sidecar host %s: keeping %s, skipping %s", rec.Host,
+ SidecarRecordKey(prev.Account, prev.App, prev.Name), key)
+ continue
+ }
+ recs[rec.Host] = rec
+ }
+
+ // --- manifest cache, per (acct, app), lazily ------------------------------
+ // known=false covers BOTH "no manifest" and "the read failed", and in both
+ // cases this pass declines to mint or converge: every manifest-owned
+ // column (argv, the ceilings, hibernation) would otherwise be stamped from
+ // a zero value — an empty argv over a working one, on a blip. This is
+ // seedFor's seedKnown discipline, applied to a family where the manifest
+ // carries the RUN instruction rather than a seed.
+ manCache := map[acctApp]SidecarManifest{}
+ manKnown := map[acctApp]bool{}
+ manSeen := map[acctApp]bool{}
+ manifestFor := func(acct, app string) (SidecarManifest, bool) {
+ k := acctApp{acct, app}
+ if manSeen[k] {
+ return manCache[k], manKnown[k]
+ }
+ manSeen[k] = true
+ m, err := GetSidecarManifest(ctx, s, acct, app)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ // Absence IS the answer (design §3.1): sidecars are not
+ // enabled for this app. Not a read failure — removal stays
+ // armed for the pass — but the records themselves stay in
+ // recs, so nothing of theirs is retired on it.
+ log.Printf("provision: %s/%s has sidecar records but no sidecars manifest — not minting", acct, app)
+ return SidecarManifest{}, false
+ }
+ log.Printf("provision: sidecar manifest %s/%s: read failed: %v", acct, app, err)
+ readFailed = true
+ return SidecarManifest{}, false
+ }
+ manCache[k], manKnown[k] = *m, true
+ return manCache[k], manKnown[k]
+ }
+
+ trashed := trashGate(ctx, s, "sidecar")
+
+ // --- mint / converge -------------------------------------------------------
+ routes, err := reg.List()
+ if err != nil {
+ return fmt.Errorf("provision: list routes: %w", err)
+ }
+ byHost := map[string]registry.Route{}
+ for _, rt := range routes {
+ byHost[rt.Host] = rt
+ }
+
+ for _, rec := range recs {
+ if trashed(rec.Account, rec.App) {
+ continue
+ }
+ m, mKnown := manifestFor(rec.Account, rec.App)
+ if !mKnown {
+ continue
+ }
+ // Placement (design §8). "none" is a property of the BOX's network;
+ // "no-inbound" is a requirement of the WORKLOAD, and the point of the
+ // two halves being independent is that a member gets a mechanical
+ // guarantee rather than an operator's promise.
+ if place := sidecarPlacement(rec, m); place == PlacementNoInbound && cfg.Inbound != InboundNone {
+ log.Printf("provision: not placing sidecar %s here: it asks for %q placement and this box accepts inbound work (CARLOS_INBOUND=%q)",
+ rec.Host, place, cfg.Inbound)
+ continue
+ }
+ existing, ok := byHost[rec.Host]
+ switch {
+ case !ok:
+ // One host, one mechanism: a host already carrying the legacy
+ // file-staged co-spawn must not ALSO get a native sidecar row, or
+ // the same workload runs twice under two owners that know nothing
+ // of each other.
+ if coSpawnStaged(cfg.DataDir, rec.Host) {
+ log.Printf("provision: REFUSING to mint sidecar %s: %s%s is staged on this box — "+
+ "the legacy file co-spawn and a native sidecar row are two mechanisms for one host; "+
+ "remove the file to migrate this host onto sidecar records",
+ rec.Host, rec.Host, sidecarCoSpawnSuffix)
+ continue
+ }
+ nr := sidecarRouteFor(rec, m, cfg.DataDir)
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would mint sidecar route %s (app %q acct %q channel %q args %s)",
+ rec.Host, rec.App, rec.Account, nr.Channel, nr.Args)
+ continue
+ }
+ if err := reg.Upsert(nr); err != nil {
+ log.Printf("provision: mint sidecar %s: %v", rec.Host, err)
+ continue
+ }
+ log.Printf("provision: minted sidecar route %s (app %q acct %q args %s)", rec.Host, rec.App, rec.Account, nr.Args)
+ case existing.Kind == SidecarKind && existing.Provisioned &&
+ existing.App == rec.App && existing.Account == rec.Account:
+ convergeSidecar(reg, existing, rec, m, cfg)
+ default:
+ // A row exists at this host that this pass does not own. There is
+ // deliberately NO adoption path here, unlike the instance pass:
+ // the .sidecar.internal suffix is reserved and refused at every
+ // door a name could enter through, so no hand-added row to
+ // backfill can legitimately exist — and adoption is an authority
+ // TRANSFER (it is what grants the removal phase power over a row),
+ // never a shape coincidence.
+ log.Printf("provision: %s exists but is not a sidecar row this pass owns (kind %q provisioned %v app %q acct %q) — leaving it alone",
+ rec.Host, existing.Kind, existing.Provisioned, existing.App, existing.Account)
+ }
+ }
+
+ // --- removal, only when nothing failed to read this pass --------------------
+ // Four conditions, each load-bearing: this pass's OWN Kind (the guard at
+ // the top of this file — the instance pass carries its mirror), Provisioned
+ // (the flag, not the app and not the channel, is what grants removal
+ // authority; a hand-added row never has it), scope (this box's own
+ // region/pool jurisdiction), and absence from the record set this pass just
+ // read (the bucket's own decision to retire the sidecar, not a guess).
+ if !readFailed {
+ for _, rt := range routes {
+ if rt.Kind != SidecarKind || !rt.Provisioned {
+ continue
+ }
+ if rt.Region != cfg.Region || !poolServed(cfg.Pools, rt.Pool) {
+ continue
+ }
+ if _, ok := recs[rt.Host]; ok {
+ continue
+ }
+ if protect[rt.Host] {
+ log.Printf("provision: not removing sidecar %s: its record exists but could not be read cleanly this pass", rt.Host)
+ continue
+ }
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would remove retired sidecar route %s (record gone)", rt.Host)
+ continue
+ }
+ if err := reg.Delete(rt.Host); err != nil {
+ log.Printf("provision: remove retired sidecar route %s: %v", rt.Host, err)
+ continue
+ }
+ log.Printf("provision: removed retired sidecar route %s (record gone)", rt.Host)
+ }
+ }
+ return nil
+}
+
+// convergeSidecar applies the columns the record and the manifest are
+// authoritative for onto a row this pass owns. Narrow UPDATEs throughout,
+// never a re-Upsert: a read-modify-write racing the activator's own writes
+// (a worker's reported version, its next-due stamp) would clobber them.
+//
+// The split of authority is the same one the design states: the RECORD owns
+// channel and environment (what this sidecar runs and which config it gets —
+// the #130 discipline), the MANIFEST owns argv, the ceilings and hibernation
+// (app-wide policy). Both are re-stamped every pass, exactly as an instance's
+// seed is, because truth lives in the bucket and a box-local edit to one of
+// these columns is drift, not a decision.
+//
+// Every write logs on success as well as on failure: these are the writes
+// that REVERT an operator's own `carlos route` edit, and a row that moved with
+// nothing saying so is the failure class behind #130 seen from the outside.
+func convergeSidecar(reg *registry.Registry, existing registry.Route, rec SidecarRecord, m SidecarManifest, cfg ReconcileConfig) {
+ want := sidecarRouteFor(rec, m, cfg.DataDir)
+
+ if want.Channel != "" && existing.Channel != want.Channel {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would converge sidecar %s channel %s -> %s", rec.Host, existing.Channel, want.Channel)
+ } else if err := reg.SetChannel(rec.Host, want.Channel); err != nil {
+ log.Printf("provision: converge channel on sidecar %s: %v", rec.Host, err)
+ } else {
+ log.Printf("provision: converged sidecar %s channel %s -> %s", rec.Host, existing.Channel, want.Channel)
+ }
+ }
+ if existing.Environment != want.Environment {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would converge sidecar %s environment %q -> %q", rec.Host, existing.Environment, want.Environment)
+ } else if err := reg.SetEnvironment(rec.Host, want.Environment); err != nil {
+ log.Printf("provision: converge environment on sidecar %s: %v", rec.Host, err)
+ } else {
+ log.Printf("provision: converged sidecar %s environment %q -> %q", rec.Host, existing.Environment, want.Environment)
+ }
+ }
+ if existing.Hibernate != want.Hibernate {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would converge sidecar %s hibernate %v -> %v", rec.Host, existing.Hibernate, want.Hibernate)
+ } else if err := reg.SetHibernate(rec.Host, want.Hibernate); err != nil {
+ log.Printf("provision: converge hibernate on sidecar %s: %v", rec.Host, err)
+ } else {
+ log.Printf("provision: converged sidecar %s hibernate %v -> %v", rec.Host, existing.Hibernate, want.Hibernate)
+ }
+ }
+ if existing.Args != want.Args {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would converge sidecar %s args %s -> %s", rec.Host, existing.Args, want.Args)
+ } else if err := reg.SetArgs(rec.Host, want.Args); err != nil {
+ log.Printf("provision: converge args on sidecar %s: %v", rec.Host, err)
+ } else {
+ log.Printf("provision: converged sidecar %s args %s -> %s", rec.Host, existing.Args, want.Args)
+ }
+ }
+ // One UPDATE for both ceilings, because they are one policy: the idle
+ // window and the run ceiling together decide when the sweeper stops a
+ // worker, and a row carrying the new idle under the old maximum (or the
+ // reverse) is a combination the manifest never stated.
+ if existing.IdleSeconds != want.IdleSeconds || existing.MaxRunSeconds != want.MaxRunSeconds {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would converge sidecar %s ceilings %d/%d -> %d/%d", rec.Host,
+ existing.IdleSeconds, existing.MaxRunSeconds, want.IdleSeconds, want.MaxRunSeconds)
+ } else if err := reg.SetWorkerLimits(rec.Host, want.IdleSeconds, want.MaxRunSeconds); err != nil {
+ log.Printf("provision: converge ceilings on sidecar %s: %v", rec.Host, err)
+ } else {
+ log.Printf("provision: converged sidecar %s ceilings %d/%d -> %d/%d", rec.Host,
+ existing.IdleSeconds, existing.MaxRunSeconds, want.IdleSeconds, want.MaxRunSeconds)
+ }
+ }
+ // The columns nothing converges in place: the database path (moving it
+ // under a running process would point it at a second copy of its own
+ // data) and the two reachability columns a sidecar row must never carry.
+ // Reported, never rewritten — `carlos remove` purges a route's capability
+ // grants, so it is a last resort, not a routine fix.
+ if existing.DBPath != want.DBPath || existing.Socket != "" || existing.Addr != "" {
+ log.Printf("provision: sidecar %s: row disagrees with the shape this record describes "+
+ "(db %q/%q socket %q addr %q) — not converged in place; a sidecar binds nothing, "+
+ "so a socket or addr on this row is a row that was not minted by this pass",
+ rec.Host, existing.DBPath, want.DBPath, existing.Socket, existing.Addr)
+ }
+}
diff --git a/internal/provision/sidecar_reconcile_test.go b/internal/provision/sidecar_reconcile_test.go
new file mode 100644
index 000000000..f0a8d7e17
--- /dev/null
+++ b/internal/provision/sidecar_reconcile_test.go
@@ -0,0 +1,435 @@
+package provision
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// --- fixtures -----------------------------------------------------------------
+
+// failStore is the read-failure seam these tests need and no other store
+// shape provides: a Store whose Get (and List) refuse a chosen prefix, so a
+// pass can be put in exactly the state the fail-safe exists for — a record
+// that IS there and could not be read — rather than the state that looks the
+// same from the outside and means the opposite (a record the bucket retired).
+type failStore struct {
+ release.Store
+ fail []string
+}
+
+func (s *failStore) failReadsFor(prefix string) { s.fail = append(s.fail, prefix) }
+
+func (s *failStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ for _, p := range s.fail {
+ if strings.HasPrefix(key, p) {
+ return nil, "", fmt.Errorf("failStore: reads disabled under %s", p)
+ }
+ }
+ return s.Store.Get(ctx, key)
+}
+
+func (s *failStore) List(ctx context.Context, prefix string) ([]string, error) {
+ for _, p := range s.fail {
+ if strings.HasPrefix(prefix, p) {
+ return nil, fmt.Errorf("failStore: reads disabled under %s", p)
+ }
+ }
+ return s.Store.List(ctx, prefix)
+}
+
+type sidecarFixture struct {
+ store *failStore
+ reg *registry.Registry
+ dir string
+ sqid string
+}
+
+// sidecarSetup writes a valid account manifest and hands back a fixture whose
+// data dir and registry are empty — the sidecar family's equivalent of
+// setup() above, minus the instances manifest (a sidecar app opts in through
+// its OWN manifest, sidecars.json).
+func sidecarSetup(t *testing.T) *sidecarFixture {
+ t.Helper()
+ s := &failStore{Store: dirStore(t)}
+ sqid := writeAccountManifest(t, s, acctID, "Eleven")
+ return &sidecarFixture{store: s, reg: regWith(t), dir: t.TempDir(), sqid: sqid}
+}
+
+func (f *sidecarFixture) host(name, app string) string { return SidecarHost(name, app, f.sqid) }
+
+func (f *sidecarFixture) putManifest(t *testing.T, app string, m SidecarManifest) {
+ t.Helper()
+ if err := PutSidecarManifest(ctx, f.store, f.sqid, app, m); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (f *sidecarFixture) putRecord(t *testing.T, rec SidecarRecord) {
+ t.Helper()
+ if rec.Account == "" {
+ rec.Account = f.sqid
+ }
+ if rec.Host == "" {
+ rec.Host = SidecarHost(rec.Name, rec.App, rec.Account)
+ }
+ if err := CreateSidecarRecord(ctx, f.store, rec); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (f *sidecarFixture) mustUpsert(t *testing.T, rt registry.Route) {
+ t.Helper()
+ if err := f.reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (f *sidecarFixture) reconcile(t *testing.T, cfg ReconcileConfig) {
+ t.Helper()
+ cfg.DataDir = f.dir
+ if err := ReconcileSidecars(ctx, f.store, f.reg, cfg); err != nil {
+ t.Fatalf("ReconcileSidecars: %v", err)
+ }
+}
+
+// noRoute asserts a host has no row. registry.Get answers a missing host with
+// (nil, nil), so a bare `err == nil` check would pass whether the row is
+// there or not.
+func noRoute(t *testing.T, reg *registry.Registry, host, msg string) {
+ t.Helper()
+ rt, err := reg.Get(host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rt != nil {
+ t.Fatal(msg)
+ }
+}
+
+func hasRoute(t *testing.T, reg *registry.Registry, host, msg string) *registry.Route {
+ t.Helper()
+ rt, err := reg.Get(host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rt == nil {
+ t.Fatal(msg)
+ }
+ return rt
+}
+
+// --- the removal-authority collision -------------------------------------------
+
+// TestInstanceReconcileNeverRemovesSidecarRows is the guard on the OTHER
+// pass. A sidecar row is Provisioned (that flag is what grants a pass removal
+// authority) and can never appear in the instance pass's record set, so
+// without a Kind guard the instance pass deletes every sidecar row on every
+// tick — minted by one pass, destroyed by the other, forever, with no error
+// anywhere and nothing in the logs that reads as a failure.
+func TestInstanceReconcileNeverRemovesSidecarRows(t *testing.T) {
+ f := sidecarSetup(t)
+ host := f.host("bot", "eleven")
+ f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot",
+ App: "eleven", Account: f.sqid, Provisioned: true, Status: "ready"})
+
+ // No instance records at all: the pass's removal phase runs in full.
+ if err := Reconcile(ctx, f.store, f.reg, ReconcileConfig{DataDir: f.dir}); err != nil {
+ t.Fatalf("Reconcile: %v", err)
+ }
+ hasRoute(t, f.reg, host, "the instance pass deleted a sidecar row — each pass's removal authority is scoped to its OWN kind, or the two passes destroy each other's work forever")
+}
+
+// TestSidecarReconcileRemovesOnlyItsOwnKind is the same guard from the other
+// side: this pass retires its own kind and must never reach an instance row,
+// which can never appear in ITS record set either.
+func TestSidecarReconcileRemovesOnlyItsOwnKind(t *testing.T) {
+ f := sidecarSetup(t)
+ gone := f.host("gone", "eleven")
+ f.mustUpsert(t, registry.Route{Host: gone, Kind: "sidecar", Slug: "gone",
+ App: "eleven", Account: f.sqid, Provisioned: true})
+ f.mustUpsert(t, registry.Route{Host: "app.example.com", Kind: "instance",
+ App: "eleven", Account: f.sqid, Provisioned: true})
+ // A sidecar-kind row nothing provisioned: Provisioned is the flag that
+ // GRANTS removal authority, so a row without it is not this pass's to
+ // retire however well it matches on every other column.
+ hand := f.host("hand", "eleven")
+ f.mustUpsert(t, registry.Route{Host: hand, Kind: "sidecar", Slug: "hand",
+ App: "eleven", Account: f.sqid})
+
+ f.reconcile(t, ReconcileConfig{})
+
+ noRoute(t, f.reg, gone, "a provisioned sidecar with no record must be removed")
+ hasRoute(t, f.reg, "app.example.com", "the sidecar pass must never touch an instance row")
+ hasRoute(t, f.reg, hand, "a sidecar row that was never provisioned is not this pass's to remove")
+}
+
+// --- mint / converge / remove --------------------------------------------------
+
+func TestSidecarReconcileMints(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{
+ Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}, Channel: "stable"})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"})
+
+ f.reconcile(t, ReconcileConfig{})
+
+ rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "no sidecar route minted")
+ if rt.Kind != "sidecar" {
+ t.Fatalf("Kind = %q", rt.Kind)
+ }
+ if rt.Socket != "" || rt.Addr != "" {
+ t.Fatalf("a sidecar row must carry NO socket and NO addr, got socket=%q addr=%q", rt.Socket, rt.Addr)
+ }
+ if rt.Slug != "bot" || rt.App != "eleven" || rt.Account != f.sqid || rt.Channel != "stable" {
+ t.Fatalf("route identity = %+v", rt)
+ }
+ if rt.DBPath != DefaultDBPath(f.dir, rt.Host) || !rt.Provisioned || rt.Args != `["sidecar","run"]` {
+ t.Fatalf("route = %+v", rt)
+ }
+ if !rt.Hibernate {
+ t.Fatal("Hibernate defaults true (HibernateOrDefault), and the row is what the activator reads")
+ }
+ if rt.IdleSeconds != 300 || rt.MaxRunSeconds != 3600 {
+ t.Fatalf("ceilings = %d/%d, want the manifest defaults 300/3600 materialized onto the row — the sweeper must never need a bucket read",
+ rt.IdleSeconds, rt.MaxRunSeconds)
+ }
+}
+
+func TestSidecarReconcileConvergesArgsFromTheManifest(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"})
+ f.reconcile(t, ReconcileConfig{})
+
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"worker", "run"}})
+ f.reconcile(t, ReconcileConfig{})
+
+ rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone")
+ if rt.Args != `["worker","run"]` {
+ t.Fatalf("Args = %q — truth lives in the bucket and is re-stamped every pass, exactly as seed is", rt.Args)
+ }
+}
+
+func TestSidecarReconcileConvergesTheCeilings(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven",
+ Args: []string{"sidecar", "run"}, IdleSeconds: 300, MaxRunSeconds: 3600})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"})
+ f.reconcile(t, ReconcileConfig{})
+
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven",
+ Args: []string{"sidecar", "run"}, IdleSeconds: 60, MaxRunSeconds: 120})
+ f.reconcile(t, ReconcileConfig{})
+
+ rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone")
+ if rt.IdleSeconds != 60 || rt.MaxRunSeconds != 120 {
+ t.Fatalf("ceilings = %d/%d, want 60/120 — the sweeper reads the row, so the row must follow the manifest", rt.IdleSeconds, rt.MaxRunSeconds)
+ }
+}
+
+// TestSidecarReconcileConvergesChannelEnvironmentAndHibernate pins the rest
+// of the owned columns: the record owns channel and environment (the #130
+// discipline), and the manifest owns hibernation.
+func TestSidecarReconcileConvergesChannelEnvironmentAndHibernate(t *testing.T) {
+ f := sidecarSetup(t)
+ off := false
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven",
+ Args: []string{"sidecar", "run"}, Hibernate: &off})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven",
+ Channel: "canary/eleven", Environment: "relay"})
+ f.reconcile(t, ReconcileConfig{})
+
+ rt := hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone")
+ if rt.Channel != "canary/eleven" || rt.Environment != "relay" || rt.Hibernate {
+ t.Fatalf("minted route = channel %q env %q hibernate %v", rt.Channel, rt.Environment, rt.Hibernate)
+ }
+
+ // Drift the row the way `carlos route` would, then converge it back.
+ if err := f.reg.SetChannel(rt.Host, "stable"); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.reg.SetEnvironment(rt.Host, "other"); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.reg.SetHibernate(rt.Host, true); err != nil {
+ t.Fatal(err)
+ }
+ f.reconcile(t, ReconcileConfig{})
+
+ rt = hasRoute(t, f.reg, f.host("bot", "eleven"), "route gone")
+ if rt.Channel != "canary/eleven" || rt.Environment != "relay" || rt.Hibernate {
+ t.Fatalf("after converge = channel %q env %q hibernate %v — the record and manifest own these columns",
+ rt.Channel, rt.Environment, rt.Hibernate)
+ }
+}
+
+// --- the fail-safe --------------------------------------------------------------
+
+// TestSidecarReconcileSkipsRemovalWhenAReadFailed pins the fail-safe: a
+// transient read failure must never mass-retire a tenant's fleet. Minting
+// still proceeds, because minting is additive.
+func TestSidecarReconcileSkipsRemovalWhenAReadFailed(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"})
+ keep := f.host("keep", "eleven")
+ f.mustUpsert(t, registry.Route{Host: keep, Kind: "sidecar", Slug: "keep",
+ App: "eleven", Account: f.sqid, Provisioned: true})
+
+ // The record IS there; this pass simply cannot read it.
+ f.store.failReadsFor(SidecarsPrefix(f.sqid, "eleven"))
+ f.reconcile(t, ReconcileConfig{})
+
+ hasRoute(t, f.reg, keep, "a read failure skipped nothing — one bad list must not retire a fleet")
+}
+
+// TestSidecarReconcileProtectsAMalformedRecord is the narrower guard: only
+// the host whose body could not be trusted is held back, and it is held back
+// rather than read as "the bucket retired it" — a non-additive record
+// encoding change reaching an older binary would otherwise delete every
+// sidecar in scope in one tick.
+func TestSidecarReconcileProtectsAMalformedRecord(t *testing.T) {
+ f := sidecarSetup(t)
+ host := f.host("bot", "eleven")
+ f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot",
+ App: "eleven", Account: f.sqid, Provisioned: true})
+ if _, err := f.store.Put(ctx, SidecarRecordKey(f.sqid, "eleven", "bot"), []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+
+ f.reconcile(t, ReconcileConfig{})
+
+ hasRoute(t, f.reg, host, "a record that cannot be PARSED is not a record that was retired")
+}
+
+// --- placement -------------------------------------------------------------------
+
+// TestSidecarReconcilePlacementGate pins §8's second half: a no-inbound
+// workload lands only on a box that declares it accepts no inbound work. Both
+// spellings are honoured — the record's own placement and the app manifest's.
+func TestSidecarReconcilePlacementGate(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Placement: PlacementNoInbound})
+ // A second app whose MANIFEST asks for the placement, record silent.
+ f.putManifest(t, "relay", SidecarManifest{Schema: 1, App: "relay",
+ Args: []string{"relay"}, Placement: PlacementNoInbound})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "hub", App: "relay"})
+
+ f.reconcile(t, ReconcileConfig{}) // an ordinary box: accepts inbound work
+ noRoute(t, f.reg, f.host("bot", "eleven"), "a no-inbound workload must not land on a box that accepts inbound work")
+ noRoute(t, f.reg, f.host("hub", "relay"), "a manifest-stated no-inbound placement must gate exactly as a record-stated one does")
+
+ f.reconcile(t, ReconcileConfig{Inbound: InboundNone})
+ hasRoute(t, f.reg, f.host("bot", "eleven"), "it must land on a box that declares CARLOS_INBOUND=none")
+ hasRoute(t, f.reg, f.host("hub", "relay"), "it must land on a box that declares CARLOS_INBOUND=none")
+}
+
+// TestSidecarReconcilePlacementMismatchDoesNotRemove: a record this box may
+// not RUN is still a record the bucket has not retired. Declining to mint is
+// the reversible direction; deleting the row is not (registry.Delete purges
+// the route's capability grants), and nothing else would put it back.
+func TestSidecarReconcilePlacementMismatchDoesNotRemove(t *testing.T) {
+ f := sidecarSetup(t)
+ host := f.host("bot", "eleven")
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Placement: PlacementNoInbound})
+ f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot",
+ App: "eleven", Account: f.sqid, Provisioned: true})
+
+ f.reconcile(t, ReconcileConfig{}) // ordinary box: placement does not match
+
+ rt := hasRoute(t, f.reg, host, "a placement this box cannot satisfy means DON'T MINT — it is not the bucket retiring the sidecar")
+ if rt.Args != "" {
+ t.Fatalf("Args = %q — a record this box may not place is a record this box does not act on at all", rt.Args)
+ }
+}
+
+// --- the co-spawn conflict ---------------------------------------------------------
+
+// TestSidecarReconcileRefusesHostWithAStagedCoSpawnFile: the file-staged
+// co-spawn is legacy but LIVE (Eleven runs on it today). One host, one
+// mechanism.
+func TestSidecarReconcileRefusesHostWithAStagedCoSpawnFile(t *testing.T) {
+ f := sidecarSetup(t)
+ host := f.host("bot", "eleven")
+ if err := os.WriteFile(filepath.Join(f.dir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"})
+
+ f.reconcile(t, ReconcileConfig{})
+
+ noRoute(t, f.reg, host, "a host with a staged legacy co-spawn file must not also get a native sidecar row — one host, one mechanism")
+}
+
+// --- scope ---------------------------------------------------------------------------
+
+// TestSidecarReconcileSkipsAnotherBoxesScope: region and pool scope the
+// sidecar pass exactly as they scope the instance one — a record for another
+// agent's jurisdiction is neither minted nor (for the row) removed here.
+func TestSidecarReconcileSkipsAnotherBoxesScope(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Region: "ie", Pool: "eleven"})
+ // A foreign box's own provisioned sidecar row, visible in this registry.
+ foreign := f.host("far", "eleven")
+ f.mustUpsert(t, registry.Route{Host: foreign, Kind: "sidecar", Slug: "far", App: "eleven",
+ Account: f.sqid, Region: "de", Pool: "other", Provisioned: true})
+
+ f.reconcile(t, ReconcileConfig{}) // this box: no region, no pools
+
+ noRoute(t, f.reg, f.host("bot", "eleven"), "a record for another region/pool is not this pass's business")
+ hasRoute(t, f.reg, foreign, "a foreign box's provisioned route is not this pass's to retire")
+
+ f.reconcile(t, ReconcileConfig{Region: "ie", Pools: []string{"eleven"}})
+ hasRoute(t, f.reg, f.host("bot", "eleven"), "the in-scope box must mint it")
+}
+
+// TestSidecarReconcileDryRunWritesNothing mirrors the instance pass's §4.3
+// mitigation: a pass that reports is a pass that writes nothing at all.
+func TestSidecarReconcileDryRunWritesNothing(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"})
+ gone := f.host("gone", "eleven")
+ f.mustUpsert(t, registry.Route{Host: gone, Kind: "sidecar", Slug: "gone",
+ App: "eleven", Account: f.sqid, Provisioned: true})
+
+ f.reconcile(t, ReconcileConfig{DryRun: true})
+
+ noRoute(t, f.reg, f.host("bot", "eleven"), "dry-run minted a route")
+ hasRoute(t, f.reg, gone, "dry-run removed a route")
+}
+
+// TestSidecarReconcileLeavesAForeignRowAlone: a row this pass does not own at
+// a host a record names is left exactly as alone as before. There is no
+// adoption path for sidecars — the suffix is reserved and refused at every
+// door, so nothing but this pass can have minted one — and adoption is an
+// authority transfer, never a shape coincidence.
+func TestSidecarReconcileLeavesAForeignRowAlone(t *testing.T) {
+ f := sidecarSetup(t)
+ host := f.host("bot", "eleven")
+ f.putManifest(t, "eleven", SidecarManifest{Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven"})
+ f.mustUpsert(t, registry.Route{Host: host, Kind: "sidecar", Slug: "bot",
+ App: "eleven", Account: f.sqid, Args: `["hand","written"]`})
+
+ f.reconcile(t, ReconcileConfig{})
+
+ rt := hasRoute(t, f.reg, host, "route gone")
+ if rt.Provisioned {
+ t.Fatal("an un-provisioned row must not be adopted by this pass — adoption grants removal authority")
+ }
+ if rt.Args != `["hand","written"]` {
+ t.Fatalf("Args = %q — a row this pass does not own must be left alone", rt.Args)
+ }
+}