amadan
provision: adopt an existing route whose shape matches an appearing record
Backfilling the fleet becomes a pure bucket-side write per route — describe what is already there and the next serial bump adopts it, with no box commands. The match test excludes Channel/Environment/Hibernate, which are the columns the record is taking ownership of, and refuses to adopt a row with a non-empty Backing — a record has no field for it (box-scoped operator column), so it can never describe a carlos-app@<host> unit tenant. Adoption grants the removal phase authority over the row, so shapeMatches compares the derived Socket and DBPath too. The Channel/Environment converge is factored into convergeOwned and shared by the owned-route converge branch and the adoption branch, so there is one place these two columns are written from a record, not two. Design: docs/superpowers/specs/2026-08-10-route-records-design.md §4.2
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
4 files changed,
+328
−13
internal/provision/reconcile.go+83 −13internal/provision/reconcile_test.go+196 −0internal/registry/registry.go+29 −0internal/registry/registry_test.go+20 −0
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go| index 9847fb897..66e93c12b 100644 |
| --- a/internal/provision/reconcile.go |
| +++ b/internal/provision/reconcile.go |
| @@ -70,6 +70,62 @@ func routeFor(rec Record, dataDir, seed string) registry.Route { |
| return rt |
| } |
| +// shapeMatches reports whether an existing route is the same ROUTE the |
| +// record describes — the gate on adoption (design §4.2), and so the gate on |
| +// this pass acquiring removal authority over a row it did not mint. |
| +// |
| +// Channel, Environment and Hibernate are deliberately absent: those are the |
| +// columns the record is taking ownership OF, and a hand-added route on the |
| +// wrong channel is the whole reason to adopt one. Requiring them to agree |
| +// would make adoption impossible in exactly the case it exists for. |
| +// |
| +// Backing is deliberately required to be empty on the existing row. A |
| +// record has no field for Backing — it is a box-scoped operator column, |
| +// alongside capability grants and tunnels (registry.Route.Backing's own |
| +// doc comment) — so a record can never describe a carlos-app@<host> unit |
| +// tenant at all. A row this pass cannot state the Backing of is a row this |
| +// pass must not claim to describe, no matter how well everything else |
| +// lines up. |
| +// |
| +// Everything else that identifies the route is compared, including the |
| +// derived Socket and DBPath — two rows agreeing on kind and app but not on |
| +// where the database lives are not the same route, and adopting across that |
| +// difference would point a running process at a second copy of its own data. |
| +func shapeMatches(existing, want registry.Route) bool { |
| + return existing.Backing == "" && |
| + existing.App == want.App && |
| + existing.Account == want.Account && |
| + existing.Kind == want.Kind && |
| + existing.Addr == want.Addr && |
| + existing.Unit == want.Unit && |
| + existing.Slug == want.Slug && |
| + existing.Pool == want.Pool && |
| + existing.Region == want.Region && |
| + existing.Socket == want.Socket && |
| + existing.DBPath == want.DBPath |
| +} |
| + |
| +// convergeOwned applies the record's Channel and Environment onto a route it |
| +// owns — the two columns the record is authoritative for once it owns the |
| +// row, whether ownership was just granted by adoption or was already held |
| +// by an earlier pass. Both the owned-route converge branch and the |
| +// adoption branch call this, so there is exactly one place these two |
| +// columns get written from a record, never two copies drifting apart. |
| +func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record) { |
| + if rec.Channel != "" && existing.Channel != rec.Channel { |
| + if err := reg.SetChannel(rec.Host, rec.Channel); err != nil { |
| + log.Printf("provision: converge channel on %s: %v", rec.Host, err) |
| + } else { |
| + log.Printf("provision: converged %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel) |
| + } |
| + } |
| + if existing.Environment != rec.Environment { |
| + if err := reg.SetEnvironment(rec.Host, rec.Environment); err != nil { |
| + log.Printf("provision: converge environment on %s: %v", rec.Host, err) |
| + } |
| + } |
| +} |
| + |
| // Reconcile materializes bucket instance records into registry routes for |
| // THIS agent's region and pools: mint on appearance, converge seed, remove |
| // on record deletion — removal only for routes this pass (or an earlier one |
| @@ -286,18 +342,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // deploy's SetVersions or the activator's SetNextDue would |
| // clobber those, and Upsert cannot express "clear Environment" |
| // at all (it preserves a non-empty value by design). |
| - if rec.Channel != "" && existing.Channel != rec.Channel { |
| - if err := reg.SetChannel(rec.Host, rec.Channel); err != nil { |
| - log.Printf("provision: converge channel on %s: %v", rec.Host, err) |
| - } else { |
| - log.Printf("provision: converged %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel) |
| - } |
| - } |
| - if existing.Environment != rec.Environment { |
| - if err := reg.SetEnvironment(rec.Host, rec.Environment); err != nil { |
| - log.Printf("provision: converge environment on %s: %v", rec.Host, err) |
| - } |
| - } |
| + convergeOwned(reg, existing, rec) |
| // Absent means "this record does not speak to hibernation" — |
| // never a derivation stamped over an operator's own setting. |
| if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate { |
| @@ -329,8 +374,33 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| want.Socket, existing.Socket, want.DBPath, existing.DBPath, rec.Host) |
| } |
| default: |
| - log.Printf("provision: %s exists but was not minted by this pass (app %q acct %q) — leaving it alone", |
| + // A route exists that this pass did not mint. If it is the same |
| + // route the record describes, adopt it — that is what makes |
| + // backfilling the fleet a bucket-side write rather than a box |
| + // session (design §4.2). If it is not, leave it exactly as alone |
| + // as before: adoption grants this pass's removal phase authority |
| + // over the row, so the match test is the only thing standing |
| + // between a backfill and a retired production route. |
| + want := routeFor(rec, cfg.DataDir, seed) |
| + if !shapeMatches(existing, want) { |
| + log.Printf("provision: %s exists but is not the route this record describes "+ |
| + "(app %q/%q kind %q/%q addr %q/%q) — leaving it alone", |
| + rec.Host, existing.App, want.App, existing.Kind, want.Kind, existing.Addr, want.Addr) |
| + continue |
| + } |
| + if err := reg.SetProvisioned(rec.Host, true); err != nil { |
| + log.Printf("provision: adopt %s: %v", rec.Host, err) |
| + continue |
| + } |
| + log.Printf("provision: adopted existing route %s (app %q acct %q) — this record now owns its channel and environment", |
| rec.Host, rec.App, rec.Account) |
| + // Hibernate is deliberately NOT converged here even when |
| + // explicit: this first pass over a hand-added route should |
| + // change what it serves and who owns the column, not whether a |
| + // live process is allowed to sleep. It converges on the next |
| + // pass through the owned branch above, by which point the |
| + // adoption is already visible in the log. |
| + convergeOwned(reg, existing, rec) |
| } |
| } |
diff --git a/internal/provision/reconcile_test.go b/internal/provision/reconcile_test.go| index 27609993b..ef6f30117 100644 |
| --- a/internal/provision/reconcile_test.go |
| +++ b/internal/provision/reconcile_test.go |
| @@ -917,3 +917,199 @@ func TestReconcileDeclinesUnreadableTrashMarker(t *testing.T) { |
| t.Fatalf("an unreadable trash marker still minted a route: %+v %v", rt, err) |
| } |
| } |
| + |
| +// Design §4.2 — the payoff. Backfilling the fleet is a pure bucket-side |
| +// write per route: describe what is already there, and the next serial bump |
| +// adopts it. No carlos add, no SSM session, no on-box verb. |
| +func TestReconcileAdoptsMatchingHandAddedRoute(t *testing.T) { |
| + s, sqid := setup(t) |
| + rec := acmeRecord(sqid) |
| + rec.Channel = "edge" |
| + if err := CreateRecord(ctx, s, rec); err != nil { |
| + t.Fatal(err) |
| + } |
| + // The route as an operator hand-added it before records existed: the |
| + // same shape, a different channel, and no Provisioned flag. |
| + handAdded := registry.Route{ |
| + Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| + Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| + App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| + Channel: "stable", Hibernate: true, Status: "ready", |
| + } |
| + reg := regWith(t, handAdded) |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + got := *mustGet(t, reg, "acme.go.tito.io") |
| + if !got.Provisioned { |
| + t.Fatal("a matching hand-added route was not adopted") |
| + } |
| + if got.Channel != "edge" { |
| + t.Errorf("channel = %q, want edge — adoption should converge in the same pass", got.Channel) |
| + } |
| +} |
| + |
| +// The match test EXCLUDES the three columns the record is taking ownership |
| +// of. Requiring them to agree first would make adoption impossible in |
| +// exactly the case it exists for — a hand-added route on the wrong channel |
| +// is the whole reason to adopt it. |
| +func TestReconcileAdoptionIgnoresOwnedColumns(t *testing.T) { |
| + s, sqid := setup(t) |
| + rec := acmeRecord(sqid) |
| + rec.Channel = "edge" |
| + if err := CreateRecord(ctx, s, rec); err != nil { |
| + t.Fatal(err) |
| + } |
| + handAdded := registry.Route{ |
| + Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| + Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| + App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| + Channel: "canary/rehearsal", Environment: "staging", Hibernate: false, Status: "ready", |
| + } |
| + reg := regWith(t, handAdded) |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !mustGet(t, reg, "acme.go.tito.io").Provisioned { |
| + t.Error("adoption refused over a Channel/Environment/Hibernate difference") |
| + } |
| +} |
| + |
| +// A shape that differs is left exactly as alone as it is today. This is the |
| +// guard that stands between "adopt on match" and reconcile's removal phase |
| +// acquiring authority over a route it does not describe. |
| +func TestReconcileRefusesToAdoptMismatchedRoute(t *testing.T) { |
| + s, sqid := setup(t) |
| + if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| + t.Fatal(err) |
| + } |
| + mismatched := registry.Route{ |
| + Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| + Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| + App: "other", Account: sqid, Pool: "tito", Region: "ie", |
| + Channel: "stable", Hibernate: true, Status: "ready", |
| + } |
| + reg := regWith(t, mismatched) |
| + before := *mustGet(t, reg, "acme.go.tito.io") |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + after := *mustGet(t, reg, "acme.go.tito.io") |
| + if after != before { |
| + t.Errorf("a mismatched route was touched: before %+v after %+v", before, after) |
| + } |
| + if after.Provisioned { |
| + t.Error("a mismatched route was adopted — removal authority granted over a route the record does not describe") |
| + } |
| +} |
| + |
| +// A record has no field for Backing — registry.Route.Backing's own doc |
| +// comment calls it a box-scoped operator column, alongside capability |
| +// grants and tunnels — so a record can never describe a carlos-app@<host> |
| +// unit tenant at all. Even a row that otherwise matches in every column |
| +// must not be adopted: adoption would hand this pass's removal phase |
| +// authority over a row it cannot even state the shape of. |
| +func TestReconcileRefusesToAdoptBackedRoute(t *testing.T) { |
| + s, sqid := setup(t) |
| + if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| + t.Fatal(err) |
| + } |
| + handAdded := registry.Route{ |
| + Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| + Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| + App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| + Channel: "stable", Hibernate: true, Status: "ready", |
| + Backing: registry.BackingUnit, |
| + } |
| + reg := regWith(t, handAdded) |
| + before := *mustGet(t, reg, "acme.go.tito.io") |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + after := *mustGet(t, reg, "acme.go.tito.io") |
| + if after != before { |
| + t.Errorf("a Backing-owned route was touched: before %+v after %+v", before, after) |
| + } |
| + if after.Provisioned { |
| + t.Error("a carlos-app@ unit tenant was adopted — a record cannot state its Backing") |
| + } |
| +} |
| + |
| +// An adopted route is removable by the pass that adopted it, on the same |
| +// terms as one it minted: the record disappearing is the bucket's own |
| +// decision to retire the instance. |
| +func TestReconcileRemovesAdoptedRouteWhenRecordGoes(t *testing.T) { |
| + s, sqid := setup(t) |
| + if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| + t.Fatal(err) |
| + } |
| + handAdded := registry.Route{ |
| + Host: "acme.go.tito.io", Slug: "acme", Kind: "instance", |
| + Socket: "/run/carlos/acme.go.tito.io.sock", DBPath: "/data/acme.go.tito.io.db", |
| + App: "tito", Account: sqid, Pool: "tito", Region: "ie", |
| + Channel: "stable", Hibernate: true, Status: "ready", |
| + } |
| + reg := regWith(t, handAdded) |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := DeleteRecord(ctx, s, sqid, "tito", "acme.go.tito.io"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt, err := reg.Get("acme.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if rt != nil { |
| + t.Errorf("adopted route survived its record's deletion: %+v", rt) |
| + } |
| +} |
| + |
| +// The other side of that coin, and the one that matters more: a route no |
| +// record ever described is never removed, no matter how many passes run. |
| +// Adoption is the ONLY way this pass acquires removal authority, and this |
| +// test is what stands between a backfill and a retired production route. |
| +func TestReconcileNeverRemovesUnadoptedRoute(t *testing.T) { |
| + s, sqid := setup(t) |
| + // A record for some OTHER host, so the pass runs a full removal phase |
| + // with a non-empty record set rather than bailing early. |
| + if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| + t.Fatal(err) |
| + } |
| + handAdded := registry.Route{ |
| + Host: "console.go.tito.io", Kind: "instance", App: "tito", Account: sqid, |
| + Pool: "tito", Region: "ie", Channel: "stable", Addr: "127.0.0.1:8090", |
| + Unit: "console.service", Status: "ready", |
| + } |
| + reg := regWith(t, handAdded) |
| + // Upsert stamps CreatedAt; read the row back so the final comparison |
| + // isn't comparing against a zero value the row never actually had. |
| + handAdded = *mustGet(t, reg, "console.go.tito.io") |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"} |
| + for i := 0; i < 3; i++ { |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + got, err := reg.Get("console.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got == nil { |
| + t.Fatal("a hand-added route with no record was removed") |
| + } |
| + if got.Provisioned { |
| + t.Error("a hand-added route with no record was adopted") |
| + } |
| + if *got != handAdded { |
| + t.Errorf("a hand-added route with no record was modified: %+v", got) |
| + } |
| +} |
diff --git a/internal/registry/registry.go b/internal/registry/registry.go| index 8fee34533..bf5d47076 100644 |
| --- a/internal/registry/registry.go |
| +++ b/internal/registry/registry.go |
| @@ -1074,6 +1074,35 @@ func (r *Registry) SetSeed(host, seed string) error { |
| return nil |
| } |
| +// SetProvisioned stamps (or clears) the flag that marks a route as the |
| +// instance-provisioning reconcile pass's own — a narrow UPDATE, same |
| +// rationale as SetSeed. It errors when no row matched, the SetChannel |
| +// contract: the pass must notice a row that vanished under it rather than |
| +// silently doing nothing. |
| +// |
| +// This flag is an authority transfer, not a label: it is the single |
| +// condition the reconcile pass's removal phase keys on. Only that pass's |
| +// adoption step (provision.Reconcile, design §4.2) should ever set it, and |
| +// it does so only after matching the row's shape against the record. |
| +func (r *Registry) SetProvisioned(host string, on bool) error { |
| + v := 0 |
| + if on { |
| + v = 1 |
| + } |
| + res, err := r.sql.Exec(`UPDATE routes SET provisioned = ? WHERE host = ?`, v, host) |
| + if err != nil { |
| + return err |
| + } |
| + n, err := res.RowsAffected() |
| + if err != nil { |
| + return err |
| + } |
| + if n == 0 { |
| + return fmt.Errorf("no route for host %q", host) |
| + } |
| + return nil |
| +} |
| + |
| // DueInstances returns hibernatable instance routes whose next_due has |
| // arrived (0 means no schedule and never matches). |
| func (r *Registry) DueInstances(now int64) ([]Route, error) { |
diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go| index b571d12de..e3927f364 100644 |
| --- a/internal/registry/registry_test.go |
| +++ b/internal/registry/registry_test.go |
| @@ -1639,3 +1639,23 @@ func TestValidTunnelNameLengthBoundary(t *testing.T) { |
| t.Errorf("ValidTunnelName(33 chars) = true, want false") |
| } |
| } |
| + |
| +func TestSetProvisioned(t *testing.T) { |
| + r := openTestRegistry(t) |
| + if err := r.Upsert(Route{Host: "a.example.com", Kind: "instance", App: "x", Status: "ready"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := r.SetProvisioned("a.example.com", true); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt, err := r.Get("a.example.com") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !rt.Provisioned { |
| + t.Error("SetProvisioned(true) did not stamp the column") |
| + } |
| + if err := r.SetProvisioned("missing.example.com", true); err == nil { |
| + t.Error("SetProvisioned on an unknown host returned nil; want an error (the SetChannel contract)") |
| + } |
| +} |