amadan

carlos / platform Public

provision: reconcile converges Channel, Environment and explicit Hibernate

Closes the divergence half of #130: a record whose channel moved now moves
the route row, so the member-visible state and the box state cannot
disagree about a column the bucket owns.

Shape changes (kind/addr/unit) are logged, not converged in place — that
would re-derive socket and DB paths under a running process.

Design: docs/superpowers/specs/2026-08-10-route-records-design.md §4.1
Paul Campbell 826d8eac47484a355fd42e8fe5fb9b68b175627b parent 78e60d0
2 files changed, +156 −0
  • internal/provision/reconcile.go +31 −0
  • internal/provision/reconcile_test.go +125 −0
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index dfcf51b66..8aaa40341 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -270,6 +270,30 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
}
log.Printf("provision: minted %s route %s (app %q acct %q)", nr.Kind, rec.Host, rec.App, rec.Account)
case existing.Provisioned && existing.App == rec.App && existing.Account == rec.Account:
+ // The columns this record owns (design §4.1, issue #130). Narrow
+ // UPDATEs, never a re-Upsert: a read-modify-write racing a
+ // deploy's SetVersions or the activator's SetNextDue would
+ // clobber those, and Upsert cannot express "clear Environment"
+ // at all (it preserves a non-empty value by design).
+ if rec.Channel != "" && existing.Channel != rec.Channel {
+ if err := reg.SetChannel(rec.Host, rec.Channel); err != nil {
+ log.Printf("provision: converge channel on %s: %v", rec.Host, err)
+ } else {
+ log.Printf("provision: converged %s channel %s -> %s", rec.Host, existing.Channel, rec.Channel)
+ }
+ }
+ if existing.Environment != rec.Environment {
+ if err := reg.SetEnvironment(rec.Host, rec.Environment); err != nil {
+ log.Printf("provision: converge environment on %s: %v", rec.Host, err)
+ }
+ }
+ // Absent means "this record does not speak to hibernation" —
+ // never a derivation stamped over an operator's own setting.
+ if rec.Hibernate != nil && existing.Hibernate != *rec.Hibernate {
+ if err := reg.SetHibernate(rec.Host, *rec.Hibernate); err != nil {
+ log.Printf("provision: converge hibernate on %s: %v", rec.Host, err)
+ }
+ }
// seedIsKnown guards against stamping a transient manifest-read
// failure's "" fallback over a working seed — see seedFor.
if seedIsKnown && existing.Seed != seed {
@@ -277,6 +301,13 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
log.Printf("provision: converge seed on %s: %v", rec.Host, err)
}
}
+ if want := routeFor(rec, cfg.DataDir, seed); want.Kind != existing.Kind ||
+ want.Addr != existing.Addr || want.Unit != existing.Unit ||
+ want.Socket != existing.Socket || want.DBPath != existing.DBPath {
+ log.Printf("provision: %s: record describes a different route SHAPE than the row "+
+ "(kind %q/%q addr %q/%q unit %q/%q) — not converged in place; `carlos remove %s` and let the next pass re-mint",
+ rec.Host, want.Kind, existing.Kind, want.Addr, existing.Addr, want.Unit, existing.Unit, rec.Host)
+ }
default:
log.Printf("provision: %s exists but was not minted by this pass (app %q acct %q) — leaving it alone",
rec.Host, rec.App, rec.Account)
diff --git a/internal/provision/reconcile_test.go b/internal/provision/reconcile_test.go
index 95bcf388f..d94fc7378 100644
--- a/internal/provision/reconcile_test.go
+++ b/internal/provision/reconcile_test.go
@@ -85,6 +85,15 @@ func putReplace(t *testing.T, s release.Store, key string, body []byte) {
}
}
+func mustJSON(t *testing.T, v any) []byte {
+ t.Helper()
+ b, err := json.Marshal(v)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
// setup writes a valid account manifest (id acctID) and that account's
// "tito" app instances manifest (domain go.tito.io, pool tito, seed
// ["seed"]) — the fixture every test case below builds records on top of.
@@ -289,6 +298,122 @@ func TestReconcileConvergesSeed(t *testing.T) {
}
}
+// Issue #130, as it happened in production on 2026-08-09: a record created
+// on stable, deleted and recreated on edge, and the route row left on
+// stable — a route pointing at a channel with no pointer, so every wake
+// failed "no adoptable head" while carlos channels cheerfully showed edge
+// promoted. Reconcile created from a record and never converged to one.
+func TestReconcileConvergesChannel(t *testing.T) {
+ s, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ before := *mustGet(t, reg, "acme.go.tito.io")
+
+ rec.Channel = "edge"
+ putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec))
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ after := *mustGet(t, reg, "acme.go.tito.io")
+ if after.Channel != "edge" {
+ t.Errorf("channel = %q, want edge — the record moved and the row did not", after.Channel)
+ }
+ after.Channel = before.Channel
+ if after != before {
+ t.Errorf("converge touched more than Channel: before %+v after %+v", before, after)
+ }
+
+ // The fleet runs on canary/* channels today (pinned in
+ // internal/release/laddercheck_test.go); a converge that could not carry
+ // one would stop deploys.
+ rec.Channel = "canary/rehearsal"
+ putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec))
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, "acme.go.tito.io").Channel; got != "canary/rehearsal" {
+ t.Errorf("channel = %q, want canary/rehearsal", got)
+ }
+}
+
+func TestReconcileConvergesEnvironment(t *testing.T) {
+ s, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+
+ rec.Environment = "staging"
+ putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec))
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, "acme.go.tito.io").Environment; got != "staging" {
+ t.Errorf("environment = %q, want staging", got)
+ }
+
+ // Clearing must reach the row too. Upsert deliberately PRESERVES a
+ // non-empty environment (it cannot tell "untouched" from "cleared"), so
+ // the clear has to go through the narrow setter — which is exactly why
+ // this converge does not re-Upsert.
+ rec.Environment = ""
+ putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec))
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, "acme.go.tito.io").Environment; got != "" {
+ t.Errorf("environment = %q, want it cleared", got)
+ }
+}
+
+// An explicit Hibernate converges; an absent one never does. Absent means
+// "the record does not speak to this", and a pass that stamped its
+// derivation over an operator's deliberate SetHibernate would be the silent
+// stomp this design exists to end.
+func TestReconcileConvergesHibernateOnlyWhenExplicit(t *testing.T) {
+ s, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetHibernate("acme.go.tito.io", false); err != nil {
+ t.Fatal(err)
+ }
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if mustGet(t, reg, "acme.go.tito.io").Hibernate {
+ t.Error("an absent Hibernate re-asserted its derivation over an operator's setting")
+ }
+
+ on := true
+ rec.Hibernate = &on
+ putReplace(t, s, RecordKey(sqid, "tito", rec.Host), mustJSON(t, rec))
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if !mustGet(t, reg, "acme.go.tito.io").Hibernate {
+ t.Error("an explicit Hibernate did not converge")
+ }
+}
+
// 5. Conflict.
func TestReconcileLeavesConflictAlone(t *testing.T) {
s, sqid := setup(t)