amadan

carlos / platform Public

host,registry: snapshot fingerprint covers all columns; DR-visibility minors (final review)

registrySnapshotPass gated on registry.Fingerprint, which deliberately
hashes only edge-reload-relevant columns — caps, unit, replication,
environment, account, canary and provisioned are excluded. Those are
exactly the box-scoped rows nothing but the snapshot can rebuild, so a
caps grant or a console unit stamp never triggered a snapshot at all.

Adds registry.SnapshotFingerprint: fresh connection, hashes every column
of routes (SELECT rowid, routeCols — a future column is covered with no
fourth place to update) plus accounts, fleets and fleet_boxes, each under
its own section marker. Hashing the VACUUM INTO page image instead was
rejected as nondeterministic. The pass and the loop's seed now use it;
the sidecar's fingerprint field carries this digest, which is safe
because it is only ever compared against itself.

Also: the pass logs loudly when it overwrites a sidecar written by a
different box (the cold-standby clobber the pinfra runbook's new Step 0
prevents — a rehearsal will see the line); the generation-collision
comment now says plainly that a clock step-back with a registry change
leaves the retained generation holding OLDER content than current +
sidecar, losing one intermediate rung of the tail rather than the
restore target; and enable_auto_recovery is documented in the
region-host variable table.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 4e384f13acf0636d6edb697a9b5d4e53451b3d92 parent 6927152
5 files changed, +416 −10
  • infra/modules/README.md +1 −0
  • internal/host/snapshot_loop.go +57 −10
  • internal/host/snapshot_loop_test.go +101 −0
  • internal/registry/registry.go +134 −0
  • internal/registry/registry_test.go +123 −0
diff --git a/infra/modules/README.md b/infra/modules/README.md
index 857b58d23..8d43b6a22 100644
--- a/infra/modules/README.md
+++ b/infra/modules/README.md
@@ -67,6 +67,7 @@ bucket, so a wrong-region wake fails loudly at the IAM layer.
| `ssm_parameter_arn` | SSM parameter ARN pattern the host may read, e.g. `arn:aws:ssm:<region>:<account>:parameter/carlos/*`. |
| `ami_name_pattern` | AMI name filter (default `debian-13-arm64-*`). |
| `ami_owner` | AMI owner account (default `136693071363`, Debian). |
+| `enable_auto_recovery` | Arm the EC2 auto-recovery alarm (HA ladder rung 0) — a system-status-check failure recovers the instance onto healthy hardware, same instance id, EIP and volumes. Default `true`; set `false` only where an external supervisor owns recovery. |
Outputs: `eip_public_ip` (point your DNS here), `instance_id`,
`security_group_id` (for sibling services that admit fleet traffic
diff --git a/internal/host/snapshot_loop.go b/internal/host/snapshot_loop.go
index 486ca01df..f09390829 100644
--- a/internal/host/snapshot_loop.go
+++ b/internal/host/snapshot_loop.go
@@ -28,6 +28,14 @@ const (
snapshotStamp = "20060102T150405Z"
)
+// snapshotMeta describes the object at snapshotKey. `Box` is the writing
+// box's label (per-instance hashed machine id) — a restore must check it
+// names the DEAD box, not the standby, or it is restoring the standby's own
+// empty registry. `Fingerprint` carries registry.SnapshotFingerprint (ALL
+// columns), not registry.Fingerprint (edge-reload columns only): it is only
+// ever compared against itself — the loop seeds lastFP from it and skips an
+// unchanged registry — so no reader outside this file depends on which
+// digest it is.
type snapshotMeta struct {
V int `json:"v"`
Box string `json:"box"`
@@ -37,11 +45,17 @@ type snapshotMeta struct {
}
// registrySnapshotPass snapshots the registry into the bucket when its
-// fingerprint moved since *lastFP. One pass; the loop owns cadence. *lastFP
-// is updated only after every write (including pruning) succeeded, so a
-// partial failure is retried whole next tick.
+// snapshot fingerprint moved since *lastFP. One pass; the loop owns cadence.
+// *lastFP is updated only after every write (including pruning) succeeded, so
+// a partial failure is retried whole next tick.
+//
+// The gate is SnapshotFingerprint, NOT Fingerprint: Fingerprint deliberately
+// omits caps, unit, replication, environment, account, canary and
+// provisioned, which are precisely the box-scoped columns only this snapshot
+// can rebuild — gating on it meant a caps grant or a console unit stamp never
+// produced a snapshot at all.
func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Registry, box string, now time.Time, lastFP *string) error {
- fp, err := reg.Fingerprint()
+ fp, err := reg.SnapshotFingerprint()
if err != nil {
return fmt.Errorf("snapshot: fingerprint: %w", err)
}
@@ -65,14 +79,35 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re
// Generation first (create-only — a same-second re-run or a clock
// step-back can collide on the stamp, in which case Put's ErrCASConflict
- // is deliberately swallowed: the existing generation for that second is
- // as good as the one we'd have written, so there is nothing to retry),
- // then current + sidecar (replace with the CAS-retry recipe
+ // is deliberately swallowed and the retained generation is KEPT rather
+ // than replaced). Be precise about what that costs: after a clock
+ // step-back WITH a registry change in between, the retained generation
+ // holds OLDER content while the current key and its sidecar hold the
+ // newer — the two are not equivalent, and we are choosing to keep the
+ // older one under that stamp. What is lost is one intermediate rung of
+ // the generation tail, never the restore target itself: current + sidecar
+ // are written unconditionally below, so a restore still gets the newest
+ // registry. Retrying under a bumped stamp would buy a slightly denser
+ // tail at the cost of a write loop against a lying clock.
+ //
+ // Then current + sidecar (replace with the CAS-retry recipe
// usage.UpsertEdgeDay uses), then prune.
genKey := snapshotGenPrefix + now.UTC().Format(snapshotStamp) + ".db"
if _, err := s.Put(ctx, genKey, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {
return fmt.Errorf("snapshot: put %s: %w", genKey, err)
}
+ // Overwriting a sidecar another box wrote is the cold-standby clobber
+ // (runbook step 0): a freshly bootstrapped standby whose agent starts
+ // before the operator restores will replace the dead box's snapshot with
+ // its own empty registry, and every self-consistency check downstream
+ // (bytes, created_at) still passes. Nothing here can safely decide whose
+ // registry is authoritative — a legitimate promotion looks identical —
+ // so this does not refuse the write; it makes the moment loud, in the
+ // agent log, where a rehearsal will see it.
+ if prior := readSnapshotMeta(ctx, s); prior != nil && prior.Box != "" && prior.Box != box {
+ log.Printf("agent: registry snapshot: OVERWRITING a snapshot written by box %q (this box is %q, prior created_at %s) — if this box is a standby that has not been restored yet, STOP and preserve the prior snapshot",
+ prior.Box, box, prior.CreatedAt.Format(time.RFC3339))
+ }
if err := putReplacing(ctx, s, snapshotKey, body); err != nil {
return err
}
@@ -111,15 +146,27 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re
// "": the loop then snapshots on its first tick, the same behavior a box
// that has never snapshotted before gets. This must never fail the loop.
func seedSnapshotFingerprint(ctx context.Context, s release.Store) string {
+ meta := readSnapshotMeta(ctx, s)
+ if meta == nil {
+ return ""
+ }
+ return meta.Fingerprint
+}
+
+// readSnapshotMeta returns the sidecar currently in the bucket, or nil when
+// there is none, it cannot be read, or it does not parse. Advisory only: both
+// callers treat nil as "no prior snapshot" and carry on, so a store outage
+// must never fail a pass.
+func readSnapshotMeta(ctx context.Context, s release.Store) *snapshotMeta {
raw, _, err := s.Get(ctx, snapshotMetaKey)
if err != nil {
- return ""
+ return nil
}
var meta snapshotMeta
if err := json.Unmarshal(raw, &meta); err != nil {
- return ""
+ return nil
}
- return meta.Fingerprint
+ return &meta
}
// registrySnapshotLoop snapshots the registry to the deployment bucket every
diff --git a/internal/host/snapshot_loop_test.go b/internal/host/snapshot_loop_test.go
index e212e0bd9..f93f87dd9 100644
--- a/internal/host/snapshot_loop_test.go
+++ b/internal/host/snapshot_loop_test.go
@@ -102,6 +102,107 @@ func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) {
if len(gens) != 2 {
t.Fatalf("changed registry did not write a new generation: %v", gens)
}
+
+ // A caps grant touches no edge-reload column, so registry.Fingerprint
+ // sits still for it — which is why this pass gates on
+ // SnapshotFingerprint instead. Caps and unit stamps are exactly the
+ // box-scoped rows only the snapshot can rebuild; if they did not trigger
+ // a pass, the standby would restore a registry missing them.
+ if err := reg.SetCaps("a.example.com", "", "net_bind_service"); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(15*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err = s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 3 {
+ t.Fatalf("caps-only change did not snapshot: generations = %v (want 3) — a caps grant would never reach the bucket", gens)
+ }
+
+ // Same for a console unit stamp.
+ if err := reg.SetUnit("a.example.com", "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(20*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err = s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 4 {
+ t.Fatalf("unit-only change did not snapshot: generations = %v (want 4)", gens)
+ }
+}
+
+// The sidecar's fingerprint field must be the digest the pass actually gates
+// on, or seedSnapshotFingerprint would seed lastFP with a value the next
+// pass can never match and every restart would re-snapshot.
+func TestSnapshotSidecarCarriesSnapshotFingerprint(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+
+ var lastFP string
+ if err := registrySnapshotPass(ctx, s, reg, "box1", time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ want, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ metaRaw, _, err := s.Get(ctx, snapshotMetaKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var meta snapshotMeta
+ if err := json.Unmarshal(metaRaw, &meta); err != nil {
+ t.Fatal(err)
+ }
+ if meta.Fingerprint != want {
+ t.Fatalf("sidecar fingerprint = %q, want SnapshotFingerprint %q", meta.Fingerprint, want)
+ }
+}
+
+// readSnapshotMeta is what makes the foreign-box overwrite loud (the
+// cold-standby clobber the runbook's step 0 exists to prevent). It must
+// report the prior writer's box, and must degrade to nil rather than
+// erroring when there is no sidecar at all.
+func TestReadSnapshotMetaReportsPriorWriter(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+ at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC)
+
+ if got := readSnapshotMeta(ctx, s); got != nil {
+ t.Fatalf("readSnapshotMeta with no sidecar = %+v, want nil", got)
+ }
+
+ var lastFP string
+ if err := registrySnapshotPass(ctx, s, reg, "dead-box", at, &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ got := readSnapshotMeta(ctx, s)
+ if got == nil || got.Box != "dead-box" {
+ t.Fatalf("readSnapshotMeta = %+v, want Box \"dead-box\"", got)
+ }
+
+ // The standby case: a different box overwrites. The pass does NOT refuse
+ // it (a legitimate promotion is indistinguishable) — it logs — so the
+ // write must still land, now naming the standby.
+ if err := reg.Upsert(registry.Route{Host: "c.example.com", Kind: "service", Addr: "10.0.0.3:3000", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "standby-box", at.Add(5*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ got = readSnapshotMeta(ctx, s)
+ if got == nil || got.Box != "standby-box" {
+ t.Fatalf("after a standby write readSnapshotMeta = %+v, want Box \"standby-box\"", got)
+ }
}
func TestSnapshotPassPrunesGenerationsBeyondKeep(t *testing.T) {
diff --git a/internal/registry/registry.go b/internal/registry/registry.go
index e62115a8a..0277d5247 100644
--- a/internal/registry/registry.go
+++ b/internal/registry/registry.go
@@ -1297,6 +1297,140 @@ FROM fleet_boxes ORDER BY fleet, label`)
return fmt.Sprintf("%d:%x", n, h.Sum(nil)), nil
}
+// SnapshotFingerprint summarizes the registry for the SNAPSHOT pass, where
+// Fingerprint summarizes it for the edge's reload poll. The two answer
+// different questions and must not be conflated:
+//
+// Fingerprint hashes only the columns a reload cares about, and deliberately
+// EXCLUDES caps, unit, replication, environment, account, canary and
+// provisioned — they change who restarts a process or what a tenant may do,
+// not what the edge serves. That exclusion is correct for reloads and wrong
+// for snapshots: those box-scoped columns are exactly the rows nothing else
+// in the bucket can rebuild, so a cold-standby restore that misses them
+// silently drops caps grants and console unit stamps. Gating the snapshot on
+// Fingerprint meant a caps grant or a unit stamp never triggered a write at
+// all. So SnapshotFingerprint hashes EVERY column of routes, plus accounts,
+// fleets and fleet_boxes — anything the restore has to reproduce.
+//
+// The routes query is `SELECT rowid, ` + routeCols, so a column added to
+// routeCols is covered here automatically (no fourth place to update); the
+// values are scanned generically and written in column order, which is why
+// this needs no per-column typing the way Fingerprint does.
+//
+// Hashing the VACUUM INTO output instead was rejected: SQLite's page image
+// is not byte-deterministic across runs of identical content (freelist and
+// page-ordering effects), so it would report a change on every pass and
+// snapshot forever.
+//
+// Fresh connection, same reason as Fingerprint and List: a long-lived
+// modernc connection can serve a stale WAL snapshot of another process's
+// commits.
+func (r *Registry) SnapshotFingerprint() (string, error) {
+ db, err := sql.Open("sqlite", r.path)
+ if err != nil {
+ return "", err
+ }
+ defer db.Close()
+ if _, err := db.Exec("PRAGMA busy_timeout=5000"); err != nil {
+ return "", err
+ }
+
+ rows, err := db.Query(`SELECT rowid, ` + routeCols + ` FROM routes ORDER BY rowid`)
+ if err != nil {
+ return "", err
+ }
+ defer rows.Close()
+ cols, err := rows.Columns()
+ if err != nil {
+ return "", err
+ }
+ h := sha256.New()
+ n := 0
+ vals := make([]sql.NullString, len(cols))
+ dest := make([]any, len(cols))
+ for i := range vals {
+ dest[i] = &vals[i]
+ }
+ for rows.Next() {
+ if err := rows.Scan(dest...); err != nil {
+ return "", err
+ }
+ for _, v := range vals {
+ // NULL and "" must not hash alike: a column that has never been
+ // written is a different restore than one explicitly cleared.
+ if v.Valid {
+ fmt.Fprintf(h, "%s\x1f", v.String)
+ } else {
+ fmt.Fprint(h, "\x00\x1f")
+ }
+ }
+ fmt.Fprint(h, "\x1e")
+ n++
+ }
+ if err := rows.Err(); err != nil {
+ return "", err
+ }
+
+ // Section markers, same discipline as Fingerprint: a value can never
+ // migrate across sections and cancel another change out.
+ fmt.Fprint(h, "\x1daccounts\x1e")
+ arows, err := db.Query(`SELECT id, sqid, name FROM accounts ORDER BY id`)
+ if err != nil {
+ return "", err
+ }
+ defer arows.Close()
+ for arows.Next() {
+ var id int64
+ var sqid, name string
+ if err := arows.Scan(&id, &sqid, &name); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(h, "%d\x1f%s\x1f%s\x1e", id, sqid, name)
+ }
+ if err := arows.Err(); err != nil {
+ return "", err
+ }
+
+ fmt.Fprint(h, "\x1dfleets\x1e")
+ frows, err := db.Query(`SELECT name, account, bucket, rev FROM fleets ORDER BY name`)
+ if err != nil {
+ return "", err
+ }
+ defer frows.Close()
+ for frows.Next() {
+ var name, account, bucket string
+ var rev int64
+ if err := frows.Scan(&name, &account, &bucket, &rev); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(h, "%s\x1f%s\x1f%s\x1f%d\x1e", name, account, bucket, rev)
+ }
+ if err := frows.Err(); err != nil {
+ return "", err
+ }
+
+ fmt.Fprint(h, "\x1dfleet_boxes\x1e")
+ brows, err := db.Query(`SELECT fleet, label, token_hash, status, rev
+FROM fleet_boxes ORDER BY fleet, label`)
+ if err != nil {
+ return "", err
+ }
+ defer brows.Close()
+ for brows.Next() {
+ var fleet, label, tokenHash, status string
+ var rev int64
+ if err := brows.Scan(&fleet, &label, &tokenHash, &status, &rev); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(h, "%s\x1f%s\x1f%s\x1f%s\x1f%d\x1e", fleet, label, tokenHash, status, rev)
+ }
+ if err := brows.Err(); err != nil {
+ return "", err
+ }
+
+ return fmt.Sprintf("%d:%x", n, h.Sum(nil)), nil
+}
+
// SnapshotTo writes a consistent single-file copy of the registry to path
// via VACUUM INTO on a fresh connection — same fresh-connection recipe as
// Fingerprint (modernc WAL visibility). path must not already exist:
diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go
index c537e3415..4fa70f2fb 100644
--- a/internal/registry/registry_test.go
+++ b/internal/registry/registry_test.go
@@ -1626,6 +1626,129 @@ func TestFingerprintIgnoresAUnitStamp(t *testing.T) {
}
}
+// The other half of TestFingerprintIgnoresAGrant and
+// TestFingerprintIgnoresAUnitStamp: what the edge is right to ignore, the
+// cold-standby snapshot must NOT ignore. caps and unit are box-scoped rows
+// nothing else in the bucket can rebuild, so if SnapshotFingerprint sat
+// still for them the snapshot pass would never fire and a restore would come
+// back without them. These two tests pin the divergence in both directions
+// at once — Fingerprint still still, SnapshotFingerprint moved.
+func TestSnapshotFingerprintSeesACapsGrant(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{
+ Host: "mail.example.com", Kind: "instance", App: "mail",
+ Addr: "127.0.0.1:8091", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ beforeSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ beforeEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetCaps("mail.example.com", "", "net_bind_service"); err != nil {
+ t.Fatal(err)
+ }
+ afterSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ afterEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if afterSnap == beforeSnap {
+ t.Fatalf("SnapshotFingerprint unchanged across a caps grant (%s) — the snapshot pass would never fire, and a cold-standby restore would silently drop the grant", beforeSnap)
+ }
+ if afterEdge != beforeEdge {
+ t.Fatalf("Fingerprint moved on a caps grant (%s -> %s) — the edge-reload digest must stay indifferent to it", beforeEdge, afterEdge)
+ }
+}
+
+func TestSnapshotFingerprintSeesAUnitStamp(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{
+ Host: "console.example.com", Kind: "instance", App: "console",
+ Addr: "127.0.0.1:8090", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ beforeSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ beforeEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetUnit("console.example.com", "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ afterSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ afterEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if afterSnap == beforeSnap {
+ t.Fatalf("SnapshotFingerprint unchanged across a unit stamp (%s) — a restored console route would come back unstamped and stop auto-cycling", beforeSnap)
+ }
+ if afterEdge != beforeEdge {
+ t.Fatalf("Fingerprint moved on a unit stamp (%s -> %s)", beforeEdge, afterEdge)
+ }
+}
+
+// Determinism: repeated calls over an unchanged registry must agree, or the
+// snapshot pass writes a generation every 5 minutes forever. (This is the
+// property that ruled out hashing the VACUUM INTO page image.)
+func TestSnapshotFingerprintStableAcrossCalls(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{
+ Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ first, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := 0; i < 3; i++ {
+ again, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if again != first {
+ t.Fatalf("SnapshotFingerprint call %d = %s, want %s — an unstable digest snapshots on every pass", i+2, again, first)
+ }
+ }
+}
+
+// An account rename has no route column behind it, and only the snapshot can
+// restore the accounts table — so it has to move the snapshot digest even
+// though no route changed at all.
+func TestSnapshotFingerprintSeesAnAccountWrite(t *testing.T) {
+ reg := openTestRegistry(t)
+ before, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := reg.sql.Exec(`INSERT INTO accounts (id, sqid, name) VALUES (?, ?, ?)`, 4242, "zzz", "later"); err != nil {
+ t.Fatal(err)
+ }
+ after, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after == before {
+ t.Fatalf("SnapshotFingerprint unchanged across an accounts write (%s)", before)
+ }
+}
+
func TestValidTunnelNameLengthBoundary(t *testing.T) {
ok32 := "01234567890123456789012345678901" // 32 chars
if len(ok32) != 32 {