amadan

carlos / platform Public

Merge pull request #156 from carlosframework/ha-snapshot

host: registry snapshot to bucket + auto-recovery alarm (HA rung 0/1)
Paul Campbell 58e65c636672373ea7ea1b6cef4800c59c04a07c parent a49a7cfparent c524ed7
9 files changed, +867 −0
  • infra/modules/README.md +1 −0
  • infra/modules/region-host/main.tf +22 −0
  • infra/modules/region-host/variables.tf +6 −0
  • internal/host/agent.go +1 −0
  • internal/host/snapshot_loop.go +222 −0
  • internal/host/snapshot_loop_test.go +280 −0
  • internal/registry/registry.go +159 −0
  • internal/registry/registry_test.go +123 −0
  • internal/registry/snapshot_test.go +53 −0
diff --git a/infra/modules/README.md b/infra/modules/README.md
index 857b58d23..8d43b6a22 100644
--- a/infra/modules/README.md
+++ b/infra/modules/README.md
@@ -67,6 +67,7 @@ bucket, so a wrong-region wake fails loudly at the IAM layer.
| `ssm_parameter_arn` | SSM parameter ARN pattern the host may read, e.g. `arn:aws:ssm:<region>:<account>:parameter/carlos/*`. |
| `ami_name_pattern` | AMI name filter (default `debian-13-arm64-*`). |
| `ami_owner` | AMI owner account (default `136693071363`, Debian). |
+| `enable_auto_recovery` | Arm the EC2 auto-recovery alarm (HA ladder rung 0) — a system-status-check failure recovers the instance onto healthy hardware, same instance id, EIP and volumes. Default `true`; set `false` only where an external supervisor owns recovery. |
Outputs: `eip_public_ip` (point your DNS here), `instance_id`,
`security_group_id` (for sibling services that admit fleet traffic
diff --git a/infra/modules/region-host/main.tf b/infra/modules/region-host/main.tf
index fb939d063..1f5187efe 100644
--- a/infra/modules/region-host/main.tf
+++ b/infra/modules/region-host/main.tf
@@ -9,6 +9,8 @@
# not include this region's bucket — a wrong-region wake fails loudly at
# the IAM layer.
+data "aws_region" "current" {}
+
data "aws_ami" "debian_arm64" {
most_recent = true
owners = [var.ami_owner]
@@ -200,3 +202,23 @@ resource "aws_eip" "host" {
instance = aws_instance.host.id
tags = { Name = var.name }
}
+
+# HA ladder rung 0 (2026-08-12 spec): system-status failure recovers the
+# instance onto healthy hardware, keeping instance id, EIP and volumes.
+resource "aws_cloudwatch_metric_alarm" "host_auto_recover" {
+ count = var.enable_auto_recovery ? 1 : 0
+ alarm_name = "${var.name}-auto-recover"
+ alarm_description = "EC2 system-status-check failure — auto-recovers ${var.name} onto healthy hardware."
+ namespace = "AWS/EC2"
+ metric_name = "StatusCheckFailed_System"
+ statistic = "Maximum"
+ period = 60
+ evaluation_periods = 2
+ threshold = 1
+ comparison_operator = "GreaterThanOrEqualToThreshold"
+ alarm_actions = ["arn:aws:automate:${data.aws_region.current.region}:ec2:recover"]
+ dimensions = {
+ InstanceId = aws_instance.host.id
+ }
+ tags = { Name = "${var.name}-auto-recover" }
+}
diff --git a/infra/modules/region-host/variables.tf b/infra/modules/region-host/variables.tf
index 1570628d8..23e1156e1 100644
--- a/infra/modules/region-host/variables.tf
+++ b/infra/modules/region-host/variables.tf
@@ -77,6 +77,12 @@ variable "route53_zone_ids" {
}
}
+variable "enable_auto_recovery" {
+ description = "Arm an EC2 auto-recovery alarm on the host (HA ladder rung 0)."
+ type = bool
+ default = true
+}
+
variable "enable_cost_explorer" {
description = "Grant ce:GetCostAndUsage so a console on this box can pull the monthly bill for the economics dashboard"
type = bool
diff --git a/internal/host/agent.go b/internal/host/agent.go
index df18b9cb1..09c483b4f 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -304,6 +304,7 @@ func RunAgent(args []string) {
go domainsLoop(reg, peekAcmeCache(args))
go instancesLoop(reg)
go retentionLoop(reg)
+ go registrySnapshotLoop(reg)
// The sweep reads the registry only through this closure — the stamped
// bespoke units to sample beside the carlos- ones, re-read each tick.
go metricsLoop(cgm.Root(), func() []string { return stampedUnitNames(reg) })
diff --git a/internal/host/snapshot_loop.go b/internal/host/snapshot_loop.go
new file mode 100644
index 000000000..f09390829
--- /dev/null
+++ b/internal/host/snapshot_loop.go
@@ -0,0 +1,222 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "math/rand/v2"
+ "os"
+ "path/filepath"
+ "sort"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The cold-standby spec (2026-08-12 HA posture, Part I rung 1): the current
+// snapshot at a fixed key a runbook can name, a sidecar describing it, and a
+// short tail of prior generations. All under control/ beside control/serial;
+// platform boxes write direct S3, so no store-door ACL is involved.
+const (
+ snapshotKey = "control/registry-snapshot.db"
+ snapshotMetaKey = "control/registry-snapshot.json"
+ snapshotGenPrefix = "control/registry-snapshots/"
+ snapshotKeep = 8
+ snapshotStamp = "20060102T150405Z"
+)
+
+// snapshotMeta describes the object at snapshotKey. `Box` is the writing
+// box's label (per-instance hashed machine id) — a restore must check it
+// names the DEAD box, not the standby, or it is restoring the standby's own
+// empty registry. `Fingerprint` carries registry.SnapshotFingerprint (ALL
+// columns), not registry.Fingerprint (edge-reload columns only): it is only
+// ever compared against itself — the loop seeds lastFP from it and skips an
+// unchanged registry — so no reader outside this file depends on which
+// digest it is.
+type snapshotMeta struct {
+ V int `json:"v"`
+ Box string `json:"box"`
+ Fingerprint string `json:"fingerprint"`
+ CreatedAt time.Time `json:"created_at"`
+ Bytes int `json:"bytes"`
+}
+
+// registrySnapshotPass snapshots the registry into the bucket when its
+// snapshot fingerprint moved since *lastFP. One pass; the loop owns cadence.
+// *lastFP is updated only after every write (including pruning) succeeded, so
+// a partial failure is retried whole next tick.
+//
+// The gate is SnapshotFingerprint, NOT Fingerprint: Fingerprint deliberately
+// omits caps, unit, replication, environment, account, canary and
+// provisioned, which are precisely the box-scoped columns only this snapshot
+// can rebuild — gating on it meant a caps grant or a console unit stamp never
+// produced a snapshot at all.
+func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Registry, box string, now time.Time, lastFP *string) error {
+ fp, err := reg.SnapshotFingerprint()
+ if err != nil {
+ return fmt.Errorf("snapshot: fingerprint: %w", err)
+ }
+ if fp == *lastFP {
+ return nil
+ }
+
+ dir, err := os.MkdirTemp("", "carlos-registry-snapshot")
+ if err != nil {
+ return fmt.Errorf("snapshot: %w", err)
+ }
+ defer os.RemoveAll(dir)
+ file := filepath.Join(dir, "registry.db")
+ if err := reg.SnapshotTo(file); err != nil {
+ return fmt.Errorf("snapshot: vacuum: %w", err)
+ }
+ body, err := os.ReadFile(file)
+ if err != nil {
+ return fmt.Errorf("snapshot: %w", err)
+ }
+
+ // Generation first (create-only — a same-second re-run or a clock
+ // step-back can collide on the stamp, in which case Put's ErrCASConflict
+ // is deliberately swallowed and the retained generation is KEPT rather
+ // than replaced). Be precise about what that costs: after a clock
+ // step-back WITH a registry change in between, the retained generation
+ // holds OLDER content while the current key and its sidecar hold the
+ // newer — the two are not equivalent, and we are choosing to keep the
+ // older one under that stamp. What is lost is one intermediate rung of
+ // the generation tail, never the restore target itself: current + sidecar
+ // are written unconditionally below, so a restore still gets the newest
+ // registry. Retrying under a bumped stamp would buy a slightly denser
+ // tail at the cost of a write loop against a lying clock.
+ //
+ // Then current + sidecar (replace with the CAS-retry recipe
+ // usage.UpsertEdgeDay uses), then prune.
+ genKey := snapshotGenPrefix + now.UTC().Format(snapshotStamp) + ".db"
+ if _, err := s.Put(ctx, genKey, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {
+ return fmt.Errorf("snapshot: put %s: %w", genKey, err)
+ }
+ // Overwriting a sidecar another box wrote is the cold-standby clobber
+ // (runbook step 0): a freshly bootstrapped standby whose agent starts
+ // before the operator restores will replace the dead box's snapshot with
+ // its own empty registry, and every self-consistency check downstream
+ // (bytes, created_at) still passes. Nothing here can safely decide whose
+ // registry is authoritative — a legitimate promotion looks identical —
+ // so this does not refuse the write; it makes the moment loud, in the
+ // agent log, where a rehearsal will see it.
+ if prior := readSnapshotMeta(ctx, s); prior != nil && prior.Box != "" && prior.Box != box {
+ log.Printf("agent: registry snapshot: OVERWRITING a snapshot written by box %q (this box is %q, prior created_at %s) — if this box is a standby that has not been restored yet, STOP and preserve the prior snapshot",
+ prior.Box, box, prior.CreatedAt.Format(time.RFC3339))
+ }
+ if err := putReplacing(ctx, s, snapshotKey, body); err != nil {
+ return err
+ }
+ meta := snapshotMeta{V: 1, Box: box, Fingerprint: fp, CreatedAt: now.UTC(), Bytes: len(body)}
+ metaRaw, err := json.Marshal(meta)
+ if err != nil {
+ return fmt.Errorf("snapshot: %w", err)
+ }
+ if err := putReplacing(ctx, s, snapshotMetaKey, metaRaw); err != nil {
+ return err
+ }
+
+ gens, err := s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ return fmt.Errorf("snapshot: list generations: %w", err)
+ }
+ sort.Strings(gens) // stamps are lexically chronological
+ for len(gens) > snapshotKeep {
+ if err := s.Delete(ctx, gens[0]); err != nil {
+ return fmt.Errorf("snapshot: prune %s: %w", gens[0], err)
+ }
+ gens = gens[1:]
+ }
+
+ *lastFP = fp
+ return nil
+}
+
+// seedSnapshotFingerprint reads the sidecar a previous run (or a previous
+// process on this same box) left at snapshotMetaKey and returns its
+// fingerprint, so a freshly started loop's in-memory lastFP starts where the
+// bucket already is rather than empty. Without this, a crash-looping agent
+// would re-snapshot an unchanged registry on every restart and burn through
+// all snapshotKeep generations with identical copies. Any read or parse
+// failure — no sidecar yet, a corrupt object, a store that is down — returns
+// "": the loop then snapshots on its first tick, the same behavior a box
+// that has never snapshotted before gets. This must never fail the loop.
+func seedSnapshotFingerprint(ctx context.Context, s release.Store) string {
+ meta := readSnapshotMeta(ctx, s)
+ if meta == nil {
+ return ""
+ }
+ return meta.Fingerprint
+}
+
+// readSnapshotMeta returns the sidecar currently in the bucket, or nil when
+// there is none, it cannot be read, or it does not parse. Advisory only: both
+// callers treat nil as "no prior snapshot" and carry on, so a store outage
+// must never fail a pass.
+func readSnapshotMeta(ctx context.Context, s release.Store) *snapshotMeta {
+ raw, _, err := s.Get(ctx, snapshotMetaKey)
+ if err != nil {
+ return nil
+ }
+ var meta snapshotMeta
+ if err := json.Unmarshal(raw, &meta); err != nil {
+ return nil
+ }
+ return &meta
+}
+
+// registrySnapshotLoop snapshots the registry to the deployment bucket every
+// 5 minutes + jitter when it changed — rung 1 of the HA ladder (cold
+// standby restores this file). Platform-box job, retentionLoop's rules:
+// skipped on fleet boxes, disabled with a log line when there is no bucket.
+// The fixed control/ keys this writes assume one platform box per deployment
+// bucket (true for every deployment today); a second concurrent writer would
+// interleave current/sidecar writes from two registries.
+func registrySnapshotLoop(reg *registry.Registry) {
+ ctx := context.Background()
+ if shouldSkipPublisherElection(os.Getenv) {
+ log.Printf("agent: registry snapshot loop is a platform-box job — skipped on a fleet box")
+ return
+ }
+ store, err := openDeployStore(ctx)
+ if err != nil {
+ log.Printf("agent: registry snapshot loop disabled: %v", err)
+ return
+ }
+ box := metricsBoxLabel(os.Getenv, readMachineID())
+ log.Printf("agent: registry snapshot loop armed (5min + jitter, %s)", snapshotKey)
+ lastFP := seedSnapshotFingerprint(ctx, store)
+ delay := time.Minute
+ for {
+ time.Sleep(delay)
+ delay = 5*time.Minute + rand.N(time.Minute)
+ if err := registrySnapshotPass(ctx, store, reg, box, time.Now(), &lastFP); err != nil {
+ log.Printf("agent: registry snapshot pass: %v", err)
+ }
+ }
+}
+
+// putReplacing writes key regardless of prior state: create-only when
+// absent, etag-replace when present, re-reading on a lost race. Three
+// attempts, like usage.UpsertEdgeDay.
+func putReplacing(ctx context.Context, s release.Store, key string, body []byte) error {
+ for attempt := 0; attempt < 3; attempt++ {
+ _, etag, err := s.Get(ctx, key)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ etag = ""
+ case err != nil:
+ return fmt.Errorf("snapshot: get %s: %w", key, err)
+ }
+ if _, err := s.Put(ctx, key, body, etag); err == nil {
+ return nil
+ } else if !errors.Is(err, release.ErrCASConflict) {
+ return fmt.Errorf("snapshot: put %s: %w", key, err)
+ }
+ }
+ return fmt.Errorf("snapshot: %s: lost CAS race three times", key)
+}
diff --git a/internal/host/snapshot_loop_test.go b/internal/host/snapshot_loop_test.go
new file mode 100644
index 000000000..f93f87dd9
--- /dev/null
+++ b/internal/host/snapshot_loop_test.go
@@ -0,0 +1,280 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func snapTestRegistry(t *testing.T) *registry.Registry {
+ t.Helper()
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ return reg
+}
+
+func TestSnapshotPassWritesCurrentSidecarAndGeneration(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+ at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC)
+
+ var lastFP string
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at, &lastFP); err != nil {
+ t.Fatal(err)
+ }
+
+ raw, _, err := s.Get(ctx, snapshotKey)
+ if err != nil {
+ t.Fatalf("current snapshot missing: %v", err)
+ }
+ if len(raw) == 0 {
+ t.Fatal("current snapshot is empty")
+ }
+
+ metaRaw, _, err := s.Get(ctx, snapshotMetaKey)
+ if err != nil {
+ t.Fatalf("sidecar missing: %v", err)
+ }
+ var meta snapshotMeta
+ if err := json.Unmarshal(metaRaw, &meta); err != nil {
+ t.Fatal(err)
+ }
+ if meta.V != 1 || meta.Box != "box1" || meta.Fingerprint == "" || meta.Bytes != len(raw) {
+ t.Fatalf("sidecar = %+v (snapshot %d bytes)", meta, len(raw))
+ }
+ if lastFP != meta.Fingerprint {
+ t.Fatalf("lastFP %q not updated to %q", lastFP, meta.Fingerprint)
+ }
+
+ gens, err := s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 1 || gens[0] != snapshotGenPrefix+"20260812T030000Z.db" {
+ t.Fatalf("generations = %v", gens)
+ }
+}
+
+func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+ at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC)
+
+ var lastFP string
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at, &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(5*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err := s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 1 {
+ t.Fatalf("unchanged registry grew generations: %v", gens)
+ }
+
+ // A registry change makes the next pass write again.
+ if err := reg.Upsert(registry.Route{Host: "b.example.com", Kind: "service", Addr: "10.0.0.2:3000", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(10*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err = s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 2 {
+ t.Fatalf("changed registry did not write a new generation: %v", gens)
+ }
+
+ // A caps grant touches no edge-reload column, so registry.Fingerprint
+ // sits still for it — which is why this pass gates on
+ // SnapshotFingerprint instead. Caps and unit stamps are exactly the
+ // box-scoped rows only the snapshot can rebuild; if they did not trigger
+ // a pass, the standby would restore a registry missing them.
+ if err := reg.SetCaps("a.example.com", "", "net_bind_service"); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(15*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err = s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 3 {
+ t.Fatalf("caps-only change did not snapshot: generations = %v (want 3) — a caps grant would never reach the bucket", gens)
+ }
+
+ // Same for a console unit stamp.
+ if err := reg.SetUnit("a.example.com", "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(20*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err = s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 4 {
+ t.Fatalf("unit-only change did not snapshot: generations = %v (want 4)", gens)
+ }
+}
+
+// The sidecar's fingerprint field must be the digest the pass actually gates
+// on, or seedSnapshotFingerprint would seed lastFP with a value the next
+// pass can never match and every restart would re-snapshot.
+func TestSnapshotSidecarCarriesSnapshotFingerprint(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+
+ var lastFP string
+ if err := registrySnapshotPass(ctx, s, reg, "box1", time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ want, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ metaRaw, _, err := s.Get(ctx, snapshotMetaKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var meta snapshotMeta
+ if err := json.Unmarshal(metaRaw, &meta); err != nil {
+ t.Fatal(err)
+ }
+ if meta.Fingerprint != want {
+ t.Fatalf("sidecar fingerprint = %q, want SnapshotFingerprint %q", meta.Fingerprint, want)
+ }
+}
+
+// readSnapshotMeta is what makes the foreign-box overwrite loud (the
+// cold-standby clobber the runbook's step 0 exists to prevent). It must
+// report the prior writer's box, and must degrade to nil rather than
+// erroring when there is no sidecar at all.
+func TestReadSnapshotMetaReportsPriorWriter(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+ at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC)
+
+ if got := readSnapshotMeta(ctx, s); got != nil {
+ t.Fatalf("readSnapshotMeta with no sidecar = %+v, want nil", got)
+ }
+
+ var lastFP string
+ if err := registrySnapshotPass(ctx, s, reg, "dead-box", at, &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ got := readSnapshotMeta(ctx, s)
+ if got == nil || got.Box != "dead-box" {
+ t.Fatalf("readSnapshotMeta = %+v, want Box \"dead-box\"", got)
+ }
+
+ // The standby case: a different box overwrites. The pass does NOT refuse
+ // it (a legitimate promotion is indistinguishable) — it logs — so the
+ // write must still land, now naming the standby.
+ if err := reg.Upsert(registry.Route{Host: "c.example.com", Kind: "service", Addr: "10.0.0.3:3000", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "standby-box", at.Add(5*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ got = readSnapshotMeta(ctx, s)
+ if got == nil || got.Box != "standby-box" {
+ t.Fatalf("after a standby write readSnapshotMeta = %+v, want Box \"standby-box\"", got)
+ }
+}
+
+func TestSnapshotPassPrunesGenerationsBeyondKeep(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+ at := time.Date(2026, 8, 12, 0, 0, 0, 0, time.UTC)
+
+ var lastFP string
+ for i := 0; i < snapshotKeep+3; i++ {
+ // Change the registry each round so no pass skips.
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready", Version: string(rune('a' + i))}); err != nil {
+ t.Fatal(err)
+ }
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(time.Duration(i)*5*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ }
+ gens, err := s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != snapshotKeep {
+ t.Fatalf("kept %d generations, want %d: %v", len(gens), snapshotKeep, gens)
+ }
+ // The survivors must be the newest ones: the earliest kept stamp is round 3's.
+ if gens[0] != snapshotGenPrefix+"20260812T001500Z.db" {
+ t.Fatalf("oldest kept generation = %q, want the round-3 stamp", gens[0])
+ }
+}
+
+func TestSeedSnapshotFingerprintEmptyWhenNoSidecar(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ if got := seedSnapshotFingerprint(context.Background(), s); got != "" {
+ t.Fatalf("seedSnapshotFingerprint = %q, want empty with no sidecar written yet", got)
+ }
+}
+
+// TestSeedSnapshotFingerprintMatchingSidecarSkipsFirstPass proves the
+// registrySnapshotLoop wiring this backs: reading lastFP from the bucket
+// sidecar at loop start (rather than starting it empty in memory) makes the
+// very first pass after a restart a no-op when the registry has not changed
+// since the last snapshot — the crash-loop case addition 1 of the Task 2
+// review exists to fix.
+func TestSeedSnapshotFingerprintMatchingSidecarSkipsFirstPass(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ ctx := context.Background()
+ reg := snapTestRegistry(t)
+ at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC)
+
+ // A prior process ran a pass and wrote the sidecar, then exited (or
+ // crashed) — its in-memory lastFP is gone.
+ var priorFP string
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at, &priorFP); err != nil {
+ t.Fatal(err)
+ }
+
+ // A fresh process seeds lastFP from the bucket instead of starting empty.
+ seeded := seedSnapshotFingerprint(ctx, s)
+ if seeded == "" || seeded != priorFP {
+ t.Fatalf("seeded fingerprint = %q, want %q", seeded, priorFP)
+ }
+
+ lastFP := seeded
+ if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(5*time.Minute), &lastFP); err != nil {
+ t.Fatal(err)
+ }
+ gens, err := s.List(ctx, snapshotGenPrefix)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(gens) != 1 {
+ t.Fatalf("seeded lastFP did not skip the unchanged registry: generations = %v (want 1, no new write)", gens)
+ }
+}
diff --git a/internal/registry/registry.go b/internal/registry/registry.go
index d051d5195..14b7be821 100644
--- a/internal/registry/registry.go
+++ b/internal/registry/registry.go
@@ -13,6 +13,7 @@ import (
"crypto/sha256"
"database/sql"
"fmt"
+ "os"
"regexp"
"strings"
"time"
@@ -1670,3 +1671,161 @@ FROM fleet_boxes ORDER BY fleet, label`)
return fmt.Sprintf("%d:%x", n, h.Sum(nil)), nil
}
+
+// SnapshotFingerprint summarizes the registry for the SNAPSHOT pass, where
+// Fingerprint summarizes it for the edge's reload poll. The two answer
+// different questions and must not be conflated:
+//
+// Fingerprint hashes only the columns a reload cares about, and deliberately
+// EXCLUDES caps, unit, replication, environment, account, canary and
+// provisioned — they change who restarts a process or what a tenant may do,
+// not what the edge serves. That exclusion is correct for reloads and wrong
+// for snapshots: those box-scoped columns are exactly the rows nothing else
+// in the bucket can rebuild, so a cold-standby restore that misses them
+// silently drops caps grants and console unit stamps. Gating the snapshot on
+// Fingerprint meant a caps grant or a unit stamp never triggered a write at
+// all. So SnapshotFingerprint hashes EVERY column of routes, plus accounts,
+// fleets and fleet_boxes — anything the restore has to reproduce.
+//
+// The routes query is `SELECT rowid, ` + routeCols, so a column added to
+// routeCols is covered here automatically (no fourth place to update); the
+// values are scanned generically and written in column order, which is why
+// this needs no per-column typing the way Fingerprint does.
+//
+// Hashing the VACUUM INTO output instead was rejected: SQLite's page image
+// is not byte-deterministic across runs of identical content (freelist and
+// page-ordering effects), so it would report a change on every pass and
+// snapshot forever.
+//
+// Fresh connection, same reason as Fingerprint and List: a long-lived
+// modernc connection can serve a stale WAL snapshot of another process's
+// commits.
+func (r *Registry) SnapshotFingerprint() (string, error) {
+ db, err := sql.Open("sqlite", r.path)
+ if err != nil {
+ return "", err
+ }
+ defer db.Close()
+ if _, err := db.Exec("PRAGMA busy_timeout=5000"); err != nil {
+ return "", err
+ }
+
+ rows, err := db.Query(`SELECT rowid, ` + routeCols + ` FROM routes ORDER BY rowid`)
+ if err != nil {
+ return "", err
+ }
+ defer rows.Close()
+ cols, err := rows.Columns()
+ if err != nil {
+ return "", err
+ }
+ h := sha256.New()
+ n := 0
+ vals := make([]sql.NullString, len(cols))
+ dest := make([]any, len(cols))
+ for i := range vals {
+ dest[i] = &vals[i]
+ }
+ for rows.Next() {
+ if err := rows.Scan(dest...); err != nil {
+ return "", err
+ }
+ for _, v := range vals {
+ // NULL and "" must not hash alike: a column that has never been
+ // written is a different restore than one explicitly cleared.
+ if v.Valid {
+ fmt.Fprintf(h, "%s\x1f", v.String)
+ } else {
+ fmt.Fprint(h, "\x00\x1f")
+ }
+ }
+ fmt.Fprint(h, "\x1e")
+ n++
+ }
+ if err := rows.Err(); err != nil {
+ return "", err
+ }
+
+ // Section markers, same discipline as Fingerprint: a value can never
+ // migrate across sections and cancel another change out.
+ fmt.Fprint(h, "\x1daccounts\x1e")
+ arows, err := db.Query(`SELECT id, sqid, name FROM accounts ORDER BY id`)
+ if err != nil {
+ return "", err
+ }
+ defer arows.Close()
+ for arows.Next() {
+ var id int64
+ var sqid, name string
+ if err := arows.Scan(&id, &sqid, &name); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(h, "%d\x1f%s\x1f%s\x1e", id, sqid, name)
+ }
+ if err := arows.Err(); err != nil {
+ return "", err
+ }
+
+ fmt.Fprint(h, "\x1dfleets\x1e")
+ frows, err := db.Query(`SELECT name, account, bucket, rev FROM fleets ORDER BY name`)
+ if err != nil {
+ return "", err
+ }
+ defer frows.Close()
+ for frows.Next() {
+ var name, account, bucket string
+ var rev int64
+ if err := frows.Scan(&name, &account, &bucket, &rev); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(h, "%s\x1f%s\x1f%s\x1f%d\x1e", name, account, bucket, rev)
+ }
+ if err := frows.Err(); err != nil {
+ return "", err
+ }
+
+ fmt.Fprint(h, "\x1dfleet_boxes\x1e")
+ brows, err := db.Query(`SELECT fleet, label, token_hash, status, rev
+FROM fleet_boxes ORDER BY fleet, label`)
+ if err != nil {
+ return "", err
+ }
+ defer brows.Close()
+ for brows.Next() {
+ var fleet, label, tokenHash, status string
+ var rev int64
+ if err := brows.Scan(&fleet, &label, &tokenHash, &status, &rev); err != nil {
+ return "", err
+ }
+ fmt.Fprintf(h, "%s\x1f%s\x1f%s\x1f%s\x1f%d\x1e", fleet, label, tokenHash, status, rev)
+ }
+ if err := brows.Err(); err != nil {
+ return "", err
+ }
+
+ return fmt.Sprintf("%d:%x", n, h.Sum(nil)), nil
+}
+
+// SnapshotTo writes a consistent single-file copy of the registry to path
+// via VACUUM INTO on a fresh connection — same fresh-connection recipe as
+// Fingerprint (modernc WAL visibility). path must not already exist:
+// modernc's VACUUM INTO silently overwrites an existing file rather than
+// refusing it (unlike upstream SQLite's CLI), so we check first — callers
+// stage into a temp dir. The bound-parameter form (not fmt.Sprintf/%q) is
+// required: %q is Go string escaping, not SQL escaping, so a path
+// containing a backslash gets VACUUM INTO writing to a different on-disk
+// path than the one os.Stat just checked, silently defeating the guard.
+func (r *Registry) SnapshotTo(path string) error {
+ if _, err := os.Stat(path); err == nil {
+ return fmt.Errorf("registry: snapshot target already exists: %s", path)
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ db, err := sql.Open("sqlite", r.path)
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+ _, err = db.Exec("VACUUM INTO ?", path)
+ return err
+}
diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go
index b2b9fe8a6..fdef66c1a 100644
--- a/internal/registry/registry_test.go
+++ b/internal/registry/registry_test.go
@@ -1740,6 +1740,129 @@ func TestFingerprintIgnoresAUnitStamp(t *testing.T) {
}
}
+// The other half of TestFingerprintIgnoresAGrant and
+// TestFingerprintIgnoresAUnitStamp: what the edge is right to ignore, the
+// cold-standby snapshot must NOT ignore. caps and unit are box-scoped rows
+// nothing else in the bucket can rebuild, so if SnapshotFingerprint sat
+// still for them the snapshot pass would never fire and a restore would come
+// back without them. These two tests pin the divergence in both directions
+// at once — Fingerprint still still, SnapshotFingerprint moved.
+func TestSnapshotFingerprintSeesACapsGrant(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{
+ Host: "mail.example.com", Kind: "instance", App: "mail",
+ Addr: "127.0.0.1:8091", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ beforeSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ beforeEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetCaps("mail.example.com", "", "net_bind_service"); err != nil {
+ t.Fatal(err)
+ }
+ afterSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ afterEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if afterSnap == beforeSnap {
+ t.Fatalf("SnapshotFingerprint unchanged across a caps grant (%s) — the snapshot pass would never fire, and a cold-standby restore would silently drop the grant", beforeSnap)
+ }
+ if afterEdge != beforeEdge {
+ t.Fatalf("Fingerprint moved on a caps grant (%s -> %s) — the edge-reload digest must stay indifferent to it", beforeEdge, afterEdge)
+ }
+}
+
+func TestSnapshotFingerprintSeesAUnitStamp(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{
+ Host: "console.example.com", Kind: "instance", App: "console",
+ Addr: "127.0.0.1:8090", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ beforeSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ beforeEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SetUnit("console.example.com", "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ afterSnap, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ afterEdge, err := reg.Fingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if afterSnap == beforeSnap {
+ t.Fatalf("SnapshotFingerprint unchanged across a unit stamp (%s) — a restored console route would come back unstamped and stop auto-cycling", beforeSnap)
+ }
+ if afterEdge != beforeEdge {
+ t.Fatalf("Fingerprint moved on a unit stamp (%s -> %s)", beforeEdge, afterEdge)
+ }
+}
+
+// Determinism: repeated calls over an unchanged registry must agree, or the
+// snapshot pass writes a generation every 5 minutes forever. (This is the
+// property that ruled out hashing the VACUUM INTO page image.)
+func TestSnapshotFingerprintStableAcrossCalls(t *testing.T) {
+ reg := openTestRegistry(t)
+ if err := reg.Upsert(Route{
+ Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready",
+ }); err != nil {
+ t.Fatal(err)
+ }
+ first, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := 0; i < 3; i++ {
+ again, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if again != first {
+ t.Fatalf("SnapshotFingerprint call %d = %s, want %s — an unstable digest snapshots on every pass", i+2, again, first)
+ }
+ }
+}
+
+// An account rename has no route column behind it, and only the snapshot can
+// restore the accounts table — so it has to move the snapshot digest even
+// though no route changed at all.
+func TestSnapshotFingerprintSeesAnAccountWrite(t *testing.T) {
+ reg := openTestRegistry(t)
+ before, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := reg.sql.Exec(`INSERT INTO accounts (id, sqid, name) VALUES (?, ?, ?)`, 4242, "zzz", "later"); err != nil {
+ t.Fatal(err)
+ }
+ after, err := reg.SnapshotFingerprint()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after == before {
+ t.Fatalf("SnapshotFingerprint unchanged across an accounts write (%s)", before)
+ }
+}
+
func TestValidTunnelNameLengthBoundary(t *testing.T) {
ok32 := "01234567890123456789012345678901" // 32 chars
if len(ok32) != 32 {
diff --git a/internal/registry/snapshot_test.go b/internal/registry/snapshot_test.go
new file mode 100644
index 000000000..65d73c1e2
--- /dev/null
+++ b/internal/registry/snapshot_test.go
@@ -0,0 +1,53 @@
+package registry
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestSnapshotToProducesOpenableCopy(t *testing.T) {
+ dir := t.TempDir()
+ reg, err := Open(filepath.Join(dir, "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ if err := reg.Upsert(Route{Host: "a.example.com", Kind: "service", Addr: "10.0.0.1:3000", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+
+ out := filepath.Join(dir, "snap.db")
+ if err := reg.SnapshotTo(out); err != nil {
+ t.Fatal(err)
+ }
+
+ copyReg, err := Open(out)
+ if err != nil {
+ t.Fatalf("open snapshot: %v", err)
+ }
+ defer copyReg.Close()
+ rt, err := copyReg.Get("a.example.com")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rt.Addr != "10.0.0.1:3000" {
+ t.Fatalf("snapshot route addr = %q, want 10.0.0.1:3000", rt.Addr)
+ }
+}
+
+func TestSnapshotToRefusesExistingTarget(t *testing.T) {
+ dir := t.TempDir()
+ reg, err := Open(filepath.Join(dir, "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ out := filepath.Join(dir, "snap.db")
+ if err := os.WriteFile(out, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.SnapshotTo(out); err == nil {
+ t.Fatal("SnapshotTo over an existing file: want error, got nil")
+ }
+}