carlos / platform Public

host: per-box registry snapshot keys — boxes never contend (#158)

Replace the fixed control/registry-snapshot.{db,json} and
control/registry-snapshots/ keys with per-box functions
(snapshotDBKey, snapshotMetaKey, snapshotGenPrefix), so N platform
boxes sharing one deployment bucket each snapshot under their own
name instead of overwriting each other every 5 minutes.

seedSnapshotFingerprint and readSnapshotMeta now take the box name
and read that box's own sidecar; the generation prune lists only
that box's prefix. The pre-multi-region fixed keys are abandoned in
place, not migrated. The foreign-box overwrite log guard is kept
as-is — unreachable for differently-named boxes now that keys don't
collide, but still relevant to the runbook's same-named-reuse
restore path.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell pushed by paul@keymail.dev 2 months ago b466af44437dae9b37b3a746f62c97d27a51fa6d parent 13e4d45

2 files changed, +112 −55

internal/host/snapshot_loop.go +37 −28
internal/host/snapshot_loop_test.go +75 −27

diff --git a/internal/host/snapshot_loop.go b/internal/host/snapshot_loop.go

index f09390829..247cd541b 100644

--- a/internal/host/snapshot_loop.go

+++ b/internal/host/snapshot_loop.go

@@ -17,18 +17,27 @@ import (

)

// The cold-standby spec (2026-08-12 HA posture, Part I rung 1): the current

-// snapshot at a fixed key a runbook can name, a sidecar describing it, and a

-// short tail of prior generations. All under control/ beside control/serial;

-// platform boxes write direct S3, so no store-door ACL is involved.

+// snapshot at a per-box key a runbook can name (snapshotDBKey), a sidecar

+// describing it, and a short tail of prior generations. All under control/

+// beside control/serial; platform boxes write direct S3, so no store-door

+// ACL is involved.

const (

- snapshotKey = "control/registry-snapshot.db"

- snapshotMetaKey = "control/registry-snapshot.json"

- snapshotGenPrefix = "control/registry-snapshots/"

- snapshotKeep = 8

- snapshotStamp = "20060102T150405Z"

+ snapshotKeep = 8

+ snapshotStamp = "20060102T150405Z"

)

-// snapshotMeta describes the object at snapshotKey. `Box` is the writing

+// Per-box keys (spec 2026-08-20 Part 1, closes #158): every platform box

+// snapshots ITS OWN registry under its own name; boxes never contend.

+// The pre-multi-region fixed keys ("control/registry-snapshot.db" etc.)

+// are still present in production buckets until both deployments roll —

+// they are abandoned, not rewritten; deletion is a rollout step, not code.

+func snapshotDBKey(box string) string { return "control/registry-snapshot/" + box + ".db" }

+func snapshotMetaKey(box string) string { return "control/registry-snapshot/" + box + ".json" }

+func snapshotGenPrefix(box string) string {

+ return "control/registry-snapshots/" + box + "/"

+// snapshotMeta describes the object at snapshotDBKey(box). `Box` is the writing

// box's label (per-instance hashed machine id) — a restore must check it

// names the DEAD box, not the standby, or it is restoring the standby's own

// empty registry. `Fingerprint` carries registry.SnapshotFingerprint (ALL

@@ -92,7 +101,7 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re

// Then current + sidecar (replace with the CAS-retry recipe

// usage.UpsertEdgeDay uses), then prune.

- genKey := snapshotGenPrefix + now.UTC().Format(snapshotStamp) + ".db"

+ genKey := snapshotGenPrefix(box) + now.UTC().Format(snapshotStamp) + ".db"

if _, err := s.Put(ctx, genKey, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {

return fmt.Errorf("snapshot: put %s: %w", genKey, err)

}

@@ -104,11 +113,11 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re

// registry is authoritative — a legitimate promotion looks identical —

// so this does not refuse the write; it makes the moment loud, in the

// agent log, where a rehearsal will see it.

- if prior := readSnapshotMeta(ctx, s); prior != nil && prior.Box != "" && prior.Box != box {

+ if prior := readSnapshotMeta(ctx, s, box); prior != nil && prior.Box != "" && prior.Box != box {

log.Printf("agent: registry snapshot: OVERWRITING a snapshot written by box %q (this box is %q, prior created_at %s) — if this box is a standby that has not been restored yet, STOP and preserve the prior snapshot",

prior.Box, box, prior.CreatedAt.Format(time.RFC3339))

}

- if err := putReplacing(ctx, s, snapshotKey, body); err != nil {

+ if err := putReplacing(ctx, s, snapshotDBKey(box), body); err != nil {

return err

}

meta := snapshotMeta{V: 1, Box: box, Fingerprint: fp, CreatedAt: now.UTC(), Bytes: len(body)}

@@ -116,11 +125,11 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re

if err != nil {

return fmt.Errorf("snapshot: %w", err)

}

- if err := putReplacing(ctx, s, snapshotMetaKey, metaRaw); err != nil {

+ if err := putReplacing(ctx, s, snapshotMetaKey(box), metaRaw); err != nil {

return err

}

- gens, err := s.List(ctx, snapshotGenPrefix)

+ gens, err := s.List(ctx, snapshotGenPrefix(box))

if err != nil {

return fmt.Errorf("snapshot: list generations: %w", err)

}

@@ -137,7 +146,7 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re

}

// seedSnapshotFingerprint reads the sidecar a previous run (or a previous

-// process on this same box) left at snapshotMetaKey and returns its

+// process on this same box) left at snapshotMetaKey(box) and returns its

// fingerprint, so a freshly started loop's in-memory lastFP starts where the

// bucket already is rather than empty. Without this, a crash-looping agent

// would re-snapshot an unchanged registry on every restart and burn through

@@ -145,20 +154,20 @@ func registrySnapshotPass(ctx context.Context, s release.Store, reg *registry.Re

// failure — no sidecar yet, a corrupt object, a store that is down — returns

// "": the loop then snapshots on its first tick, the same behavior a box

// that has never snapshotted before gets. This must never fail the loop.

-func seedSnapshotFingerprint(ctx context.Context, s release.Store) string {

- meta := readSnapshotMeta(ctx, s)

+func seedSnapshotFingerprint(ctx context.Context, s release.Store, box string) string {

+ meta := readSnapshotMeta(ctx, s, box)

if meta == nil {

return ""

}

return meta.Fingerprint

}

-// readSnapshotMeta returns the sidecar currently in the bucket, or nil when

-// there is none, it cannot be read, or it does not parse. Advisory only: both

-// callers treat nil as "no prior snapshot" and carry on, so a store outage

-// must never fail a pass.

-func readSnapshotMeta(ctx context.Context, s release.Store) *snapshotMeta {

- raw, _, err := s.Get(ctx, snapshotMetaKey)

+// readSnapshotMeta returns the sidecar currently in the bucket at this box's

+// own key, or nil when there is none, it cannot be read, or it does not

+// parse. Advisory only: both callers treat nil as "no prior snapshot" and

+// carry on, so a store outage must never fail a pass.

+func readSnapshotMeta(ctx context.Context, s release.Store, box string) *snapshotMeta {

+ raw, _, err := s.Get(ctx, snapshotMetaKey(box))

if err != nil {

return nil

}

@@ -173,9 +182,9 @@ func readSnapshotMeta(ctx context.Context, s release.Store) *snapshotMeta {

// 5 minutes + jitter when it changed — rung 1 of the HA ladder (cold

// standby restores this file). Platform-box job, retentionLoop's rules:

// skipped on fleet boxes, disabled with a log line when there is no bucket.

-// The fixed control/ keys this writes assume one platform box per deployment

-// bucket (true for every deployment today); a second concurrent writer would

-// interleave current/sidecar writes from two registries.

+// Per-box keys (spec 2026-08-20 Part 1, closes #158): each platform box

+// writes under its own name, so N boxes sharing one deployment bucket never

+// interleave each other's current/sidecar writes.

func registrySnapshotLoop(reg *registry.Registry) {

ctx := context.Background()

if shouldSkipPublisherElection(os.Getenv) {

@@ -188,8 +197,8 @@ func registrySnapshotLoop(reg *registry.Registry) {

return

}

box := metricsBoxLabel(os.Getenv, readMachineID())

- log.Printf("agent: registry snapshot loop armed (5min + jitter, %s)", snapshotKey)

- lastFP := seedSnapshotFingerprint(ctx, store)

+ log.Printf("agent: registry snapshot loop armed (5min + jitter, %s)", snapshotDBKey(box))

+ lastFP := seedSnapshotFingerprint(ctx, store, box)

delay := time.Minute

for {

time.Sleep(delay)

diff --git a/internal/host/snapshot_loop_test.go b/internal/host/snapshot_loop_test.go

index f93f87dd9..a762be914 100644

--- a/internal/host/snapshot_loop_test.go

+++ b/internal/host/snapshot_loop_test.go

@@ -35,7 +35,7 @@ func TestSnapshotPassWritesCurrentSidecarAndGeneration(t *testing.T) {

t.Fatal(err)

}

- raw, _, err := s.Get(ctx, snapshotKey)

+ raw, _, err := s.Get(ctx, snapshotDBKey("box1"))

if err != nil {

t.Fatalf("current snapshot missing: %v", err)

}

@@ -43,7 +43,7 @@ func TestSnapshotPassWritesCurrentSidecarAndGeneration(t *testing.T) {

t.Fatal("current snapshot is empty")

}

- metaRaw, _, err := s.Get(ctx, snapshotMetaKey)

+ metaRaw, _, err := s.Get(ctx, snapshotMetaKey("box1"))

if err != nil {

t.Fatalf("sidecar missing: %v", err)

}

@@ -58,11 +58,11 @@ func TestSnapshotPassWritesCurrentSidecarAndGeneration(t *testing.T) {

t.Fatalf("lastFP %q not updated to %q", lastFP, meta.Fingerprint)

}

- gens, err := s.List(ctx, snapshotGenPrefix)

+ gens, err := s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}

- if len(gens) != 1 || gens[0] != snapshotGenPrefix+"20260812T030000Z.db" {

+ if len(gens) != 1 || gens[0] != snapshotGenPrefix("box1")+"20260812T030000Z.db" {

t.Fatalf("generations = %v", gens)

}

@@ -80,7 +80,7 @@ func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) {

if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(5*time.Minute), &lastFP); err != nil {

t.Fatal(err)

}

- gens, err := s.List(ctx, snapshotGenPrefix)

+ gens, err := s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}

@@ -95,7 +95,7 @@ func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) {

if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(10*time.Minute), &lastFP); err != nil {

t.Fatal(err)

}

- gens, err = s.List(ctx, snapshotGenPrefix)

+ gens, err = s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}

@@ -114,7 +114,7 @@ func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) {

if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(15*time.Minute), &lastFP); err != nil {

t.Fatal(err)

}

- gens, err = s.List(ctx, snapshotGenPrefix)

+ gens, err = s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}

@@ -129,7 +129,7 @@ func TestSnapshotPassSkipsWhenFingerprintUnchanged(t *testing.T) {

if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(20*time.Minute), &lastFP); err != nil {

t.Fatal(err)

}

- gens, err = s.List(ctx, snapshotGenPrefix)

+ gens, err = s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}

@@ -154,7 +154,7 @@ func TestSnapshotSidecarCarriesSnapshotFingerprint(t *testing.T) {

if err != nil {

t.Fatal(err)

}

- metaRaw, _, err := s.Get(ctx, snapshotMetaKey)

+ metaRaw, _, err := s.Get(ctx, snapshotMetaKey("box1"))

if err != nil {

t.Fatal(err)

}

@@ -167,17 +167,19 @@ func TestSnapshotSidecarCarriesSnapshotFingerprint(t *testing.T) {

}

-// readSnapshotMeta is what makes the foreign-box overwrite loud (the

-// cold-standby clobber the runbook's step 0 exists to prevent). It must

-// report the prior writer's box, and must degrade to nil rather than

-// erroring when there is no sidecar at all.

+// readSnapshotMeta is box-scoped: it must report the prior writer at THIS

+// box's own key, degrade to nil when there is no sidecar there yet, and

+// never see another box's sidecar at all — per-box keys is exactly what

+// makes a foreign-differently-named box's write harmless (it lands under a

+// different key entirely, unlike the pre-multi-region shared key this

+// guard was written against).

func TestReadSnapshotMetaReportsPriorWriter(t *testing.T) {

s := release.NewDirStore(t.TempDir())

ctx := context.Background()

reg := snapTestRegistry(t)

at := time.Date(2026, 8, 12, 3, 0, 0, 0, time.UTC)

- if got := readSnapshotMeta(ctx, s); got != nil {

+ if got := readSnapshotMeta(ctx, s, "dead-box"); got != nil {

t.Fatalf("readSnapshotMeta with no sidecar = %+v, want nil", got)

}

@@ -185,23 +187,31 @@ func TestReadSnapshotMetaReportsPriorWriter(t *testing.T) {

if err := registrySnapshotPass(ctx, s, reg, "dead-box", at, &lastFP); err != nil {

t.Fatal(err)

}

- got := readSnapshotMeta(ctx, s)

+ got := readSnapshotMeta(ctx, s, "dead-box")

if got == nil || got.Box != "dead-box" {

t.Fatalf("readSnapshotMeta = %+v, want Box \"dead-box\"", got)

}

- // The standby case: a different box overwrites. The pass does NOT refuse

- // it (a legitimate promotion is indistinguishable) — it logs — so the

- // write must still land, now naming the standby.

+ // A different box name never sees dead-box's sidecar — per-box keys

+ // isolate them, unlike the pre-multi-region shared key.

+ if got := readSnapshotMeta(ctx, s, "standby-box"); got != nil {

+ t.Fatalf("readSnapshotMeta(\"standby-box\") = %+v, want nil (no cross-box read)", got)

+ }

+ // The runbook's restore path: a rebuilt standby deliberately reuses the

+ // dead box's name, so its pass writes to the SAME key. The write must

+ // still land (the pass never refuses it — a legitimate promotion looks

+ // identical from here), and readSnapshotMeta for that name reports the

+ // newest sidecar.

if err := reg.Upsert(registry.Route{Host: "c.example.com", Kind: "service", Addr: "10.0.0.3:3000", Status: "ready"}); err != nil {

t.Fatal(err)

}

- if err := registrySnapshotPass(ctx, s, reg, "standby-box", at.Add(5*time.Minute), &lastFP); err != nil {

+ if err := registrySnapshotPass(ctx, s, reg, "dead-box", at.Add(5*time.Minute), &lastFP); err != nil {

t.Fatal(err)

}

- got = readSnapshotMeta(ctx, s)

- if got == nil || got.Box != "standby-box" {

- t.Fatalf("after a standby write readSnapshotMeta = %+v, want Box \"standby-box\"", got)

+ got = readSnapshotMeta(ctx, s, "dead-box")

+ if got == nil || !got.CreatedAt.Equal(at.Add(5*time.Minute)) {

+ t.Fatalf("after the reused-name write readSnapshotMeta = %+v, want CreatedAt %s", got, at.Add(5*time.Minute))

}

@@ -221,7 +231,7 @@ func TestSnapshotPassPrunesGenerationsBeyondKeep(t *testing.T) {

t.Fatal(err)

}

- gens, err := s.List(ctx, snapshotGenPrefix)

+ gens, err := s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}

@@ -229,14 +239,52 @@ func TestSnapshotPassPrunesGenerationsBeyondKeep(t *testing.T) {

t.Fatalf("kept %d generations, want %d: %v", len(gens), snapshotKeep, gens)

}

// The survivors must be the newest ones: the earliest kept stamp is round 3's.

- if gens[0] != snapshotGenPrefix+"20260812T001500Z.db" {

+ if gens[0] != snapshotGenPrefix("box1")+"20260812T001500Z.db" {

t.Fatalf("oldest kept generation = %q, want the round-3 stamp", gens[0])

}

+func TestSnapshotKeysArePerBox(t *testing.T) {

+ if got, want := snapshotDBKey("annie"), "control/registry-snapshot/annie.db"; got != want {

+ t.Fatalf("db key = %q, want %q", got, want)

+ }

+ if got, want := snapshotMetaKey("annie"), "control/registry-snapshot/annie.json"; got != want {

+ t.Fatalf("meta key = %q, want %q", got, want)

+ }

+ if got, want := snapshotGenPrefix("annie"), "control/registry-snapshots/annie/"; got != want {

+ t.Fatalf("gen prefix = %q, want %q", got, want)

+ }

+func TestTwoBoxesSnapshotWithoutColliding(t *testing.T) {

+ s := release.NewDirStore(t.TempDir())

+ regA, regB := snapTestRegistry(t), snapTestRegistry(t)

+ var fpA, fpB string

+ now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)

+ if err := registrySnapshotPass(context.Background(), s, regA, "annie", now, &fpA); err != nil {

+ t.Fatal(err)

+ }

+ if err := registrySnapshotPass(context.Background(), s, regB, "betty", now, &fpB); err != nil {

+ t.Fatal(err)

+ }

+ for _, key := range []string{

+ "control/registry-snapshot/annie.db", "control/registry-snapshot/annie.json",

+ "control/registry-snapshot/betty.db", "control/registry-snapshot/betty.json",

+ } {

+ if _, _, err := s.Get(context.Background(), key); err != nil {

+ t.Fatalf("missing %s: %v", key, err)

+ }

+ // betty's pass must not have disturbed annie's generations.

+ gens, err := s.List(context.Background(), "control/registry-snapshots/annie/")

+ if err != nil || len(gens) != 1 {

+ t.Fatalf("annie generations = %v, %v; want exactly 1", gens, err)

+ }

func TestSeedSnapshotFingerprintEmptyWhenNoSidecar(t *testing.T) {

s := release.NewDirStore(t.TempDir())

- if got := seedSnapshotFingerprint(context.Background(), s); got != "" {

+ if got := seedSnapshotFingerprint(context.Background(), s, "box1"); got != "" {

t.Fatalf("seedSnapshotFingerprint = %q, want empty with no sidecar written yet", got)

}

@@ -261,7 +309,7 @@ func TestSeedSnapshotFingerprintMatchingSidecarSkipsFirstPass(t *testing.T) {

}

// A fresh process seeds lastFP from the bucket instead of starting empty.

- seeded := seedSnapshotFingerprint(ctx, s)

+ seeded := seedSnapshotFingerprint(ctx, s, "box1")

if seeded == "" || seeded != priorFP {

t.Fatalf("seeded fingerprint = %q, want %q", seeded, priorFP)

}

@@ -270,7 +318,7 @@ func TestSeedSnapshotFingerprintMatchingSidecarSkipsFirstPass(t *testing.T) {

if err := registrySnapshotPass(ctx, s, reg, "box1", at.Add(5*time.Minute), &lastFP); err != nil {

t.Fatal(err)

}

- gens, err := s.List(ctx, snapshotGenPrefix)

+ gens, err := s.List(ctx, snapshotGenPrefix("box1"))

if err != nil {

t.Fatal(err)

}