amadan
litestream: unit-tenant DB trees, and refuse to shrink the replication set
Two changes that together let `carlos ops litestream` take over the boxes where
the config is still hand-maintained.
TREES. A unit tenant's "database" is sometimes a tree, not a file: kass on the
flagship keeps registry.db plus an instances/ directory that grows as the
tenant creates instances. A single -db path cannot say that, which is the
stated reason that box's config is still hand-written ("carlos ops litestream
takes over once nested layouts are supported"). A BackingUnit route whose -db
names a DIRECTORY now replicates every non-empty *.db beneath it, skipping
dot-directories (litestream's own shadow WAL lives in one, and some of its
files are named *.db) and sidecars. Exec routes are untouched: the activator
consumes their DBPath verbatim, so a directory there is a mistake, not a tree.
The drift this fixes is already real. The flagship's hand-written list names
four kass databases; the tenant has five. instances/ma248ch8zrg7.db was created
on 2026-08-07 and has never been replicated, because a human wrote the list
once and the tenant kept going.
NO SILENT SHRINK. Cutting a box from a hand-maintained config to a generated
one is the moment a database can silently stop being replicated: the new list
comes from the registry, so anything the registry does not know about is simply
absent — and nothing says so. Replication stops, and the drill keeps passing,
because the drill only checks what the config names. The failure hides in both
directions at once.
`carlos ops litestream` now refuses to write a config that drops any database
the current one names, printing which, and changing nothing. -allow-shrink is
the deliberate override, and it lists what it is dropping.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
3 files changed,
+242
−1
internal/ops/litestream.go+97 −1internal/ops/ops.go+18 −0internal/ops/ops_test.go+127 −0
diff --git a/internal/ops/litestream.go b/internal/ops/litestream.go| index a3e7baeeb..ae4a60c78 100644 |
| --- a/internal/ops/litestream.go |
| +++ b/internal/ops/litestream.go |
| @@ -158,6 +158,34 @@ func ParseLitestreamDBs(cfg string) []string { |
| return dbs |
| } |
| +// CoverageShortfall names every database the EXISTING config replicates that |
| +// the proposed set does not — the databases a write would stop replicating. |
| +// |
| +// This is the guard for the one genuinely dangerous moment in this verb's |
| +// life: cutting a box over from a hand-maintained config to a generated one. |
| +// The generated list comes from the registry, so anything the registry does |
| +// not know about is simply absent from it — and nothing downstream would say |
| +// so. Replication of the dropped database stops, and the drill keeps passing |
| +// because the drill only checks what the config names. The failure is |
| +// invisible in both directions at once. |
| +// |
| +// Growing the set is always fine and needs no ceremony; shrinking it is a |
| +// decision someone has to make on purpose. |
| +func CoverageShortfall(existingConfig string, proposed []string) []string { |
| + have := map[string]bool{} |
| + for _, p := range proposed { |
| + have[filepath.Clean(p)] = true |
| + } |
| + var missing []string |
| + for _, p := range ParseLitestreamDBs(existingConfig) { |
| + if !have[filepath.Clean(p)] { |
| + missing = append(missing, p) |
| + } |
| + } |
| + sort.Strings(missing) |
| + return missing |
| +} |
| + |
| // FleetDBs lists the non-empty *.db files this box-wide replicator owns: |
| // everything found by globbing dataDir (the agent's own exec-child |
| // databases, plus the registry itself), PLUS — separately, not via the glob |
| @@ -190,6 +218,69 @@ func ParseLitestreamDBs(cfg string) []string { |
| // carried verbatim.) That exclusion never applies to BackingUnit routes — |
| // their replication has no other owner at all, so they are added here |
| // rather than filtered. |
| +// treeDBWalkDepth bounds the descent. It is a loop guard, not a policy: os.Stat |
| +// follows symlinks (which it must — systemd's StateDirectory is a symlink into |
| +// /var/lib/private), and a symlink pointing at an ancestor would otherwise |
| +// recurse forever. No real tenant layout is anywhere near this deep. |
| +const treeDBWalkDepth = 8 |
| + |
| +// treeDBs lists the *.db files beneath root. |
| +// |
| +// It exists because a unit tenant's "database" is sometimes a TREE. kass on |
| +// the flagship keeps registry.db plus an instances/ directory that grows as the |
| +// tenant creates instances; a single -db path cannot say that, which is why |
| +// that box's config was still hand-maintained — and why it had silently |
| +// drifted, missing an instance database created after the list was written. |
| +// |
| +// Two things are deliberately skipped: |
| +// |
| +// - dot-prefixed directories, which is where litestream keeps its own shadow |
| +// WAL (".<db>-litestream"). Replicating a replicator's bookkeeping would be |
| +// absurd, and worse, some of those files are named *.db. |
| +// - anything that is not a non-empty *.db. Sidecars (-wal, -shm) travel with |
| +// their database and are not replicated separately; an empty file is a |
| +// placeholder, not a database. |
| +// |
| +// Discovery requires the .db convention, unlike a single-file DBPath which is |
| +// taken verbatim: when the operator names one file we believe them, but when |
| +// we go looking we only claim what we can recognise. |
| +func treeDBs(root string) []string { |
| + var out []string |
| + var walk func(dir string, depth int) |
| + walk = func(dir string, depth int) { |
| + if depth > treeDBWalkDepth { |
| + return |
| + } |
| + entries, err := os.ReadDir(dir) |
| + if err != nil { |
| + return // unreadable subtree: nothing to claim, and not this pass's business to fail |
| + } |
| + for _, e := range entries { |
| + name := e.Name() |
| + if strings.HasPrefix(name, ".") { |
| + continue |
| + } |
| + p := filepath.Join(dir, name) |
| + // Stat, not e.Info(): entries may be symlinks and the target is |
| + // what matters. |
| + fi, err := os.Stat(p) |
| + if err != nil { |
| + continue |
| + } |
| + if fi.IsDir() { |
| + walk(p, depth+1) |
| + continue |
| + } |
| + if !strings.HasSuffix(name, ".db") || fi.Size() == 0 { |
| + continue |
| + } |
| + out = append(out, p) |
| + } |
| + } |
| + walk(root, 0) |
| + return out |
| +} |
| + |
| // consoleDBPath resolves where this box's console keeps its database: |
| // CONSOLE_DB when set (the same env var cmd/console reads, so a relocated |
| // console cannot silently drop out of replication), else the production |
| @@ -231,8 +322,13 @@ func FleetDBs(reg *registry.Registry, dataDir string) ([]string, error) { |
| return nil, err |
| } |
| // Unit-tenant DBPaths are added on top of the glob: they typically live |
| - // entirely outside dataDir, at their StateDirectory path. |
| + // entirely outside dataDir, at their StateDirectory path. A DBPath naming |
| + // a DIRECTORY is a tree — see treeDBs. |
| for p := range unitPaths { |
| + if fi, err := os.Stat(p); err == nil && fi.IsDir() { |
| + matches = append(matches, treeDBs(p)...) |
| + continue |
| + } |
| matches = append(matches, p) |
| } |
| // And the console's own database, for the same reason in miniature: it |
diff --git a/internal/ops/ops.go b/internal/ops/ops.go| index 19806a7d4..5a1cefc30 100644 |
| --- a/internal/ops/ops.go |
| +++ b/internal/ops/ops.go |
| @@ -96,6 +96,7 @@ func runLitestream(args []string) { |
| region := fs.String("region", "", "S3 region for the replica stanza") |
| out := fs.String("out", defaultConfigPath, "config file to write") |
| lsBin := fs.String("litestream-bin", "litestream", "litestream binary to check the snapshot policy against (pinned release "+litestreamVersion+")") |
| + allowShrink := fs.Bool("allow-shrink", false, "permit a config that replicates FEWER databases than the one it replaces") |
| _ = fs.Parse(args) |
| if *bucket == "" || *prefix == "" { |
| @@ -118,6 +119,23 @@ func runLitestream(args []string) { |
| log.Fatalf("litestream: no non-empty *.db files under %s — wrong box?", dir) |
| } |
| + // Never quietly replicate less than the file being replaced — see |
| + // CoverageShortfall. This is what makes a cutover from a hand-maintained |
| + // config to a generated one safe to do unattended. |
| + if prior, err := os.ReadFile(*out); err == nil { |
| + if missing := CoverageShortfall(string(prior), dbs); len(missing) > 0 && !*allowShrink { |
| + log.Fatalf(`litestream: REFUSING to write %s — it would stop replicating %d database(s) that %s names today: |
| + %s |
| +Nothing has been changed. Either make the registry describe them (a route with |
| +-db, or a unit tenant's state directory), or pass -allow-shrink if they really |
| +should no longer be replicated.`, |
| + *out, len(missing), *out, strings.Join(missing, "\n ")) |
| + } else if len(missing) > 0 { |
| + fmt.Fprintf(os.Stderr, "litestream: -allow-shrink: dropping %d database(s) from replication:\n %s\n", |
| + len(missing), strings.Join(missing, "\n ")) |
| + } |
| + } |
| + |
| cfg := LitestreamConfig(dbs, *bucket, *prefix, *region) |
| wrote, err := writeIfChanged(*out, cfg) |
| if err != nil { |
diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go| index 1e1d059d0..7aa68fc33 100644 |
| --- a/internal/ops/ops_test.go |
| +++ b/internal/ops/ops_test.go |
| @@ -541,3 +541,130 @@ func TestRestoreVerifyFailsOnAStaleReplica(t *testing.T) { |
| t.Fatalf("error does not name the failure: %v", err) |
| } |
| } |
| + |
| +// A unit tenant whose app manages its OWN sub-instances keeps a TREE of |
| +// databases, not one file: kass on the flagship has registry.db plus an |
| +// instances/ directory that grows as the tenant creates instances. Naming a |
| +// single -db path cannot express that, which is why the flagship's config is |
| +// still hand-maintained ("carlos ops litestream takes over once nested layouts |
| +// are supported") — and why it had already drifted, missing one instance |
| +// database that the tenant created after the list was written. |
| +// |
| +// So: a BackingUnit route whose -db names a DIRECTORY replicates every *.db |
| +// beneath it. Exec routes are untouched by this — the activator consumes their |
| +// DBPath verbatim (Restore, Seed, the --db flag), so a directory there would |
| +// be meaningless. |
| +func TestFleetDBsWalksAUnitTenantDirectory(t *testing.T) { |
| + dir := t.TempDir() |
| + reg, err := registry.Open(filepath.Join(dir, "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + state := t.TempDir() |
| + nested := filepath.Join(state, "instances") |
| + if err := os.MkdirAll(nested, 0o750); err != nil { |
| + t.Fatal(err) |
| + } |
| + top := filepath.Join(state, "registry.db") |
| + inst1 := filepath.Join(nested, "5vepbveytb8y.db") |
| + inst2 := filepath.Join(nested, "eafu5brhjs26.db") |
| + for _, p := range []string{top, inst1, inst2} { |
| + if err := os.WriteFile(p, []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + // Litestream's own shadow-WAL directory sits beside each database and is |
| + // full of files that are not databases. Replicating a replicator's |
| + // bookkeeping would be absurd, and dot-directories are its convention. |
| + meta := filepath.Join(state, ".registry.db-litestream", "generations", "abc", "wal") |
| + if err := os.MkdirAll(meta, 0o750); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Join(meta, "00000000.db"), []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Sidecars and empty placeholders are not databases either. |
| + for _, p := range []string{top + "-wal", top + "-shm", filepath.Join(state, "empty.db")} { |
| + if err := os.WriteFile(p, nil, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + |
| + if err := reg.Upsert(registry.Route{Host: "home.kass.training", Backing: registry.BackingUnit, DBPath: state}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + dbs, err := FleetDBs(reg, dir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, want := range []string{top, inst1, inst2} { |
| + if !slices.Contains(dbs, want) { |
| + t.Fatalf("tree database %s missing: %v", want, dbs) |
| + } |
| + } |
| + for _, unwanted := range []string{meta, top + "-wal", top + "-shm", filepath.Join(state, "empty.db")} { |
| + if slices.Contains(dbs, unwanted) { |
| + t.Fatalf("FleetDBs swept up %s: %v", unwanted, dbs) |
| + } |
| + } |
| + for _, got := range dbs { |
| + if strings.Contains(got, "-litestream") { |
| + t.Fatalf("FleetDBs descended into a litestream meta directory: %s", got) |
| + } |
| + } |
| +} |
| + |
| +// The tree rule must not leak to exec routes: their DBPath is consumed |
| +// verbatim by the activator, so a directory there is a mistake, not a tree. |
| +func TestFleetDBsDoesNotWalkExecRouteDBPath(t *testing.T) { |
| + dir := t.TempDir() |
| + reg, err := registry.Open(filepath.Join(dir, "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + state := t.TempDir() |
| + if err := os.WriteFile(filepath.Join(state, "inner.db"), []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.Upsert(registry.Route{Host: "exec.example.com", DBPath: state}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + dbs, err := FleetDBs(reg, dir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if slices.Contains(dbs, filepath.Join(state, "inner.db")) { |
| + t.Fatalf("an exec route's directory DBPath was walked: %v", dbs) |
| + } |
| +} |
| + |
| +// Cutting a box over from a hand-maintained config to a generated one is the |
| +// moment a database can silently stop being replicated: the new list is |
| +// derived from the registry, and anything the registry does not know about |
| +// simply is not in it. Nobody would notice — replication of a dropped database |
| +// just stops, and the drill only checks what the config names. |
| +// |
| +// So the generator refuses to shrink the set unless told to. |
| +func TestCoverageShortfall(t *testing.T) { |
| + existing := LitestreamConfig([]string{"/data/a.db", "/data/b.db", "/data/registry.db"}, "b", "p", "eu-west-1") |
| + |
| + // A superset is fine — that is the ordinary case (console.db arriving). |
| + if missing := CoverageShortfall(existing, []string{"/data/a.db", "/data/b.db", "/data/registry.db", "/data/console/console.db"}); len(missing) != 0 { |
| + t.Fatalf("a superset was reported as a shortfall: %v", missing) |
| + } |
| + // Identical is fine. |
| + if missing := CoverageShortfall(existing, []string{"/data/registry.db", "/data/b.db", "/data/a.db"}); len(missing) != 0 { |
| + t.Fatalf("an identical set was reported as a shortfall: %v", missing) |
| + } |
| + // Dropping one is the thing to catch, and the caller must be told WHICH. |
| + missing := CoverageShortfall(existing, []string{"/data/a.db", "/data/registry.db"}) |
| + if len(missing) != 1 || missing[0] != "/data/b.db" { |
| + t.Fatalf("CoverageShortfall = %v, want [/data/b.db]", missing) |
| + } |
| + // No existing config (a fresh box) can never be a shortfall. |
| + if missing := CoverageShortfall("", []string{"/data/a.db"}); len(missing) != 0 { |
| + t.Fatalf("a fresh box reported a shortfall: %v", missing) |
| + } |
| +} |