amadan
Merge pull request #213 from carlosframework/unit-tenant-db-trees
litestream: unit-tenant DB trees, and refuse to shrink the replication set
3 files changed,
+307
−2
internal/ops/litestream.go+124 −2internal/ops/ops.go+18 −0internal/ops/ops_test.go+165 −0
diff --git a/internal/ops/litestream.go b/internal/ops/litestream.go| index a3e7baeeb..e246f3e1f 100644 |
| --- a/internal/ops/litestream.go |
| +++ b/internal/ops/litestream.go |
| @@ -137,7 +137,7 @@ func LitestreamConfig(dbPaths []string, bucket, prefix, region string) string { |
| for _, p := range sorted { |
| fmt.Fprintf(&b, " - path: %s\n", p) |
| b.WriteString(" replicas:\n") |
| - fmt.Fprintf(&b, " - url: s3://%s/%s/%s\n", bucket, prefix, filepath.Base(p)) |
| + fmt.Fprintf(&b, " - url: s3://%s/%s/%s\n", bucket, prefix, replicaKey(p)) |
| if region != "" { |
| fmt.Fprintf(&b, " region: %s\n", region) |
| } |
| @@ -145,6 +145,32 @@ func LitestreamConfig(dbPaths []string, bucket, prefix, region string) string { |
| return b.String() |
| } |
| +// replicaKey is a database's place in the replica bucket: its full path with |
| +// the leading separator dropped, so /data/registry.db becomes |
| +// data/registry.db. |
| +// |
| +// It is the full path and not the basename because the basename is NOT |
| +// unique, and the consequence of pretending otherwise is severe. Every |
| +// SQLite-backed tenant on this fleet calls its database registry.db or |
| +// data.db, and the box's own control DB is /data/registry.db; a basename key |
| +// collapses all of them onto one S3 prefix, where two litestream replicators |
| +// interleave their WAL into a replica that restores as neither database. |
| +// Tree discovery (treeDBs) turns that from a curiosity into the normal case — |
| +// every tenant tree contains a registry.db. |
| +// |
| +// Keeping the path readable rather than hashing it is deliberate: the person |
| +// reading this bucket is doing a restore, probably at speed, and needs to see |
| +// which database each key is. |
| +// |
| +// Changing this scheme moves every database to a new replica lineage, so it |
| +// is a one-time freedom: no box has ever run a config generated by this verb |
| +// (the flagship's is hand-written, tito's comes from the legacy `titogo ops |
| +// litestream`), so there is no live lineage to break. There will be after the |
| +// first cutover — treat it as fixed then. |
| +func replicaKey(dbPath string) string { |
| + return strings.TrimPrefix(filepath.Clean(dbPath), "/") |
| +} |
| + |
| // ParseLitestreamDBs recovers the database paths from a generated config. |
| // It reads only the exact shape LitestreamConfig writes (the round-trip is |
| // pinned by TestConfigRoundTrip) — it is not a YAML parser. |
| @@ -158,6 +184,34 @@ func ParseLitestreamDBs(cfg string) []string { |
| return dbs |
| } |
| +// CoverageShortfall names every database the EXISTING config replicates that |
| +// the proposed set does not — the databases a write would stop replicating. |
| +// |
| +// This is the guard for the one genuinely dangerous moment in this verb's |
| +// life: cutting a box over from a hand-maintained config to a generated one. |
| +// The generated list comes from the registry, so anything the registry does |
| +// not know about is simply absent from it — and nothing downstream would say |
| +// so. Replication of the dropped database stops, and the drill keeps passing |
| +// because the drill only checks what the config names. The failure is |
| +// invisible in both directions at once. |
| +// |
| +// Growing the set is always fine and needs no ceremony; shrinking it is a |
| +// decision someone has to make on purpose. |
| +func CoverageShortfall(existingConfig string, proposed []string) []string { |
| + have := map[string]bool{} |
| + for _, p := range proposed { |
| + have[filepath.Clean(p)] = true |
| + } |
| + var missing []string |
| + for _, p := range ParseLitestreamDBs(existingConfig) { |
| + if !have[filepath.Clean(p)] { |
| + missing = append(missing, p) |
| + } |
| + } |
| + sort.Strings(missing) |
| + return missing |
| +} |
| + |
| // FleetDBs lists the non-empty *.db files this box-wide replicator owns: |
| // everything found by globbing dataDir (the agent's own exec-child |
| // databases, plus the registry itself), PLUS — separately, not via the glob |
| @@ -190,6 +244,69 @@ func ParseLitestreamDBs(cfg string) []string { |
| // carried verbatim.) That exclusion never applies to BackingUnit routes — |
| // their replication has no other owner at all, so they are added here |
| // rather than filtered. |
| +// treeDBWalkDepth bounds the descent. It is a loop guard, not a policy: os.Stat |
| +// follows symlinks (which it must — systemd's StateDirectory is a symlink into |
| +// /var/lib/private), and a symlink pointing at an ancestor would otherwise |
| +// recurse forever. No real tenant layout is anywhere near this deep. |
| +const treeDBWalkDepth = 8 |
| + |
| +// treeDBs lists the *.db files beneath root. |
| +// |
| +// It exists because a unit tenant's "database" is sometimes a TREE. kass on |
| +// the flagship keeps registry.db plus an instances/ directory that grows as the |
| +// tenant creates instances; a single -db path cannot say that, which is why |
| +// that box's config was still hand-maintained — and why it had silently |
| +// drifted, missing an instance database created after the list was written. |
| +// |
| +// Two things are deliberately skipped: |
| +// |
| +// - dot-prefixed directories, which is where litestream keeps its own shadow |
| +// WAL (".<db>-litestream"). Replicating a replicator's bookkeeping would be |
| +// absurd, and worse, some of those files are named *.db. |
| +// - anything that is not a non-empty *.db. Sidecars (-wal, -shm) travel with |
| +// their database and are not replicated separately; an empty file is a |
| +// placeholder, not a database. |
| +// |
| +// Discovery requires the .db convention, unlike a single-file DBPath which is |
| +// taken verbatim: when the operator names one file we believe them, but when |
| +// we go looking we only claim what we can recognise. |
| +func treeDBs(root string) []string { |
| + var out []string |
| + var walk func(dir string, depth int) |
| + walk = func(dir string, depth int) { |
| + if depth > treeDBWalkDepth { |
| + return |
| + } |
| + entries, err := os.ReadDir(dir) |
| + if err != nil { |
| + return // unreadable subtree: nothing to claim, and not this pass's business to fail |
| + } |
| + for _, e := range entries { |
| + name := e.Name() |
| + if strings.HasPrefix(name, ".") { |
| + continue |
| + } |
| + p := filepath.Join(dir, name) |
| + // Stat, not e.Info(): entries may be symlinks and the target is |
| + // what matters. |
| + fi, err := os.Stat(p) |
| + if err != nil { |
| + continue |
| + } |
| + if fi.IsDir() { |
| + walk(p, depth+1) |
| + continue |
| + } |
| + if !strings.HasSuffix(name, ".db") || fi.Size() == 0 { |
| + continue |
| + } |
| + out = append(out, p) |
| + } |
| + } |
| + walk(root, 0) |
| + return out |
| +} |
| + |
| // consoleDBPath resolves where this box's console keeps its database: |
| // CONSOLE_DB when set (the same env var cmd/console reads, so a relocated |
| // console cannot silently drop out of replication), else the production |
| @@ -231,8 +348,13 @@ func FleetDBs(reg *registry.Registry, dataDir string) ([]string, error) { |
| return nil, err |
| } |
| // Unit-tenant DBPaths are added on top of the glob: they typically live |
| - // entirely outside dataDir, at their StateDirectory path. |
| + // entirely outside dataDir, at their StateDirectory path. A DBPath naming |
| + // a DIRECTORY is a tree — see treeDBs. |
| for p := range unitPaths { |
| + if fi, err := os.Stat(p); err == nil && fi.IsDir() { |
| + matches = append(matches, treeDBs(p)...) |
| + continue |
| + } |
| matches = append(matches, p) |
| } |
| // And the console's own database, for the same reason in miniature: it |
diff --git a/internal/ops/ops.go b/internal/ops/ops.go| index 19806a7d4..5a1cefc30 100644 |
| --- a/internal/ops/ops.go |
| +++ b/internal/ops/ops.go |
| @@ -96,6 +96,7 @@ func runLitestream(args []string) { |
| region := fs.String("region", "", "S3 region for the replica stanza") |
| out := fs.String("out", defaultConfigPath, "config file to write") |
| lsBin := fs.String("litestream-bin", "litestream", "litestream binary to check the snapshot policy against (pinned release "+litestreamVersion+")") |
| + allowShrink := fs.Bool("allow-shrink", false, "permit a config that replicates FEWER databases than the one it replaces") |
| _ = fs.Parse(args) |
| if *bucket == "" || *prefix == "" { |
| @@ -118,6 +119,23 @@ func runLitestream(args []string) { |
| log.Fatalf("litestream: no non-empty *.db files under %s — wrong box?", dir) |
| } |
| + // Never quietly replicate less than the file being replaced — see |
| + // CoverageShortfall. This is what makes a cutover from a hand-maintained |
| + // config to a generated one safe to do unattended. |
| + if prior, err := os.ReadFile(*out); err == nil { |
| + if missing := CoverageShortfall(string(prior), dbs); len(missing) > 0 && !*allowShrink { |
| + log.Fatalf(`litestream: REFUSING to write %s — it would stop replicating %d database(s) that %s names today: |
| + %s |
| +Nothing has been changed. Either make the registry describe them (a route with |
| +-db, or a unit tenant's state directory), or pass -allow-shrink if they really |
| +should no longer be replicated.`, |
| + *out, len(missing), *out, strings.Join(missing, "\n ")) |
| + } else if len(missing) > 0 { |
| + fmt.Fprintf(os.Stderr, "litestream: -allow-shrink: dropping %d database(s) from replication:\n %s\n", |
| + len(missing), strings.Join(missing, "\n ")) |
| + } |
| + } |
| + |
| cfg := LitestreamConfig(dbs, *bucket, *prefix, *region) |
| wrote, err := writeIfChanged(*out, cfg) |
| if err != nil { |
diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go| index 1e1d059d0..97cea384b 100644 |
| --- a/internal/ops/ops_test.go |
| +++ b/internal/ops/ops_test.go |
| @@ -541,3 +541,168 @@ func TestRestoreVerifyFailsOnAStaleReplica(t *testing.T) { |
| t.Fatalf("error does not name the failure: %v", err) |
| } |
| } |
| + |
| +// A unit tenant whose app manages its OWN sub-instances keeps a TREE of |
| +// databases, not one file: kass on the flagship has registry.db plus an |
| +// instances/ directory that grows as the tenant creates instances. Naming a |
| +// single -db path cannot express that, which is why the flagship's config is |
| +// still hand-maintained ("carlos ops litestream takes over once nested layouts |
| +// are supported") — and why it had already drifted, missing one instance |
| +// database that the tenant created after the list was written. |
| +// |
| +// So: a BackingUnit route whose -db names a DIRECTORY replicates every *.db |
| +// beneath it. Exec routes are untouched by this — the activator consumes their |
| +// DBPath verbatim (Restore, Seed, the --db flag), so a directory there would |
| +// be meaningless. |
| +func TestFleetDBsWalksAUnitTenantDirectory(t *testing.T) { |
| + dir := t.TempDir() |
| + reg, err := registry.Open(filepath.Join(dir, "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + state := t.TempDir() |
| + nested := filepath.Join(state, "instances") |
| + if err := os.MkdirAll(nested, 0o750); err != nil { |
| + t.Fatal(err) |
| + } |
| + top := filepath.Join(state, "registry.db") |
| + inst1 := filepath.Join(nested, "5vepbveytb8y.db") |
| + inst2 := filepath.Join(nested, "eafu5brhjs26.db") |
| + for _, p := range []string{top, inst1, inst2} { |
| + if err := os.WriteFile(p, []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + // Litestream's own shadow-WAL directory sits beside each database and is |
| + // full of files that are not databases. Replicating a replicator's |
| + // bookkeeping would be absurd, and dot-directories are its convention. |
| + meta := filepath.Join(state, ".registry.db-litestream", "generations", "abc", "wal") |
| + if err := os.MkdirAll(meta, 0o750); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Join(meta, "00000000.db"), []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Sidecars and empty placeholders are not databases either. |
| + for _, p := range []string{top + "-wal", top + "-shm", filepath.Join(state, "empty.db")} { |
| + if err := os.WriteFile(p, nil, 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + |
| + if err := reg.Upsert(registry.Route{Host: "home.kass.training", Backing: registry.BackingUnit, DBPath: state}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + dbs, err := FleetDBs(reg, dir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, want := range []string{top, inst1, inst2} { |
| + if !slices.Contains(dbs, want) { |
| + t.Fatalf("tree database %s missing: %v", want, dbs) |
| + } |
| + } |
| + for _, unwanted := range []string{meta, top + "-wal", top + "-shm", filepath.Join(state, "empty.db")} { |
| + if slices.Contains(dbs, unwanted) { |
| + t.Fatalf("FleetDBs swept up %s: %v", unwanted, dbs) |
| + } |
| + } |
| + for _, got := range dbs { |
| + if strings.Contains(got, "-litestream") { |
| + t.Fatalf("FleetDBs descended into a litestream meta directory: %s", got) |
| + } |
| + } |
| +} |
| + |
| +// The tree rule must not leak to exec routes: their DBPath is consumed |
| +// verbatim by the activator, so a directory there is a mistake, not a tree. |
| +func TestFleetDBsDoesNotWalkExecRouteDBPath(t *testing.T) { |
| + dir := t.TempDir() |
| + reg, err := registry.Open(filepath.Join(dir, "registry.db")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + state := t.TempDir() |
| + if err := os.WriteFile(filepath.Join(state, "inner.db"), []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.Upsert(registry.Route{Host: "exec.example.com", DBPath: state}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + dbs, err := FleetDBs(reg, dir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if slices.Contains(dbs, filepath.Join(state, "inner.db")) { |
| + t.Fatalf("an exec route's directory DBPath was walked: %v", dbs) |
| + } |
| +} |
| + |
| +// Cutting a box over from a hand-maintained config to a generated one is the |
| +// moment a database can silently stop being replicated: the new list is |
| +// derived from the registry, and anything the registry does not know about |
| +// simply is not in it. Nobody would notice — replication of a dropped database |
| +// just stops, and the drill only checks what the config names. |
| +// |
| +// So the generator refuses to shrink the set unless told to. |
| +func TestCoverageShortfall(t *testing.T) { |
| + existing := LitestreamConfig([]string{"/data/a.db", "/data/b.db", "/data/registry.db"}, "b", "p", "eu-west-1") |
| + |
| + // A superset is fine — that is the ordinary case (console.db arriving). |
| + if missing := CoverageShortfall(existing, []string{"/data/a.db", "/data/b.db", "/data/registry.db", "/data/console/console.db"}); len(missing) != 0 { |
| + t.Fatalf("a superset was reported as a shortfall: %v", missing) |
| + } |
| + // Identical is fine. |
| + if missing := CoverageShortfall(existing, []string{"/data/registry.db", "/data/b.db", "/data/a.db"}); len(missing) != 0 { |
| + t.Fatalf("an identical set was reported as a shortfall: %v", missing) |
| + } |
| + // Dropping one is the thing to catch, and the caller must be told WHICH. |
| + missing := CoverageShortfall(existing, []string{"/data/a.db", "/data/registry.db"}) |
| + if len(missing) != 1 || missing[0] != "/data/b.db" { |
| + t.Fatalf("CoverageShortfall = %v, want [/data/b.db]", missing) |
| + } |
| + // No existing config (a fresh box) can never be a shortfall. |
| + if missing := CoverageShortfall("", []string{"/data/a.db"}); len(missing) != 0 { |
| + t.Fatalf("a fresh box reported a shortfall: %v", missing) |
| + } |
| +} |
| + |
| +// Two databases with the same basename must not replicate to the same place. |
| +// |
| +// Every SQLite-backed tenant on the fleet names its database registry.db or |
| +// data.db; the box's own registry is /data/registry.db. Deriving the replica |
| +// URL from filepath.Base collapses all of them onto one S3 key, and two |
| +// litestream replicators writing one prefix interleave their WAL into a |
| +// replica that restores as neither database. Tree discovery makes the |
| +// collision the normal case rather than a curiosity: every tenant tree has a |
| +// registry.db in it. |
| +func TestReplicaURLsAreUniquePerPath(t *testing.T) { |
| + cfg := LitestreamConfig([]string{ |
| + "/data/registry.db", |
| + "/var/lib/carlos-app/home.kass.training/registry.db", |
| + "/var/lib/carlos-app/other.example.com/registry.db", |
| + }, "bucket-x", "fleet", "eu-west-1") |
| + |
| + var urls []string |
| + for _, line := range strings.Split(cfg, "\n") { |
| + if u, ok := strings.CutPrefix(strings.TrimSpace(line), "- url: "); ok { |
| + urls = append(urls, u) |
| + } |
| + } |
| + if len(urls) != 3 { |
| + t.Fatalf("expected 3 replica urls, got %d:\n%s", len(urls), cfg) |
| + } |
| + seen := map[string]bool{} |
| + for _, u := range urls { |
| + if seen[u] { |
| + t.Fatalf("two databases share the replica url %q — their WALs would interleave:\n%s", u, cfg) |
| + } |
| + seen[u] = true |
| + } |
| + // And the key should still be legible to whoever is doing the restore. |
| + if !strings.Contains(cfg, "home.kass.training") { |
| + t.Fatalf("replica keys lost the path that identifies them:\n%s", cfg) |
| + } |
| +} |