amadan
lease: hibernating instances keep 7 days of snapshots, like the box-wide replicator (#263)
The activator's embedded litestream store set no snapshot retention, so it took litestream's 24h default, and every wake pruned snapshots older than a day. carlos db pull --at (which reads only sleeping instances) reached back about a day. The window now lives in lease (SnapshotInterval 24h, SnapshotRetention 7d) and internal/ops uses the same constants for /etc/litestream.yml. A bound generation (a move) still has retention off. No extra S3 requests: the snapshot monitor's cadence is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Human
Prompt
Yes.
From this commit's own message. The prompt ledger has no entries for it — which is what a squashed, rebased or amended commit looks like, and what every commit looks like in a repo without a ledger.
Summary
Pins 7-day snapshot retention on hibernating instances' replicas, shared with ops.
Agent notes
Mutation-checked: dropping store.SnapshotRetention fails TestNewStorePinsTheRecoveryWindow with 24h.
3 files changed,
+77
−17
internal/lease/lease.go+27 −0internal/lease/retention_window_test.go+44 −0internal/ops/litestream.go+6 −17
CI cancelled — superseded by a newer push — run details
diff --git a/internal/lease/lease.go b/internal/lease/lease.go| index d4ae8b158..a9ab03ba5 100644 |
| --- a/internal/lease/lease.go |
| +++ b/internal/lease/lease.go |
| @@ -59,6 +59,31 @@ var compactionLevels = litestream.CompactionLevels{ |
| {Level: 3, Interval: 24 * time.Hour}, |
| } |
| +// SnapshotInterval and SnapshotRetention are the fleet's point-in-time |
| +// recovery window, for every replica the platform writes: this store (a |
| +// hibernating instance's) and the box-wide /etc/litestream.yml, which |
| +// internal/ops writes from these same two constants so they cannot drift. |
| +// |
| +// litestream 0.5's DefaultSnapshotRetention is 24h. That covers a dead box; |
| +// it does not cover LOGICAL corruption, where integrity_check says "ok" and |
| +// the data is simply wrong. A bad migration or an errant DELETE is |
| +// typically noticed days later, and by then the only copy is the |
| +// faithfully-replicated bad one. Here the default was worse than a stated |
| +// day: retention runs in the snapshot monitor, which runs only while the |
| +// instance is awake, so each wake pruned every snapshot older than a day |
| +// (#263), and `carlos db pull --at` reached back about a day. |
| +// |
| +// A week is the deliberate choice: seven daily snapshots per database, of |
| +// files measured in megabytes, against S3 storage at ~$0.023/GB-mo. It adds |
| +// no requests: the snapshot monitor's cadence is SnapshotInterval either |
| +// way, and only how far back its deletes reach changes. Lengthen it if a |
| +// restore is ever wanted from further back than "we noticed within the |
| +// week". |
| +const ( |
| + SnapshotInterval = 24 * time.Hour |
| + SnapshotRetention = 7 * 24 * time.Hour |
| +) |
| + |
| // dbMonitorInterval slows litestream's DB monitor (default 1s) — the local |
| // WAL poll that shovels new frames into shadow LTX. It makes no S3 calls, |
| // but 1s polling per awake instance is pointless churn when the sleep-time |
| @@ -299,6 +324,8 @@ func (h *Held) newStore(dbPath string, syncInterval time.Duration) (*litestream. |
| db.Replica.SyncInterval = syncInterval |
| } |
| store := litestream.NewStore([]*litestream.DB{db}, compactionLevels) |
| + store.SnapshotInterval = SnapshotInterval |
| + store.SnapshotRetention = SnapshotRetention |
| if h.generationPrefix != "" { |
| // NewStore resets the DB flag. Set all three store/DB/compactor flags |
| // afterward, before Open starts any monitor or compaction work. |
diff --git a/internal/lease/retention_window_test.go b/internal/lease/retention_window_test.go| new file mode 100644 |
| index 000000000..c5235c0e1 |
| --- /dev/null |
| +++ b/internal/lease/retention_window_test.go |
| @@ -0,0 +1,44 @@ |
| +package lease |
| + |
| +import ( |
| + "context" |
| + "path/filepath" |
| + "testing" |
| + "time" |
| + |
| + lss3 "github.com/benbjohnson/litestream/s3" |
| +) |
| + |
| +// A hibernating instance's replica keeps the same recovery window the |
| +// box-wide replicator does (#263). litestream's own default is 24h, which |
| +// a wake would enforce, deleting every snapshot older than a day. |
| +func TestNewStorePinsTheRecoveryWindow(t *testing.T) { |
| + l := lss3.NewLeaser() |
| + l.Path = "instances/x.example/inc" |
| + h := &Held{leaser: l, replicaClient: &generationFake{}} |
| + store, db := h.newStore(filepath.Join(t.TempDir(), "db"), time.Second) |
| + if store.SnapshotRetention != SnapshotRetention || SnapshotRetention != 7*24*time.Hour { |
| + t.Errorf("SnapshotRetention = %v (constant %v), want 7 days", store.SnapshotRetention, SnapshotRetention) |
| + } |
| + if store.SnapshotInterval != SnapshotInterval || SnapshotInterval != 24*time.Hour { |
| + t.Errorf("SnapshotInterval = %v (constant %v), want 24h", store.SnapshotInterval, SnapshotInterval) |
| + } |
| + if !store.RetentionEnabled || !db.RetentionEnabled { |
| + t.Error("retention disabled on an unbound store; the window must be enforced, not only stated") |
| + } |
| +} |
| + |
| +// A bound generation (an instance move) keeps retention off whatever the |
| +// window says: the new setting must not turn it back on. |
| +func TestNewStoreWindowLeavesAGenerationsRetentionOff(t *testing.T) { |
| + l := lss3.NewLeaser() |
| + l.Path = "instances/x.example/inc" |
| + h := &Held{leaser: l, replicaClient: &generationFake{}} |
| + if err := h.BindGeneration(l.Path+"/g-one/", func(context.Context) error { return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + store, db := h.newStore(filepath.Join(t.TempDir(), "db"), time.Second) |
| + if store.RetentionEnabled || db.RetentionEnabled { |
| + t.Fatal("a bound generation's store has retention on") |
| + } |
| +} |
diff --git a/internal/ops/litestream.go b/internal/ops/litestream.go| index 312aaddcd..de6307f40 100644 |
| --- a/internal/ops/litestream.go |
| +++ b/internal/ops/litestream.go |
| @@ -47,24 +47,13 @@ const litestreamVersion = "0.5.15" |
| var retryDelay = 10 * time.Second |
| // SnapshotInterval and SnapshotRetention pin the fleet's point-in-time |
| -// recovery window instead of inheriting it. |
| -// |
| -// This is the one setting whose DEFAULT was a real hole: litestream 0.5's |
| -// DefaultSnapshotRetention is 24h, so a database this replicator owns could |
| -// only ever be restored to roughly the last day. That covers a dead box; it |
| -// does not cover the failure mode replication is otherwise useless against — |
| -// LOGICAL corruption, where integrity_check says "ok" and the data is simply |
| -// wrong. A bad migration or an errant DELETE is typically noticed days later, |
| -// and by then the only copy is the faithfully-replicated bad one. |
| -// |
| -// A week is the deliberate choice: seven daily snapshots per database, of |
| -// files measured in megabytes, against S3 storage at ~$0.023/GB-mo — noise |
| -// next to the request bill this fleet actually watches (see internal/lease's |
| -// LeaseTTL for that story). Lengthen it if a restore is ever wanted from |
| -// further back than "we noticed within the week". |
| +// recovery window instead of inheriting litestream's 24h default. They are |
| +// lease's constants, which hibernating instances' replicas use too, so the |
| +// box-wide window and the per-instance one are one decision; lease.go |
| +// records why a week. |
| const ( |
| - SnapshotInterval = 24 * time.Hour |
| - SnapshotRetention = 7 * 24 * time.Hour |
| + SnapshotInterval = lease.SnapshotInterval |
| + SnapshotRetention = lease.SnapshotRetention |
| ) |
| // SnapshotPolicySupported reports whether `litestream version` output names a |