f36718c54 · carlos/platform

diff --git a/README.md b/README.md

index eee29768b..e2a7dcccf 100644

--- a/README.md

+++ b/README.md

@@ -438,6 +438,24 @@ placements (#409).

release: only this release's DNS steering leaves replicas out of

latency routing.

+### Seven days of history for sleeping instances (2026-10-08)

+

+This release keeps seven days of database snapshots for instances that

+sleep, the same as instances that stay awake (#263). `carlos db pull --at`

+(#262) can then reach back a week on them, not a day.

+

+- boxes only. the console is unaffected.

+- the window fills in over the week after the roll. snapshots older than a

+ day were already deleted under the old setting, so on the day of the roll a

+ sleeping instance still reaches back about a day, and a week only seven

+ days later.

+- an instance's history moves only while it is awake. one that sleeps for

+ several days takes no snapshot in that time, and loses none either:

+ retention also runs only while it is awake.

+- storage grows by up to six more daily snapshots per sleeping instance,

+ plus the hourly files back to the oldest one. there are no extra s3

+ requests.

+

## App config & secrets

Per-app config lives in the deployment bucket

diff --git a/internal/lease/lease.go b/internal/lease/lease.go

index d4ae8b158..1de6e5479 100644

--- a/internal/lease/lease.go

+++ b/internal/lease/lease.go

@@ -59,6 +59,39 @@ var compactionLevels = litestream.CompactionLevels{

{Level: 3, Interval: 24 * time.Hour},

}

+// SnapshotInterval and SnapshotRetention are the fleet's point-in-time

+// recovery window, for every replica the platform writes: this store (a

+// hibernating instance's) and the box-wide /etc/litestream.yml, which

+// internal/ops writes from these same two constants so they cannot drift.

+//

+// litestream 0.5's DefaultSnapshotRetention is 24h. That covers a dead box;

+// it does not cover LOGICAL corruption, where integrity_check says "ok" and

+// the data is simply wrong. A bad migration or an errant DELETE is

+// typically noticed days later, and by then the only copy is the

+// faithfully-replicated bad one. Here the default was worse than a stated

+// day: retention runs in the snapshot monitor, which runs only while the

+// instance is awake, so each wake pruned every snapshot older than a day

+// (#263), and `carlos db pull --at` reached back about a day.

+//

+// A week is the deliberate choice. It adds no requests: the snapshot

+// monitor's cadence is SnapshotInterval either way, and only how far back

+// its deletes reach changes. It does add storage, and more than the seven

+// snapshots: retention keeps every L1-L3 file (compactionLevels: hourly,

+// 6-hourly, daily) back to the oldest kept snapshot too. Each holds only

+// the pages that changed in its window, so a typical small, lightly

+// written database adds little. The worst case is a database rewriting

+// most of its pages every hour: about 168 L1 + 28 L2 + 7 L3 files plus 7

+// snapshots, some 200 database sizes (100 MiB -> ~20 GiB, ~$0.50/mo at

+// ~$0.023/GB-mo), against about 30 under the old 24h. Lengthen it if a

+// restore is ever wanted from further back than "we noticed within the

+// week"; shorten it if the bucket's size says the worst case is common.

+// (Astra's review of #263 corrected an estimate that counted only the

+// snapshots.)

+const (

+ SnapshotInterval = 24 * time.Hour

+ SnapshotRetention = 7 * 24 * time.Hour

+)

+

// dbMonitorInterval slows litestream's DB monitor (default 1s) — the local

// WAL poll that shovels new frames into shadow LTX. It makes no S3 calls,

// but 1s polling per awake instance is pointless churn when the sleep-time

@@ -299,6 +332,8 @@ func (h *Held) newStore(dbPath string, syncInterval time.Duration) (*litestream.

db.Replica.SyncInterval = syncInterval

}

store := litestream.NewStore([]*litestream.DB{db}, compactionLevels)

+ store.SnapshotInterval = SnapshotInterval

+ store.SnapshotRetention = SnapshotRetention

if h.generationPrefix != "" {

// NewStore resets the DB flag. Set all three store/DB/compactor flags

// afterward, before Open starts any monitor or compaction work.

diff --git a/internal/lease/retention_window_test.go b/internal/lease/retention_window_test.go

new file mode 100644

index 000000000..c5235c0e1

--- /dev/null

+++ b/internal/lease/retention_window_test.go

@@ -0,0 +1,44 @@

+package lease

+

+import (

+ "context"

+ "path/filepath"

+ "testing"

+ "time"

+

+ lss3 "github.com/benbjohnson/litestream/s3"

+)

+

+// A hibernating instance's replica keeps the same recovery window the

+// box-wide replicator does (#263). litestream's own default is 24h, which

+// a wake would enforce, deleting every snapshot older than a day.

+func TestNewStorePinsTheRecoveryWindow(t *testing.T) {

+ l := lss3.NewLeaser()

+ l.Path = "instances/x.example/inc"

+ h := &Held{leaser: l, replicaClient: &generationFake{}}

+ store, db := h.newStore(filepath.Join(t.TempDir(), "db"), time.Second)

+ if store.SnapshotRetention != SnapshotRetention || SnapshotRetention != 7*24*time.Hour {

+ t.Errorf("SnapshotRetention = %v (constant %v), want 7 days", store.SnapshotRetention, SnapshotRetention)

+ }

+ if store.SnapshotInterval != SnapshotInterval || SnapshotInterval != 24*time.Hour {

+ t.Errorf("SnapshotInterval = %v (constant %v), want 24h", store.SnapshotInterval, SnapshotInterval)

+ }

+ if !store.RetentionEnabled || !db.RetentionEnabled {

+ t.Error("retention disabled on an unbound store; the window must be enforced, not only stated")

+ }

+}

+

+// A bound generation (an instance move) keeps retention off whatever the

+// window says: the new setting must not turn it back on.

+func TestNewStoreWindowLeavesAGenerationsRetentionOff(t *testing.T) {

+ l := lss3.NewLeaser()

+ l.Path = "instances/x.example/inc"

+ h := &Held{leaser: l, replicaClient: &generationFake{}}

+ if err := h.BindGeneration(l.Path+"/g-one/", func(context.Context) error { return nil }); err != nil {

+ t.Fatal(err)

+ }

+ store, db := h.newStore(filepath.Join(t.TempDir(), "db"), time.Second)

+ if store.RetentionEnabled || db.RetentionEnabled {

+ t.Fatal("a bound generation's store has retention on")

+ }

+}

diff --git a/internal/ops/litestream.go b/internal/ops/litestream.go

index 312aaddcd..de6307f40 100644

--- a/internal/ops/litestream.go

+++ b/internal/ops/litestream.go

@@ -47,24 +47,13 @@ const litestreamVersion = "0.5.15"

var retryDelay = 10 * time.Second

// SnapshotInterval and SnapshotRetention pin the fleet's point-in-time

-// recovery window instead of inheriting it.

-//

-// This is the one setting whose DEFAULT was a real hole: litestream 0.5's

-// DefaultSnapshotRetention is 24h, so a database this replicator owns could

-// only ever be restored to roughly the last day. That covers a dead box; it

-// does not cover the failure mode replication is otherwise useless against —

-// LOGICAL corruption, where integrity_check says "ok" and the data is simply

-// wrong. A bad migration or an errant DELETE is typically noticed days later,

-// and by then the only copy is the faithfully-replicated bad one.

-//

-// A week is the deliberate choice: seven daily snapshots per database, of

-// files measured in megabytes, against S3 storage at ~$0.023/GB-mo — noise

-// next to the request bill this fleet actually watches (see internal/lease's

-// LeaseTTL for that story). Lengthen it if a restore is ever wanted from

-// further back than "we noticed within the week".

+// recovery window instead of inheriting litestream's 24h default. They are

+// lease's constants, which hibernating instances' replicas use too, so the

+// box-wide window and the per-instance one are one decision; lease.go

+// records why a week.

const (

- SnapshotInterval = 24 * time.Hour

- SnapshotRetention = 7 * 24 * time.Hour

+ SnapshotInterval = lease.SnapshotInterval

+ SnapshotRetention = lease.SnapshotRetention

)

// SnapshotPolicySupported reports whether `litestream version` output names a

carlos / platform Public

hibernate-retention: sleeping instances keep 7 days of snapshots, like awake ones (#263)