amadan
Merge pull request #204 from carlosframework/verify-tmp-on-disk
verify drills: restore to disk, because tmpfs files are memory
4 files changed,
+90
−13
docs/superpowers/specs/2026-08-15-sqlite-corruption-defence-design.md+34 −0internal/host/host_test.go+28 −0internal/host/units/carlos-hibernate-verify.service+10 −4internal/host/units/carlos-restore-verify.service+18 −9
diff --git a/docs/superpowers/specs/2026-08-15-sqlite-corruption-defence-design.md b/docs/superpowers/specs/2026-08-15-sqlite-corruption-defence-design.md| index 54b5cdf00..f1a8332ce 100644 |
| --- a/docs/superpowers/specs/2026-08-15-sqlite-corruption-defence-design.md |
| +++ b/docs/superpowers/specs/2026-08-15-sqlite-corruption-defence-design.md |
| @@ -149,6 +149,40 @@ Three consequences worth stating plainly: |
| (`SnapshotPolicySupported`). Loud, not fatal: an old box still wants its |
| database set kept current, and that is what carries `console.db`. |
| +## 5a. What the first live runs taught (2026-08-15/16, rolled at 104e261) |
| + |
| +Both boxes rolled; both drills run. The first results: |
| + |
| +- **flagship `hibernate-verify`**: 4 of 6 proven, 2 `unverified` ("no replica |
| + yet" — registered but never woken). The first time those replicas had ever |
| + been shown to restore. |
| +- **tito `hibernate-verify`**: 4 of 4 proven, 0 unverified. |
| +- **tito `restore-verify`** (re-enabled after the fix): "all 27 replicas |
| + restore clean", 194.9M peak — where the same drill was OOM-killed at 256.2M |
| + before copies were deleted as they were checked. |
| +- `control/restore-verify.json` published to both deployment buckets and read |
| + back from S3 with no session on either box. |
| + |
| +And one thing the fix had only half-solved. The flagship's |
| +`hibernate-verify` peaked at **485.6M against the 512M ceiling**, because its |
| +largest hibernating tenant (`seapointish.bag.oncarlos.com.db`) is **412M** and |
| +`PrivateTmp=yes` puts the restored copy on a tmpfs, where it is charged to the |
| +cgroup as memory. Deleting copies as they are checked bounded the peak at the |
| +largest single database — but on tmpfs that still means **the ceiling has to |
| +exceed the biggest tenant**, so the drill would begin failing on exactly the |
| +databases it exists to vouch for, about 30M of growth away. |
| + |
| +The fix is to stop restoring into RAM: both drill units now carry |
| +`CacheDirectory=carlos-verify` and `Environment=TMPDIR=%C/carlos-verify`, so |
| +restores land on disk and `MemoryMax` bounds litestream's working set (tens of |
| +MB) rather than tenant size. Pinned by |
| +`TestVerifyDrillsRestoreToDiskNotTmpfs`, because it is a one-line property |
| +that regresses silently — the drill keeps passing right up until a tenant |
| +grows. |
| + |
| +The general lesson, worth carrying: **under `PrivateTmp=yes`, temp files are |
| +memory.** Any unit that stages data through /tmp is charged for it. |
| + |
| ## 6. Known follow-ups |
| - **Console surface.** `control/restore-verify.json` is read from the bucket |
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index 81387cf9c..6f442c850 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -1346,3 +1346,31 @@ func shipVersion(t *testing.T, s release.Store, acct, app, version string) { |
| t.Fatalf("Ship %s/%s: %v", app, version, err) |
| } |
| } |
| + |
| +// Both restore drills must write their restored copies to DISK, never to the |
| +// PrivateTmp tmpfs. On tmpfs the restored database is charged to the unit's |
| +// own cgroup as memory, so the drill's ceiling has to exceed the fleet's |
| +// largest tenant — and the drill starts failing on exactly the databases it |
| +// exists to vouch for. Measured on the flagship 2026-08-15: a 412M tenant |
| +// produced a 485.6M peak against MemoryMax=512M. |
| +// |
| +// This is a one-line property that would regress silently (the drill still |
| +// passes, right up until a tenant grows), so it is pinned here. |
| +func TestVerifyDrillsRestoreToDiskNotTmpfs(t *testing.T) { |
| + for _, unit := range []string{"carlos-restore-verify.service", "carlos-hibernate-verify.service"} { |
| + body, err := unitsFS.ReadFile("units/" + unit) |
| + if err != nil { |
| + t.Fatalf("%s: %v", unit, err) |
| + } |
| + s := string(body) |
| + if !strings.Contains(s, "CacheDirectory=carlos-verify") { |
| + t.Errorf("%s has no CacheDirectory — nothing creates the on-disk scratch dir", unit) |
| + } |
| + if !strings.Contains(s, "Environment=TMPDIR=%C/carlos-verify") { |
| + t.Errorf("%s does not point TMPDIR at the on-disk dir; restores land on the tmpfs and count as memory", unit) |
| + } |
| + if !strings.Contains(s, "MemoryMax=") { |
| + t.Errorf("%s has no MemoryMax — a runaway drill should hit its own ceiling, not the box's", unit) |
| + } |
| + } |
| +} |
diff --git a/internal/host/units/carlos-hibernate-verify.service b/internal/host/units/carlos-hibernate-verify.service| index 2b9f1e26a..e491c837f 100644 |
| --- a/internal/host/units/carlos-hibernate-verify.service |
| +++ b/internal/host/units/carlos-hibernate-verify.service |
| @@ -29,6 +29,13 @@ Type=oneshot |
| # wrong store. |
| EnvironmentFile=-/etc/carlos/host.env |
| ExecStart=/opt/carlos/carlos ops hibernate-verify |
| +# Disk, not the PrivateTmp tmpfs — see carlos-restore-verify.service's note. |
| +# This drill hit it hardest: the flagship's largest hibernating tenant is 412M |
| +# and the first live run peaked at 485.6M against a 512M ceiling, i.e. the |
| +# drill was ~30M of tenant growth away from failing on the very databases it |
| +# is there to vouch for. |
| +CacheDirectory=carlos-verify |
| +Environment=TMPDIR=%C/carlos-verify |
| User=carlos |
| Group=carlos |
| CapabilityBoundingSet= |
| @@ -43,8 +50,7 @@ PrivateTmp=yes |
| PrivateDevices=yes |
| RestrictAddressFamilies=AF_INET AF_INET6 |
| MemoryDenyWriteExecute=yes |
| -# Same tmpfs-is-memory arithmetic as carlos-restore-verify.service (read its |
| -# note): PrivateTmp charges every restored byte to this cgroup. This drill is |
| -# one-at-a-time by construction — lease.VerifyReplica deletes each throwaway |
| -# copy in a defer — so the peak is one database, and the same 512M applies. |
| +# Bounds the working set, not the databases (TMPDIR is on disk above). This |
| +# drill is one-at-a-time by construction — lease.VerifyReplica deletes each |
| +# throwaway copy in a defer — so the same generous 512M ceiling applies. |
| MemoryMax=512M |
diff --git a/internal/host/units/carlos-restore-verify.service b/internal/host/units/carlos-restore-verify.service| index fe90f9df4..6fb57944e 100644 |
| --- a/internal/host/units/carlos-restore-verify.service |
| +++ b/internal/host/units/carlos-restore-verify.service |
| @@ -13,6 +13,16 @@ Wants=network-online.target |
| Type=oneshot |
| ExecStart=/opt/carlos/carlos ops restore-verify -config /etc/litestream.yml |
| EnvironmentFile=/data/litestream.env |
| +# Restore into a DISK-backed directory, not the PrivateTmp tmpfs. A restored |
| +# database on tmpfs is charged to this cgroup as memory, so the drill's peak |
| +# tracked the size of the fleet's largest tenant and would OOM the moment one |
| +# outgrew MemoryMax — measured on the flagship 2026-08-15, a 412M tenant |
| +# produced a 485.6M peak against a 512M ceiling. On disk, the peak is |
| +# litestream's own working set (tens of MB) and the drill stops competing with |
| +# the tenants it exists to protect. systemd creates and cleans %C/carlos-verify |
| +# and binds it writable despite ProtectSystem=strict. |
| +CacheDirectory=carlos-verify |
| +Environment=TMPDIR=%C/carlos-verify |
| User=carlos |
| Group=carlos |
| CapabilityBoundingSet= |
| @@ -23,13 +33,12 @@ PrivateTmp=yes |
| PrivateDevices=yes |
| RestrictAddressFamilies=AF_INET AF_INET6 |
| MemoryDenyWriteExecute=yes |
| -# PrivateTmp=yes makes this unit's /tmp a tmpfs, so every byte the drill |
| -# restores is charged to THIS cgroup against the limit below — the restored |
| -# files are the memory. carlos-tito-ie, 2026-08-15: 54 databases totalling |
| -# 247M were OOM-killed at a "256.2M memory peak" after ~40 clean restores, |
| -# because copies accumulated. RestoreVerify now deletes each copy once it is |
| -# checked, so the peak is the LARGEST SINGLE database rather than their sum; |
| -# 512M leaves room for a database several times bigger than today's largest |
| -# (149M) plus litestream's own working set. Raise it if a single tenant |
| -# database ever approaches this, and see ops.RestoreVerify for the rest. |
| +# With TMPDIR on disk (above) this bounds litestream's WORKING SET, not the |
| +# databases — measured 194.9M for tito's 27 replicas and 485.6M on the |
| +# flagship back when a 412M tenant still landed in tmpfs. A ceiling costs |
| +# nothing unused, so it stays generous. Two earlier lessons are kept here |
| +# because both are load-bearing: tito was OOM-killed at "256.2M memory peak" |
| +# on 2026-08-15 when restored copies ACCUMULATED (RestoreVerify now deletes |
| +# each once checked — see its comment), and tmpfs is why the files counted as |
| +# memory at all. |
| MemoryMax=512M |