amadan

carlos / platform Public

litestream: size the memory cap from the database count, back off restarts (#284)

Human

Prompt

Yeah, let's do it all, and then follow or same-time do suite debt and optimisation

From this commit's own message. The prompt ledger has no entries for it — which is what a squashed, rebased or amended commit looks like, and what every commit looks like in a repo without a ledger.

Summary

carlos ops litestream writes a systemd drop-in stating MemoryMax and GOMEMLIMIT for the number of databases in the config it generates (256M + 8M each, floor 512M). The unit's own default rises from 256M to 512M with GOMEMLIMIT=384MiB, and its restart delay backs off from 2s to 60s over five restarts.
Agent notes
The plan14 root-reads census fails on this branch until an overlay is added; that is done once, when the branch's code is final. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 940d59ea94c90fa8b2c1628a706a41cc6343b3e9 parent 6dfbff1
4 files changed, +230 −1
  • internal/host/units/carlos-litestream.service +15 −1
  • internal/ops/litestream_memory.go +89 −0
  • internal/ops/ops.go +36 −0
  • internal/ops/ops_test.go +90 −0
diff --git a/internal/host/units/carlos-litestream.service b/internal/host/units/carlos-litestream.service
index edb17ddf5..89472b450 100644
--- a/internal/host/units/carlos-litestream.service
+++ b/internal/host/units/carlos-litestream.service
@@ -27,10 +27,24 @@ PrivateTmp=yes
PrivateDevices=yes
RestrictAddressFamilies=AF_INET AF_INET6
MemoryDenyWriteExecute=yes
-MemoryMax=256M
+# The default for a box whose config is not generated. `carlos ops litestream`
+# sizes both lines from the number of databases it writes and states them in a
+# drop-in (carlos-litestream.service.d/50-memory.conf), because litestream's
+# memory grows with that number: 256M held ten databases and killed the unit
+# on every start at a hundred (discussion #284). GOMEMLIMIT is three quarters
+# of the cap; Go does not read the cgroup's limit by itself.
+MemoryMax=512M
+Environment=GOMEMLIMIT=384MiB
CPUWeight=50
+# Restart always, but back off: 2s, rising over five restarts to a minute. A
+# replicator that cannot stay up restarted every 2s for hours in #284, and its
+# two startup lines per database pushed everything else out of the journal,
+# the start of the incident included. A minute is the ceiling because every
+# second this unit is down is a second of writes with no copy off the box.
Restart=always
RestartSec=2
+RestartSteps=5
+RestartMaxDelaySec=60
[Install]
WantedBy=multi-user.target
diff --git a/internal/ops/litestream_memory.go b/internal/ops/litestream_memory.go
new file mode 100644
index 000000000..16726e5ae
--- /dev/null
+++ b/internal/ops/litestream_memory.go
@@ -0,0 +1,89 @@
+package ops
+
+// The replicator's memory cap follows the number of databases it replicates.
+//
+// carlos-litestream.service used to carry one fixed MemoryMax. `litestream
+// replicate` holds memory per database, so a cap that fits a box at ten
+// databases stops fitting it at a hundred, and the way it stops fitting is the
+// worst available: the kernel kills the process a few seconds into every
+// start, systemd restarts it, and the databases too large to ship in those few
+// seconds stop being replicated at all while the unit still looks busy
+// (discussion #284: 100 databases, a 256M cap, five databases up to two hours
+// behind before a person noticed).
+//
+// The generator is the one place that knows the count, so it states the cap,
+// in a systemd drop-in beside the unit. A box whose config is not generated by
+// this verb keeps the unit's own default.
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+)
+
+const (
+ // litestreamMemoryFloor is the unit file's own MemoryMax, and so the
+ // least this verb ever states: the drop-in only raises the cap.
+ litestreamMemoryFloor = 512 << 20
+ // litestreamMemoryBase and litestreamMemoryPerDB are fitted to the one
+ // measurement there is (#284): 100 databases held steady at 229-247 MB of
+ // anonymous memory and ran without a kill under a 1G cap, where 256M
+ // killed them every start. 256M + 8M a database puts 100 databases at
+ // 1056M. The headroom over the steady figure is deliberate: the peak
+ // during startup was never measured, only that 1G survives it.
+ litestreamMemoryBase = 256 << 20
+ litestreamMemoryPerDB = 8 << 20
+
+ // defaultMemoryDropIn is where the cap goes on a box. systemd reads
+ // drop-ins in name order, so an operator's own later-sorting file in the
+ // same directory still wins.
+ defaultMemoryDropIn = "/etc/systemd/system/carlos-litestream.service.d/50-memory.conf"
+)
+
+// LitestreamMemoryMax is the replicator's memory cap, in bytes, for a config
+// naming n databases.
+func LitestreamMemoryMax(n int) int64 {
+ if n < 0 {
+ n = 0
+ }
+ want := int64(litestreamMemoryBase) + int64(n)*litestreamMemoryPerDB
+ if want < litestreamMemoryFloor {
+ return litestreamMemoryFloor
+ }
+ return want
+}
+
+// litestreamGoMemLimit is the soft limit handed to litestream's Go runtime:
+// three quarters of the cap. The runtime does not read the cgroup's limit, so
+// without this it lets the heap double before collecting and is killed at a
+// cap the live data would have fitted under. The quarter left over is for
+// what the runtime does not count: the page cache charged to the unit and
+// SQLite's own allocations.
+func litestreamGoMemLimit(max int64) int64 { return max / 4 * 3 }
+
+// LitestreamMemoryDropIn renders the drop-in for a config naming n databases.
+// Same count, same bytes: the verb's changed/unchanged report depends on it.
+func LitestreamMemoryDropIn(n int) string {
+ max := LitestreamMemoryMax(n)
+ return "# GENERATED by `carlos ops litestream` — do not hand-edit; it is rewritten when\n" +
+ "# the number of replicated databases changes. To override it, add a drop-in\n" +
+ "# that sorts after this one.\n" +
+ "[Service]\n" +
+ fmt.Sprintf("MemoryMax=%dM\n", max>>20) +
+ fmt.Sprintf("Environment=GOMEMLIMIT=%dMiB\n", litestreamGoMemLimit(max)>>20)
+}
+
+// writeMemoryDropIn writes the drop-in for n databases to path and, when the
+// file changed, has systemd re-read its units through reload. A reload that
+// fails is returned with wrote still true: the file is in place and the
+// caller says how to finish the job by hand.
+func writeMemoryDropIn(path string, n int, reload func() error) (wrote bool, err error) {
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ return false, err
+ }
+ wrote, err = writeIfChanged(path, LitestreamMemoryDropIn(n))
+ if err != nil || !wrote {
+ return wrote, err
+ }
+ return true, reload()
+}
diff --git a/internal/ops/ops.go b/internal/ops/ops.go
index dc24bb278..e4192b0ec 100644
--- a/internal/ops/ops.go
+++ b/internal/ops/ops.go
@@ -13,6 +13,7 @@ import (
"fmt"
"log"
"os"
+ "os/exec"
"strings"
"time"
@@ -99,6 +100,7 @@ func runLitestream(args []string) {
out := fs.String("out", defaultConfigPath, "config file to write")
lsBin := fs.String("litestream-bin", "litestream", "litestream binary to check the snapshot policy against (pinned release "+litestreamVersion+")")
allowShrink := fs.Bool("allow-shrink", false, "permit a config that replicates FEWER databases than the one it replaces")
+ memDropIn := fs.String("memory-dropin", "", "systemd drop-in receiving the replicator's memory cap, sized from the database count (default "+defaultMemoryDropIn+" when --out is "+defaultConfigPath+"; \"none\" to leave the unit's cap alone)")
fs.Usage = cliflag.Usage(fs, "")
_ = fs.Parse(args)
@@ -153,6 +155,30 @@ should no longer be replicated.`,
}
fmt.Fprintf(os.Stderr, "litestream: %d databases -> s3://%s/%s/ (%s)\n", len(dbs), *bucket, *prefix, *out)
+ // The cap follows the count this run just wrote (see litestream_memory.go).
+ // A run writing somewhere other than the live config is a preview or a
+ // test, and leaves systemd alone unless told where to write.
+ dropIn := *memDropIn
+ if dropIn == "" && *out == defaultConfigPath {
+ dropIn = defaultMemoryDropIn
+ }
+ if dropIn != "" && dropIn != "none" {
+ capped, err := writeMemoryDropIn(dropIn, len(dbs), systemdDaemonReload)
+ switch {
+ case err != nil && !capped:
+ // Loud, not fatal: the config above is written and correct, and
+ // refusing the restart it needs would trade a cap that may be too
+ // small for a database set that is certainly stale.
+ fmt.Fprintf(os.Stderr, "litestream: WARNING: could not write the memory cap to %s: %v\n The replicator keeps its current cap.\n", dropIn, err)
+ case err != nil:
+ fmt.Fprintf(os.Stderr, "litestream: WARNING: wrote the memory cap to %s but systemd did not reload: %v\n Run: systemctl daemon-reload && systemctl restart carlos-litestream\n", dropIn, err)
+ }
+ if capped {
+ fmt.Fprintf(os.Stderr, "litestream: memory cap %dM for %d databases (%s)\n", LitestreamMemoryMax(len(dbs))>>20, len(dbs), dropIn)
+ wrote = true
+ }
+ }
+
// The config now STATES a recovery window, so say plainly when the binary
// on this box will ignore it. Loud, not fatal: a box on an old litestream
// still wants its database SET kept current (that is what carries
@@ -175,6 +201,16 @@ should no longer be replicated.`,
}
}
+// systemdDaemonReload has systemd re-read its unit files, which is what makes
+// a drop-in this verb just wrote the unit's definition at its next start.
+func systemdDaemonReload() error {
+ out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("systemctl daemon-reload: %v: %s", err, strings.TrimSpace(string(out)))
+ }
+ return nil
+}
+
// runRestoreVerify — restore every database in the generated config from S3
// into a temp dir and prove each opens clean. Exits non-zero on any failure so
// the oneshot unit lands in 'failed' where the fleet sweeps see it.
diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go
index c16d4c5aa..190e8b784 100644
--- a/internal/ops/ops_test.go
+++ b/internal/ops/ops_test.go
@@ -2,9 +2,11 @@ package ops
import (
"database/sql"
+ "errors"
"os"
"path/filepath"
"slices"
+ "strconv"
"strings"
"testing"
"time"
@@ -870,3 +872,91 @@ func TestLayoutJournalPresentIsLstatNotAbsent(t *testing.T) {
t.Fatal("another host was reported held")
}
}
+
+// The cap follows the database count: never below the unit's own default, and
+// at the one measured point (#284: 100 databases ran clean under 1G where
+// 256M killed them on every start) at least as large as the cap that held.
+func TestLitestreamMemoryMaxFollowsTheDatabaseCount(t *testing.T) {
+ for _, n := range []int{-1, 0, 1, 10, 32} {
+ if got := LitestreamMemoryMax(n); got != litestreamMemoryFloor {
+ t.Errorf("LitestreamMemoryMax(%d) = %dM, want the %dM floor", n, got>>20, int64(litestreamMemoryFloor)>>20)
+ }
+ }
+ if got := LitestreamMemoryMax(100); got < 1<<30 {
+ t.Errorf("LitestreamMemoryMax(100) = %dM, below the 1G that held 100 databases", got>>20)
+ }
+ prev := LitestreamMemoryMax(0)
+ for n := 1; n <= 2000; n++ {
+ got := LitestreamMemoryMax(n)
+ if got < prev {
+ t.Fatalf("LitestreamMemoryMax(%d) = %d, less than at %d databases (%d)", n, got, n-1, prev)
+ }
+ prev = got
+ }
+}
+
+// The unit file's default and the generator's floor are one number written in
+// two places. If they part, the drop-in either lowers a box's cap or the
+// floor stops being what an ungenerated box runs under.
+func TestLitestreamUnitDefaultIsTheFloor(t *testing.T) {
+ unit, err := os.ReadFile(filepath.Join("..", "host", "units", "carlos-litestream.service"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ floor := int64(litestreamMemoryFloor)
+ for _, want := range []string{
+ "MemoryMax=" + strconv.FormatInt(floor>>20, 10) + "M\n",
+ "Environment=GOMEMLIMIT=" + strconv.FormatInt(litestreamGoMemLimit(floor)>>20, 10) + "MiB\n",
+ } {
+ if !strings.Contains(string(unit), "\n"+want) {
+ t.Errorf("carlos-litestream.service does not carry %q", strings.TrimSpace(want))
+ }
+ }
+ if got := LitestreamMemoryDropIn(0); !strings.Contains(got, "MemoryMax="+strconv.FormatInt(floor>>20, 10)+"M\n") {
+ t.Errorf("the drop-in for an empty config does not state the floor:\n%s", got)
+ }
+}
+
+// A drop-in is a restart-worthy change, so it is written, and systemd asked to
+// re-read its units, only when the count moved the cap.
+func TestWriteMemoryDropInReloadsOnlyOnChange(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "carlos-litestream.service.d", "50-memory.conf")
+ reloads := 0
+ reload := func() error { reloads++; return nil }
+
+ wrote, err := writeMemoryDropIn(path, 100, reload)
+ if err != nil || !wrote || reloads != 1 {
+ t.Fatalf("first write: wrote=%v err=%v reloads=%d, want true, nil, 1", wrote, err, reloads)
+ }
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, want := range []string{"[Service]\n", "MemoryMax=1056M\n", "Environment=GOMEMLIMIT=792MiB\n"} {
+ if !strings.Contains(string(body), want) {
+ t.Errorf("drop-in for 100 databases lacks %q:\n%s", strings.TrimSpace(want), body)
+ }
+ }
+
+ // The same count changes no byte, so it reloads nothing.
+ if wrote, err := writeMemoryDropIn(path, 100, reload); err != nil || wrote || reloads != 1 {
+ t.Fatalf("same count: wrote=%v err=%v reloads=%d, want false, nil, 1", wrote, err, reloads)
+ }
+
+ if wrote, err := writeMemoryDropIn(path, 101, reload); err != nil || !wrote || reloads != 2 {
+ t.Fatalf("one more database: wrote=%v err=%v reloads=%d, want true, nil, 2", wrote, err, reloads)
+ }
+}
+
+// A reload that fails leaves the file written and says so, so the verb can
+// tell the operator how to finish rather than report nothing changed.
+func TestWriteMemoryDropInReportsAFailedReload(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "d", "50-memory.conf")
+ wrote, err := writeMemoryDropIn(path, 200, func() error { return errors.New("no systemd here") })
+ if !wrote || err == nil {
+ t.Fatalf("wrote=%v err=%v, want true and the reload error", wrote, err)
+ }
+ if _, statErr := os.Stat(path); statErr != nil {
+ t.Fatalf("the drop-in was not left in place: %v", statErr)
+ }
+}