amadan

carlos / platform Public

Merge branch 'sweep-staged-versions' into main

claude caf60d122425580931bcd5626ac6a4b08f505a3e parent 851a531parent 2ad59ac
10 files changed, +967 −12
  • internal/host/adoptlock.go +66 −0
  • internal/host/agent.go +37 −0
  • internal/host/bootstrap.go +64 −0
  • internal/host/host_test.go +36 −0
  • internal/host/restart.go +1 −12
  • internal/host/systemapply.go +25 −0
  • internal/host/versionsweep.go +94 −0
  • internal/host/versionsweep_test.go +187 −0
  • internal/release/sweep.go +201 −0
  • internal/release/sweep_test.go +256 −0

CI passed — run details

diff --git a/internal/host/adoptlock.go b/internal/host/adoptlock.go
new file mode 100644
index 000000000..70bfa6599
--- /dev/null
+++ b/internal/host/adoptlock.go
@@ -0,0 +1,66 @@
+package host
+
+import (
+ "os"
+ "path/filepath"
+)
+
+// adoptPassLockFile is the lock every root adoption pass holds for its whole
+// run. It sits in the data dir beside the restart marks, so the single
+// filesystem the sandboxed agent and the root passes both write is the single
+// filesystem their coordination lives on.
+const adoptPassLockFile = "adopt-pass.lock"
+
+func adoptPassLockPath(dd string) string { return filepath.Join(dd, adoptPassLockFile) }
+
+// lockAdoptPass takes an exclusive flock for the whole of one RunAdopt, in
+// EVERY mode.
+//
+// Four units exec `carlos adopt`: carlos-adopt.service (hourly),
+// carlos-stage.service (the agent relaying a stage it is sandboxed out of),
+// carlos-restart.service and carlos-config-sync.service — and systemd will
+// start them in the same second without a thought. Until the staged-version
+// sweep existed that was tolerable: every write a pass performs is a
+// temp-file-then-rename or a symlink swap, so two passes doing the same work
+// converge on the same answer.
+//
+// os.RemoveAll is the first operation in a pass that is not idempotent
+// against a concurrent one. The window is small and real: the sweep reads the
+// live links, a stage pass repoints one at a version the sweep has just
+// decided is unpinned, and the sweep deletes the tree that link now names.
+// Rename-to-a-trash-dir does not close it either, because PointLive stats the
+// target and THEN symlinks — a rename between those two leaves a dangling
+// link and a unit that will not start. Serialising the passes does close it,
+// so that is what this is.
+//
+// The lock is a file of its own rather than a flock on something the pass
+// also writes: a flock lives on an inode, and every state file a pass owns is
+// replaced by rename, which would leave the winner holding a lock on a file
+// that is no longer the state (lockRestartMarks makes the same choice for the
+// same reason).
+//
+// Blocking, like lockRestartMarks: a relayed stage that waits out an hourly
+// pass is doing the right thing, and a stage is already a request the agent
+// re-raises on its next loop.
+func lockAdoptPass(dd string) (unlock func(), err error) {
+ return lockExclusive(adoptPassLockPath(dd))
+}
+
+// lockExclusive opens (creating) path and takes an exclusive advisory lock on
+// it, returning the release. Shared by lockAdoptPass and lockRestartMarks so
+// the two locks cannot drift in how they are taken; the ORDER between them is
+// fixed by their callers, adopt-pass first, and there is no third holder.
+func lockExclusive(path string) (unlock func(), err error) {
+ f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
+ if err != nil {
+ return nil, err
+ }
+ if err := lockFileEx(f); err != nil {
+ f.Close()
+ return nil, err
+ }
+ return func() {
+ unlockFile(f)
+ f.Close()
+ }, nil
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 79fe3e039..f7ecf08d5 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -2923,6 +2923,22 @@ func RunAdopt(args []string) {
restartOnly := fs.Bool("restart-only", false, "cycle the routes of apps whose restart generation moved, and nothing else: the agent-requested pass (carlos-restart.service)")
fs.Usage = cliflag.Usage(fs, "")
_ = fs.Parse(args)
+ // One exclusive lock for the whole pass, in every mode, because the four
+ // units that exec this verb can start in the same second and the sweep
+ // below is the first thing a pass does that a concurrent pass cannot
+ // safely repeat (lockAdoptPass).
+ //
+ // A lock that cannot be taken is NOT fatal: it drops the sweep and runs
+ // the pass exactly as this verb ran before the sweep existed. Every other
+ // half is idempotent against a concurrent pass — that is why they were
+ // unserialized for a year — so the honest response to "I cannot serialize"
+ // is to skip the one half that needs it, not to stop converging the box.
+ unlockPass, passLockErr := lockAdoptPass(dataDir())
+ if passLockErr != nil {
+ log.Printf("adopt: pass lock %s unavailable (%v) — continuing unserialized, staged-version sweep skipped", adoptPassLockPath(dataDir()), passLockErr)
+ } else {
+ defer unlockPass()
+ }
// Spec §7 belt 2: arm the self-update relay if the last hand roll did
// not. Before the fatal gates below, and never fatal itself.
if !*restartOnly {
@@ -2941,6 +2957,27 @@ func RunAdopt(args []string) {
}
log.Fatalf("adopt: open registry %s: %v", *regPath, err)
}
+
+ // Step 0 (discussion #111): evict the staged versions nothing on this box
+ // still names, BEFORE anything that can fail because the disk is full.
+ //
+ // The placement is the whole point of the fix, not a detail. Below this
+ // line are three fatal gates — openDeployStore, the release pubkey parse,
+ // and release.Pass, whose bake clock is written with CreateTemp under
+ // /data and which log.Fatalf's the pass on error. On the box this is for,
+ // with 1.2 MB free, a sweep placed after any of them never runs, and the
+ // box has no way to free itself. It needs only the registry and /opt, so
+ // it goes as early as having a registry allows. restartPass sits below
+ // release.Pass's fatality for exactly this reason and says so.
+ //
+ // Hourly only: -stage-only is triggered per relayed release, up to once a
+ // minute per loop, and walking the version store that often to free a
+ // day's worth of shipping is work for no gain. -config-only and
+ // -restart-only are narrow triggers that must do their one job.
+ if passLockErr == nil && !*stageOnly && !*configOnly && !*restartOnly {
+ sweepStagedVersionsPass(reg)
+ }
+
ctx := context.Background()
store, err := openDeployStore(ctx)
if err != nil {
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go
index e7e907956..0b258385d 100644
--- a/internal/host/bootstrap.go
+++ b/internal/host/bootstrap.go
@@ -61,6 +61,34 @@ const socketsGroup = "carlos-sockets"
const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup + " -\n" +
"d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n"
+// journaldConfName is the journald drop-in every box gets (discussion #111).
+//
+// The name matches the one placed by hand on the flagship during the
+// 2026-09-06 relief, and must keep matching it: drop-ins are read in name
+// order and the LAST assignment wins, so a differently-named file would sit
+// beside the hand-placed one and whichever sorted later would decide the cap.
+// Same name means this file supersedes it on every box that has it, and the
+// hand-placed copy can be deleted or left alone with the same result.
+const journaldConfName = "50-cap.conf"
+
+// journaldConf caps the journal. Uncapped, journald's default is 10% of the
+// filesystem — 2 GB of a 20 GB root, of which the flagship had spent 1.2 GB
+// by the time it filled. A cap is not a fix for a chatty box, it is the
+// difference between a chatty box losing old log lines and a chatty box
+// losing the ability to accept a push.
+//
+// Both halves are set explicitly. SystemMaxUse governs /var/log/journal, the
+// persistent journal, which is the one that filled; RuntimeMaxUse governs
+// /run/log/journal, which is tmpfs — so its default 10% is 10% of RAM, and on
+// a small box a journal storm there is memory pressure rather than disk. One
+// drop-in that leaves the volatile half at the default would have to be
+// reasoned about again the first time that matters.
+const journaldConf = "# Written by carlos bootstrap (discussion #111). Do not edit by hand:\n" +
+ "# the next bootstrap or self-update apply rewrites this file.\n" +
+ "[Journal]\n" +
+ "SystemMaxUse=256M\n" +
+ "RuntimeMaxUse=64M\n"
+
// enabledUnits are what a live bootstrap turns on: the always-on front door,
// the hourly adoption timer, and the path units that let the sandboxed
// agent ask a root pass for the halves it cannot do itself: carlos-adopt.path
@@ -151,6 +179,14 @@ func systemctlPlan(units []string) [][]string {
return plan
}
+// journaldReload is the one systemctl call that makes a freshly written
+// journald drop-in take effect. Kept out of systemctlPlan on purpose: that
+// plan is "enable these units and arm the triggers among them", and it is
+// also what the self-update relay's arm step and ensureSystemUnits run over
+// narrow unit lists — neither of which should be bouncing the box's logging
+// daemon. Both places that install the drop-in call this explicitly instead.
+var journaldReload = []string{"restart", "systemd-journald"}
+
// RunBootstrap is the `carlos bootstrap` verb. Idempotent — safe to re-run
// (which is also how a box enrolls a new socket consumer after the fact).
func RunBootstrap(args []string) {
@@ -281,6 +317,10 @@ func bootstrapWith(root string, consumers []string, planJSON bool) error {
// root before dropping to the unit's DynamicUser, so 0600 root works).
{filepath.Join(root, "etc", "carlos", "apps"), 0o755},
{unitDir, 0o755},
+ // The journal cap's drop-in dir. systemd creates it on demand for a
+ // hand-placed file, but nothing creates it on a fresh box, and the
+ // write below would fail ENOENT if this line were missing.
+ {filepath.Join(root, "etc", "systemd", "journald.conf.d"), 0o755},
{filepath.Join(root, "etc", "tmpfiles.d"), 0o755},
{filepath.Join(root, "etc", "sysusers.d"), 0o755},
}
@@ -379,6 +419,20 @@ func bootstrapWith(root string, consumers []string, planJSON bool) error {
}
fmt.Fprintf(out, " wrote %s\n", sysusersPath)
+ // The journal cap (discussion #111). Written on every root, live or
+ // staging, because a staging root is how a live box refreshes its etc/
+ // tree — both the self-update apply (systemapply.go's installUnits) and
+ // pinfra's update-platform-binary.sh stage with `bootstrap -root <dir>`
+ // and copy named files out of it. The apply path names this one; the
+ // pinfra script copies only units, sysusers and tmpfiles today, so a box
+ // updated by hand gets the cap on its next self-update or bootstrap
+ // rather than on that script's next run.
+ journaldPath := filepath.Join(root, "etc", "systemd", "journald.conf.d", journaldConfName)
+ if err := os.WriteFile(journaldPath, []byte(journaldConf), 0o644); err != nil {
+ return fmt.Errorf("write %s: %v", journaldPath, err)
+ }
+ fmt.Fprintf(out, " wrote %s\n", journaldPath)
+
tmpfilesPath := filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf")
if err := os.WriteFile(tmpfilesPath, []byte(tmpfilesConf), 0o644); err != nil {
return fmt.Errorf("write %s: %v", tmpfilesPath, err)
@@ -401,6 +455,7 @@ func bootstrapWith(root string, consumers []string, planJSON bool) error {
for _, args := range systemctlPlan(enabledUnits) {
fmt.Printf(" systemctl %s\n", strings.Join(args, " "))
}
+ fmt.Printf(" systemctl %s\n", strings.Join(journaldReload, " "))
return nil
}
@@ -409,6 +464,15 @@ func bootstrapWith(root string, consumers []string, planJSON bool) error {
return err
}
}
+ // A journald drop-in is inert until journald re-reads its configuration,
+ // and journald re-reads it only on restart — so a bootstrap that writes
+ // the cap and stops leaves a box whose journal is capped from its next
+ // reboot, which on a long-lived box is never. Non-fatal: the cap is a
+ // safety margin, and refusing to finish bootstrap over it would be a
+ // worse outcome than an uncapped journal on one box.
+ if err := systemctl(journaldReload...); err != nil {
+ fmt.Printf("bootstrap: could not restart systemd-journald (%v) — the journal cap in %s takes effect at the next restart\n", err, journaldConfName)
+ }
fmt.Printf("bootstrap: units installed, %s enabled (timers and path units armed, not just symlinked)\n", strings.Join(enabledUnits, " + "))
fmt.Println(" next: copy the carlos binary to /opt/carlos/carlos, stage /etc/carlos/host.env, then: systemctl start carlos-edge")
return nil
diff --git a/internal/host/host_test.go b/internal/host/host_test.go
index fcd2383bf..1a5b09dbd 100644
--- a/internal/host/host_test.go
+++ b/internal/host/host_test.go
@@ -1575,3 +1575,39 @@ func TestVerifyDrillsRestoreToDiskNotTmpfs(t *testing.T) {
}
}
}
+
+// The journal cap ships with every bootstrap (discussion #111): uncapped,
+// journald's default is 10% of the filesystem, and 1.2 GB of the flagship's
+// 20 GB root was journal by the time it filled.
+func TestBootstrapWritesJournaldCap(t *testing.T) {
+ root := t.TempDir()
+ if err := bootstrap(root, nil); err != nil {
+ t.Fatalf("bootstrap: %v", err)
+ }
+ path := filepath.Join(root, "etc", "systemd", "journald.conf.d", journaldConfName)
+ b, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("journald drop-in not written: %v", err)
+ }
+ got := string(b)
+ // The section header is not optional: journald ignores a bare assignment.
+ for _, want := range []string{"[Journal]", "SystemMaxUse=256M", "RuntimeMaxUse="} {
+ if !strings.Contains(got, want) {
+ t.Errorf("drop-in missing %q:\n%s", want, got)
+ }
+ }
+ // The name has to match the one placed by hand on the flagship during the
+ // 2026-09-06 relief, or the two files sit side by side and name order
+ // decides which cap wins.
+ if journaldConfName != "50-cap.conf" {
+ t.Errorf("journaldConfName = %q; it must supersede the hand-placed 50-cap.conf", journaldConfName)
+ }
+ // A staged root is the delivery path to boxes that predate the cap: the
+ // self-update apply copies this file out of one by name (installUnits).
+ if err := bootstrap(root, nil); err != nil {
+ t.Fatalf("re-run: %v", err)
+ }
+ if _, err := os.Stat(path); err != nil {
+ t.Errorf("drop-in gone after a re-run: %v", err)
+ }
+}
diff --git a/internal/host/restart.go b/internal/host/restart.go
index 757f7b740..0a3584904 100644
--- a/internal/host/restart.go
+++ b/internal/host/restart.go
@@ -321,18 +321,7 @@ func restartMarkKey(acct, app string) string { return acct + "|" + app }
// darwin flock, Windows refuses (see flock_windows.go — refusing dispatches
// nothing, which is the safe answer on a platform that never runs a pass).
func lockRestartMarks(path string) (unlock func(), err error) {
- f, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0o644)
- if err != nil {
- return nil, err
- }
- if err := lockFileEx(f); err != nil {
- f.Close()
- return nil, err
- }
- return func() {
- unlockFile(f)
- f.Close()
- }, nil
+ return lockExclusive(path + ".lock")
}
// newRestartMarks is an empty memory: both maps non-nil, so every caller can
diff --git a/internal/host/systemapply.go b/internal/host/systemapply.go
index f09681f5c..5ad62425e 100644
--- a/internal/host/systemapply.go
+++ b/internal/host/systemapply.go
@@ -550,6 +550,31 @@ func (a *systemApplier) installUnits(ctx context.Context, stagedRoot string, rem
return fmt.Errorf("systemd-sysusers: %w", err)
}
}
+ // The journal cap (discussion #111). This is how a box that was
+ // bootstrapped before the cap existed actually gets it: the drop-in is a
+ // named file in the staged etc/ tree, exactly like sysusers and tmpfiles
+ // below, and copying it here means the fleet converges on the cap through
+ // the self-update relay rather than one box at a time by hand.
+ //
+ // Stat-guarded like sysusers: a ROLLBACK stages carlos.prev, and a
+ // previous build whose bootstrap wrote no drop-in must leave whatever is
+ // on the box alone rather than fail the restore.
+ //
+ // journald is restarted, not just reloaded, because that is the only way
+ // it re-reads its configuration; failure is logged and the apply
+ // continues, since an uncapped journal must not roll back a good build.
+ jrnl := filepath.Join(stagedRoot, "etc", "systemd", "journald.conf.d", journaldConfName)
+ if _, err := os.Stat(jrnl); err == nil {
+ if err := os.MkdirAll(filepath.Join(a.etcDir, "systemd", "journald.conf.d"), 0o755); err != nil {
+ return err
+ }
+ if err := copyFileDurable(jrnl, filepath.Join(a.etcDir, "systemd", "journald.conf.d", journaldConfName), 0o644); err != nil {
+ return err
+ }
+ if _, err := a.ops.systemctl(journaldReload...); err != nil {
+ a.ops.logf("system apply: restart systemd-journald: %v (continuing; the journal cap takes effect at the next restart)", err)
+ }
+ }
tmp := filepath.Join(stagedRoot, "etc", "tmpfiles.d", "carlos.conf")
if err := os.MkdirAll(filepath.Join(a.etcDir, "tmpfiles.d"), 0o755); err != nil {
return err
diff --git a/internal/host/versionsweep.go b/internal/host/versionsweep.go
new file mode 100644
index 000000000..0bf858e79
--- /dev/null
+++ b/internal/host/versionsweep.go
@@ -0,0 +1,94 @@
+package host
+
+import (
+ "log"
+ "os"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// versionSweepGraceEnv overrides release.DefaultSweepGrace, as a Go duration.
+// An escape hatch for a box that ships far faster than the fleet (or an
+// operator who wants a longer rollback window on one), not something any unit
+// sets: the default is the fleet's answer.
+const versionSweepGraceEnv = "CARLOS_VERSIONS_GRACE"
+
+func versionSweepGrace(getenv func(string) string) time.Duration {
+ raw := getenv(versionSweepGraceEnv)
+ if raw == "" {
+ return release.DefaultSweepGrace
+ }
+ d, err := time.ParseDuration(raw)
+ if err != nil || d < 0 {
+ log.Printf("adopt: %s=%q is not a non-negative duration — sweeping with the default %s", versionSweepGraceEnv, raw, release.DefaultSweepGrace)
+ return release.DefaultSweepGrace
+ }
+ return d
+}
+
+// routeLister is what the sweep needs of the registry: the rows, unfiltered.
+// Deliberately NOT release.Adoptable or any awake/canary/TTL filter — those
+// answer "should this route move to a new version", and the question here is
+// "is this version still spoken for", which a hibernating route, an expired
+// canary and a route mid-rollback all answer yes to.
+type routeLister interface {
+ List() ([]registry.Route, error)
+}
+
+// sweepStagedVersions evicts the staged version trees nothing on this box
+// still names (discussion #111).
+//
+// It never fails the pass. Every failure path here declines to delete and
+// returns: an unreadable live dir or an unlistable registry means the pin set
+// is incomplete, and an incomplete pin set is how a running app loses its
+// binary. Freeing disk an hour later is cheap; deleting a live version is not.
+func sweepStagedVersions(reg routeLister, optDir string, grace time.Duration, now time.Time) {
+ pinned, err := release.LivePins(optDir)
+ if err != nil {
+ log.Printf("adopt: staged-version sweep skipped: %v", err)
+ return
+ }
+ live := len(pinned)
+ routes, err := reg.List()
+ if err != nil {
+ log.Printf("adopt: staged-version sweep skipped: list routes: %v", err)
+ return
+ }
+ for _, r := range routes {
+ // PrevVersion is a pin with no grace attached: it is the rollback
+ // target, and a rollback is wanted precisely when a box has been
+ // running the new build long enough for the grace to have expired.
+ if r.Version != "" {
+ pinned[r.Version] = true
+ }
+ if r.PrevVersion != "" {
+ pinned[r.PrevVersion] = true
+ }
+ }
+ res, err := release.SweepStaged(optDir, pinned, grace, now)
+ if err != nil {
+ log.Printf("adopt: staged-version sweep: %v", err)
+ return
+ }
+ for _, e := range res.Errs {
+ log.Printf("adopt: staged-version sweep: %v", e)
+ }
+ if len(res.Removed) == 0 {
+ // One line an hour on a box with nothing to do is worth having: it is
+ // the difference between "the sweep found nothing" and "the sweep is
+ // not running", which is the question the next full disk will ask.
+ log.Printf("adopt: staged-version sweep: %d staged, %d pinned (%d live), %d within the %s grace, none removed",
+ res.Scanned, res.Pinned, live, res.Young, grace)
+ return
+ }
+ log.Printf("adopt: staged-version sweep: removed %d of %d staged versions (%d bytes), %d pinned (%d live), %d within the %s grace: %v",
+ len(res.Removed), res.Scanned, res.Bytes, res.Pinned, live, res.Young, grace, res.Removed)
+}
+
+// sweepStagedVersionsPass is the RunAdopt entry point: the grace from the
+// environment, the clock from the wall.
+func sweepStagedVersionsPass(reg routeLister) {
+ sweepStagedVersions(reg, release.OptDir(), versionSweepGrace(os.Getenv), time.Now())
+}
diff --git a/internal/host/versionsweep_test.go b/internal/host/versionsweep_test.go
new file mode 100644
index 000000000..74d1e5f66
--- /dev/null
+++ b/internal/host/versionsweep_test.go
@@ -0,0 +1,187 @@
+package host
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type fakeLister struct {
+ routes []registry.Route
+ err error
+}
+
+func (f fakeLister) List() ([]registry.Route, error) { return f.routes, f.err }
+
+func stageSweepVersion(t *testing.T, optDir, acct, ver, app string, age time.Duration) {
+ t.Helper()
+ p := release.VersionPath(optDir, acct, ver, app)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte("binary"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ when := time.Now().Add(-age)
+ if err := os.Chtimes(p, when, when); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func staged(t *testing.T, optDir, ver string) bool {
+ t.Helper()
+ _, err := os.Stat(filepath.Join(optDir, "versions", ver))
+ return err == nil
+}
+
+// PrevVersion is a pin with no grace: a rollback is wanted precisely when the
+// new build has been running long enough for the grace to have expired.
+func TestSweepStagedVersionsPinsPrevVersion(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v3", "blog", 48*time.Hour)
+ stageSweepVersion(t, opt, "acct1", "v2", "blog", 48*time.Hour)
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+
+ reg := fakeLister{routes: []registry.Route{{Host: "blog.example.com", Version: "v3", PrevVersion: "v2"}}}
+ sweepStagedVersions(reg, opt, 6*time.Hour, time.Now())
+
+ if !staged(t, opt, "v3") {
+ t.Error("v3 (Version) was removed")
+ }
+ if !staged(t, opt, "v2") {
+ t.Error("v2 (PrevVersion) was removed — the rollback target is a pin")
+ }
+ if staged(t, opt, "v1") {
+ t.Error("v1 survived; it is pinned by nothing and older than the grace")
+ }
+}
+
+// A route with no version at all must not pin the empty string into
+// something the sweep then matches.
+func TestSweepStagedVersionsIgnoresEmptyVersions(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+ reg := fakeLister{routes: []registry.Route{{Host: "pending.example.com"}}}
+ sweepStagedVersions(reg, opt, 6*time.Hour, time.Now())
+ if staged(t, opt, "v1") {
+ t.Error("v1 survived a pin set that should be empty")
+ }
+}
+
+// An unlistable registry means an incomplete pin set, and an incomplete pin
+// set is how a running app loses its binary. Delete nothing.
+func TestSweepStagedVersionsDeletesNothingWhenRegistryFails(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+ sweepStagedVersions(fakeLister{err: errors.New("database is locked")}, opt, 6*time.Hour, time.Now())
+ if !staged(t, opt, "v1") {
+ t.Error("swept with an unreadable registry")
+ }
+}
+
+// Likewise for an unreadable live dir.
+func TestSweepStagedVersionsDeletesNothingWhenLivePinsFail(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour)
+ if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(opt, "live", "stray"), []byte("not a link"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ sweepStagedVersions(fakeLister{}, opt, 6*time.Hour, time.Now())
+ if !staged(t, opt, "v1") {
+ t.Error("swept with an unreadable live dir")
+ }
+}
+
+// The live link is believed over the registry: PointLive succeeding and
+// SetVersions then failing on a full /data is the incident's own state, and
+// the link is the only record of what is running.
+func TestSweepStagedVersionsBelievesLiveLinkOverRegistry(t *testing.T) {
+ opt := t.TempDir()
+ stageSweepVersion(t, opt, release.OpsAccount, "v2", "edge", 48*time.Hour)
+ if err := release.PointLive(opt, "edge.example.com", release.OpsAccount, "v2", "edge"); err != nil {
+ t.Fatal(err)
+ }
+ // The registry still says v1 — the version write never landed.
+ reg := fakeLister{routes: []registry.Route{{Host: "edge.example.com", Version: "v1"}}}
+ sweepStagedVersions(reg, opt, 6*time.Hour, time.Now())
+ if !staged(t, opt, "v2") {
+ t.Fatal("the running version was removed because the registry had not caught up")
+ }
+}
+
+func TestVersionSweepGrace(t *testing.T) {
+ env := func(v string) func(string) string {
+ return func(k string) string {
+ if k == versionSweepGraceEnv {
+ return v
+ }
+ return ""
+ }
+ }
+ if got := versionSweepGrace(env("")); got != release.DefaultSweepGrace {
+ t.Errorf("unset = %s, want %s", got, release.DefaultSweepGrace)
+ }
+ if got := versionSweepGrace(env("30m")); got != 30*time.Minute {
+ t.Errorf("30m = %s", got)
+ }
+ // Garbage and negatives fall back rather than sweeping with a zero grace.
+ for _, bad := range []string{"soon", "-1h"} {
+ if got := versionSweepGrace(env(bad)); got != release.DefaultSweepGrace {
+ t.Errorf("%q = %s, want the default %s", bad, got, release.DefaultSweepGrace)
+ }
+ }
+}
+
+// The pass lock is what makes the sweep safe against a concurrent
+// carlos-stage.service, so it has to actually exclude.
+func TestLockAdoptPassExcludes(t *testing.T) {
+ dd := t.TempDir()
+ unlock, err := lockAdoptPass(dd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ held := make(chan struct{})
+ go func() {
+ second, err := lockAdoptPass(dd)
+ if err == nil {
+ second()
+ }
+ close(held)
+ }()
+ select {
+ case <-held:
+ t.Fatal("a second pass took the lock while the first held it")
+ case <-time.After(100 * time.Millisecond):
+ }
+ unlock()
+ select {
+ case <-held:
+ case <-time.After(5 * time.Second):
+ t.Fatal("the second pass never got the lock after the first released it")
+ }
+}
+
+// The adopt lock and the restart marks lock are different files: RunAdopt
+// holds the first for the whole pass and restartPass takes the second inside
+// it, so sharing an inode would deadlock every pass that restarts anything.
+func TestAdoptAndRestartLocksAreDistinct(t *testing.T) {
+ dd := t.TempDir()
+ unlock, err := lockAdoptPass(dd)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer unlock()
+ inner, err := lockRestartMarks(restartMarksPath(dd))
+ if err != nil {
+ t.Fatalf("restart marks lock blocked by the pass lock: %v", err)
+ }
+ inner()
+}
diff --git a/internal/release/sweep.go b/internal/release/sweep.go
new file mode 100644
index 000000000..5e4bf1a5e
--- /dev/null
+++ b/internal/release/sweep.go
@@ -0,0 +1,201 @@
+// Cache eviction for the on-box version store (discussion #111).
+//
+// StageArtifact writes every version this box is ever asked to run under
+// <opt>/versions/<ver>/... and nothing removes one. Retention
+// (internal/host/retention.go, issue #136) prunes the BUCKET; the box's copy
+// is a cache of it, and an uncapped cache on a 20 GB root filled the flagship
+// on 2026-09-06 — 369 staged versions, 9.3 GB, at 40 to 60 new versions a day.
+//
+// Eviction, not retention: ensureStaged re-downloads a version that is
+// missing before it adopts it, so removing one costs a download, never a
+// deploy. What it must NOT do is remove a version something is USING, which is
+// the whole content of the pin set below.
+package release
+
+import (
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "time"
+)
+
+// DefaultSweepGrace is how long an unpinned version survives after its last
+// write. Time-driven rather than size-driven on purpose: a size trigger only
+// fires once the disk is ALREADY tight, which is the state in which every
+// other thing on the box (the bake clock, the registry write, the next stage)
+// has begun to fail — see RunAdopt's placement comment. Six hours is long
+// enough to cover the window between staging a version and pointing a live
+// link at it (promotions, canaries and rollbacks all happen inside minutes),
+// and short enough that a day's shipping cannot accumulate.
+const DefaultSweepGrace = 6 * time.Hour
+
+// SweepResult is one sweep's accounting, for the log line. Errs holds the
+// per-version failures: a version that cannot be scanned or removed is
+// skipped, never fatal, because one unreadable tree must not stop the sweep
+// freeing the rest of the disk.
+type SweepResult struct {
+ Scanned int // entries under <opt>/versions
+ Skipped int // not a directory, or not a version name
+ Pinned int // named by a live link or a registry row
+ Young int // unpinned but written to inside the grace window
+ Removed []string // version directories deleted
+ Bytes int64 // bytes those directories held
+ Errs []error
+}
+
+// SweepStaged removes every <optDir>/versions/<ver> that `pinned` does not
+// name and that nothing has written to for `grace`.
+//
+// The granularity is the VERSION directory, not the per-app subtree inside
+// it: a version dir is shared by every account and app staged at that version
+// (VersionPath), so one pin on any of them keeps the whole directory. Coarse
+// on purpose — the cost is a few megabytes held slightly longer, and the
+// alternative (deleting one app's binary out from under a shared version)
+// buys nothing a later sweep would not collect anyway.
+//
+// Entries that are not directories, and directories whose name ValidVersion
+// rejects, are left alone and counted in Skipped. Nothing this package wrote
+// has such a name, so such an entry is somebody else's — an operator's backup,
+// a hand-placed tree — and the sweep is not entitled to it.
+//
+// The returned error is reserved for "could not read the versions directory
+// at all", the one case in which the caller learned nothing and should not
+// treat an empty Removed list as "there was nothing to remove".
+func SweepStaged(optDir string, pinned map[string]bool, grace time.Duration, now time.Time) (SweepResult, error) {
+ var res SweepResult
+ vdir := versionsDir(optDir)
+ entries, err := os.ReadDir(vdir)
+ if err != nil {
+ if os.IsNotExist(err) {
+ // A box that has never staged anything. Not an error.
+ return res, nil
+ }
+ return res, fmt.Errorf("read %s: %w", vdir, err)
+ }
+ for _, e := range entries {
+ res.Scanned++
+ name := e.Name()
+ // e.IsDir() is false for a symlink (ReadDir does not follow), which
+ // is the answer we want: a link into the store is not a staged tree.
+ if !e.IsDir() || !ValidVersion(name) {
+ res.Skipped++
+ continue
+ }
+ if pinned[name] {
+ res.Pinned++
+ continue
+ }
+ path := filepath.Join(vdir, name)
+ newest, size, err := treeStats(path)
+ if err != nil {
+ res.Errs = append(res.Errs, fmt.Errorf("scan %s: %w", path, err))
+ continue
+ }
+ // A negative Sub (a file dated in the future, from a clock step or a
+ // tarball's preserved mtime) reads as "younger than the grace", which
+ // is the cautious direction.
+ if now.Sub(newest) < grace {
+ res.Young++
+ continue
+ }
+ if err := os.RemoveAll(path); err != nil {
+ res.Errs = append(res.Errs, fmt.Errorf("remove %s: %w", path, err))
+ continue
+ }
+ res.Removed = append(res.Removed, name)
+ res.Bytes += size
+ }
+ return res, nil
+}
+
+// treeStats reports the newest FILE mtime under root, and the bytes the
+// regular files hold.
+//
+// Files, not directories, and that is the subtle part: removing entries from a
+// directory bumps ITS mtime, so a tree the previous sweep half-deleted (a
+// RemoveAll that hit an error partway) would look freshly written and buy
+// itself another full grace period, every period, forever. The files that
+// survive carry the real age.
+//
+// A tree with no files left at all falls back to the newest directory mtime,
+// so an empty husk is still collectable rather than permanently ageless.
+func treeStats(root string) (newest time.Time, bytes int64, err error) {
+ var dirNewest time.Time
+ walkErr := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
+ if err != nil {
+ // A concurrent removal (an operator, an interrupted sweep) is not
+ // this walk's problem: skip what vanished under it.
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ info, err := d.Info()
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ if d.IsDir() {
+ if info.ModTime().After(dirNewest) {
+ dirNewest = info.ModTime()
+ }
+ return nil
+ }
+ if info.ModTime().After(newest) {
+ newest = info.ModTime()
+ }
+ if info.Mode().IsRegular() {
+ bytes += info.Size()
+ }
+ return nil
+ })
+ if walkErr != nil {
+ return time.Time{}, 0, walkErr
+ }
+ if newest.IsZero() {
+ newest = dirNewest
+ }
+ return newest, bytes, nil
+}
+
+// LivePins is the set of versions the box's live symlinks name — the hard
+// pins, because each one is a path some unit's ExecStart resolves through.
+//
+// Read from the LINKS and not from the registry's per-route Version column,
+// deliberately. The two agree on a healthy box, and the case where they do
+// not is exactly the incident's: PointLive succeeded, reg.SetVersions then
+// failed on a full /data, and the link is the only record of what is running.
+// Believing the registry there would delete a live binary.
+//
+// Every entry is read, including PointLive's `<host>.tmp` leftovers from a
+// crashed swap: a leftover names a real staged tree and costs one version dir
+// of disk, which is a better trade than interpreting which links are current.
+//
+// A link this package cannot read at all is an error, not a skipped entry.
+// The sweep's caller declines to delete anything when pins cannot be
+// established — losing a pin is how a running app gets its binary removed,
+// so an unreadable live dir must stop the sweep and be loud, not quietly
+// narrow the pin set.
+func LivePins(optDir string) (map[string]bool, error) {
+ pins := map[string]bool{}
+ entries, err := os.ReadDir(liveDir(optDir))
+ if err != nil {
+ if os.IsNotExist(err) {
+ return pins, nil
+ }
+ return nil, fmt.Errorf("read %s: %w", liveDir(optDir), err)
+ }
+ for _, e := range entries {
+ ver, err := LiveVersion(optDir, e.Name())
+ if err != nil {
+ return nil, fmt.Errorf("read live link %s: %w", LiveLink(optDir, e.Name()), err)
+ }
+ if ver != "" {
+ pins[ver] = true
+ }
+ }
+ return pins, nil
+}
diff --git a/internal/release/sweep_test.go b/internal/release/sweep_test.go
new file mode 100644
index 000000000..2ed360177
--- /dev/null
+++ b/internal/release/sweep_test.go
@@ -0,0 +1,256 @@
+package release
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+// stage writes a version tree the way StageArtifact lays one out, and dates
+// every file it wrote `age` in the past.
+func stage(t *testing.T, optDir, acct, ver, app string, size int, age time.Duration) string {
+ t.Helper()
+ p := VersionPath(optDir, acct, ver, app)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, make([]byte, size), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ when := time.Now().Add(-age)
+ if err := os.Chtimes(p, when, when); err != nil {
+ t.Fatal(err)
+ }
+ return p
+}
+
+func TestSweepStagedRemovesOnlyUnpinnedAndOld(t *testing.T) {
+ opt := t.TempDir()
+ stage(t, opt, OpsAccount, "live", "edge", 10, 48*time.Hour)
+ stage(t, opt, OpsAccount, "pinned-by-registry", "edge", 20, 48*time.Hour)
+ stage(t, opt, "acct1", "old", "blog", 30, 48*time.Hour)
+ stage(t, opt, "acct1", "fresh", "blog", 40, time.Minute)
+
+ if err := PointLive(opt, "edge.example.com", OpsAccount, "live", "edge"); err != nil {
+ t.Fatal(err)
+ }
+ pins, err := LivePins(opt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !pins["live"] || len(pins) != 1 {
+ t.Fatalf("LivePins = %v, want just {live}", pins)
+ }
+ pins["pinned-by-registry"] = true
+
+ res, err := SweepStaged(opt, pins, 6*time.Hour, time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Removed) != 1 || res.Removed[0] != "old" {
+ t.Fatalf("Removed = %v, want [old]", res.Removed)
+ }
+ if res.Bytes != 30 {
+ t.Errorf("Bytes = %d, want 30", res.Bytes)
+ }
+ if res.Pinned != 2 || res.Young != 1 || res.Scanned != 4 {
+ t.Errorf("res = %+v, want 2 pinned, 1 young, 4 scanned", res)
+ }
+ if len(res.Errs) != 0 {
+ t.Errorf("Errs = %v, want none", res.Errs)
+ }
+ for _, keep := range []string{"live", "pinned-by-registry", "fresh"} {
+ if _, err := os.Stat(filepath.Join(opt, "versions", keep)); err != nil {
+ t.Errorf("%s was removed: %v", keep, err)
+ }
+ }
+ if _, err := os.Stat(filepath.Join(opt, "versions", "old")); !os.IsNotExist(err) {
+ t.Errorf("old survived: %v", err)
+ }
+ // The live link must still resolve — the point of the pin.
+ if _, err := os.Stat(LiveLink(opt, "edge.example.com")); err != nil {
+ t.Errorf("live link broken: %v", err)
+ }
+}
+
+// A version dir holds every account's build at that version, so a pin from
+// one account keeps the whole directory — including a second account's app
+// inside it.
+func TestSweepStagedPinsWholeVersionDir(t *testing.T) {
+ opt := t.TempDir()
+ stage(t, opt, "acct1", "v1", "blog", 10, 48*time.Hour)
+ stage(t, opt, "acct2", "v1", "shop", 10, 48*time.Hour)
+
+ res, err := SweepStaged(opt, map[string]bool{"v1": true}, 6*time.Hour, time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Removed) != 0 {
+ t.Fatalf("Removed = %v, want none", res.Removed)
+ }
+ if _, err := os.Stat(VersionPath(opt, "acct2", "v1", "shop")); err != nil {
+ t.Errorf("acct2's build removed by acct1's pin: %v", err)
+ }
+}
+
+// The age of a tree is its newest FILE, not its newest directory: deleting
+// entries bumps the parent's mtime, and a half-swept tree that read as
+// freshly written would renew its own grace forever.
+func TestSweepStagedAgesOnFileMtimeNotDirMtime(t *testing.T) {
+ opt := t.TempDir()
+ p := stage(t, opt, "acct1", "v1", "blog", 10, 48*time.Hour)
+ // A sibling left behind and then removed, as a partial RemoveAll would:
+ // the parent directory's mtime is now.
+ sib := filepath.Join(filepath.Dir(p), "leftover")
+ if err := os.WriteFile(sib, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Remove(sib); err != nil {
+ t.Fatal(err)
+ }
+
+ res, err := SweepStaged(opt, nil, 6*time.Hour, time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Removed) != 1 {
+ t.Fatalf("Removed = %v, want [v1] — a fresh dir mtime must not renew the grace", res.Removed)
+ }
+}
+
+// An emptied husk has no file to age it and must still be collectable.
+func TestSweepStagedRemovesEmptyHusk(t *testing.T) {
+ opt := t.TempDir()
+ dir := filepath.Join(opt, "versions", "v1")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ when := time.Now().Add(-48 * time.Hour)
+ if err := os.Chtimes(dir, when, when); err != nil {
+ t.Fatal(err)
+ }
+ res, err := SweepStaged(opt, nil, 6*time.Hour, time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Removed) != 1 {
+ t.Fatalf("Removed = %v, want [v1]", res.Removed)
+ }
+}
+
+// Entries this package did not write are not ours to delete, however old.
+func TestSweepStagedLeavesForeignEntries(t *testing.T) {
+ opt := t.TempDir()
+ vdir := filepath.Join(opt, "versions")
+ if err := os.MkdirAll(vdir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ loose := filepath.Join(vdir, "notes.txt")
+ if err := os.WriteFile(loose, []byte("hand-written"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ odd := filepath.Join(vdir, "backup 2026") // a space: ValidVersion rejects it
+ if err := os.MkdirAll(odd, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ when := time.Now().Add(-30 * 24 * time.Hour)
+ for _, p := range []string{loose, odd} {
+ if err := os.Chtimes(p, when, when); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ res, err := SweepStaged(opt, nil, 6*time.Hour, time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Removed) != 0 {
+ t.Fatalf("Removed = %v, want none", res.Removed)
+ }
+ if res.Skipped != 2 {
+ t.Errorf("Skipped = %d, want 2", res.Skipped)
+ }
+ for _, p := range []string{loose, odd} {
+ if _, err := os.Stat(p); err != nil {
+ t.Errorf("%s removed: %v", p, err)
+ }
+ }
+}
+
+func TestSweepStagedNoVersionsDir(t *testing.T) {
+ res, err := SweepStaged(t.TempDir(), nil, 6*time.Hour, time.Now())
+ if err != nil {
+ t.Fatalf("a box that has never staged anything is not an error: %v", err)
+ }
+ if res.Scanned != 0 {
+ t.Errorf("Scanned = %d, want 0", res.Scanned)
+ }
+}
+
+// A `<host>.tmp` link from a crashed PointLive names a real staged tree and
+// pins it: reading every entry is cheaper than deciding which links are
+// current.
+func TestLivePinsIncludesTempLinks(t *testing.T) {
+ opt := t.TempDir()
+ stage(t, opt, OpsAccount, "v1", "edge", 10, time.Hour)
+ stage(t, opt, OpsAccount, "v2", "edge", 10, time.Hour)
+ if err := PointLive(opt, "edge.example.com", OpsAccount, "v1", "edge"); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(VersionPath(opt, OpsAccount, "v2", "edge"), LiveLink(opt, "edge.example.com")+".tmp"); err != nil {
+ t.Fatal(err)
+ }
+ pins, err := LivePins(opt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !pins["v1"] || !pins["v2"] {
+ t.Fatalf("LivePins = %v, want v1 and v2", pins)
+ }
+}
+
+// A dangling link still pins: the version it names is what the unit's
+// ExecStart resolves through, and the sweep must not widen the hole.
+func TestLivePinsDanglingLink(t *testing.T) {
+ opt := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(VersionPath(opt, OpsAccount, "gone", "edge"), LiveLink(opt, "edge.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ pins, err := LivePins(opt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !pins["gone"] {
+ t.Fatalf("LivePins = %v, want gone pinned", pins)
+ }
+}
+
+// A live dir this package cannot read is an error, not a narrowed pin set:
+// the caller must decline to delete rather than delete a live binary.
+func TestLivePinsErrorsOnUnreadableEntry(t *testing.T) {
+ opt := t.TempDir()
+ if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ // A regular file where a symlink belongs: Readlink returns EINVAL.
+ if err := os.WriteFile(LiveLink(opt, "stray"), []byte("not a link"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := LivePins(opt); err == nil {
+ t.Fatal("LivePins accepted a non-symlink entry; it must refuse")
+ }
+}
+
+func TestLivePinsNoLiveDir(t *testing.T) {
+ pins, err := LivePins(t.TempDir())
+ if err != nil {
+ t.Fatalf("a box with no live dir is not an error: %v", err)
+ }
+ if len(pins) != 0 {
+ t.Errorf("pins = %v, want empty", pins)
+ }
+}