amadan

carlos / platform Public

Merge pull request #349 from carlosframework/reliability-followups

Reliability follow-ups: #344 account tie-break + config-delete guard, spawn-seam tier pin, OOM polish
Paul Campbell pushed by paul@keymail.dev cf772129f9d71fcfa17ea6fdbe5b5307f29c2f43 parent d27e2f0parent aa1507e
12 files changed, +294 −48
  • internal/activator/backend_exec.go +12 −3
  • internal/activator/backend_exec_test.go +35 −0
  • internal/appconfig/syncexec.go +15 −1
  • internal/appconfig/syncexec_test.go +106 −39
  • internal/caps/render.go +4 −0
  • internal/caps/testdata/keymail.service.conf +1 −0
  • internal/cgroup/cgroup.go +23 −0
  • internal/domains/reconcile.go +17 −4
  • internal/domains/reconcile_test.go +48 −0
  • internal/host/agent.go +1 −1
  • internal/host/units/carlos-app@.service +3 −0
  • internal/release/qualified.go +29 −0
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 580e63fe9..ddd0893b5 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -130,6 +130,15 @@ func (b *ExecBackend) tier(rt registry.Route) cgroup.Limit {
return b.Tier(rt)
}
+// spawnInCgroup is the ONE seam all three child shapes — seed, instance,
+// sidecar — start through: the route's cgroup, the route's tier. A spawn
+// site that bypassed it would compile and silently ship uncapped children;
+// keeping the tier plumbing here makes that a diff a reviewer can see and
+// TestSpawnInCgroupAppliesTheTier can pin.
+func (b *ExecBackend) spawnInCgroup(rt registry.Route, build func() *exec.Cmd) (*exec.Cmd, error) {
+ return b.Cgroups.Start(rt.Host, b.tier(rt), build)
+}
+
func (b *ExecBackend) liveDir() string { return filepath.Join(b.OptDir, "live") }
func (b *ExecBackend) liveLink(host string) string { return filepath.Join(b.liveDir(), host) }
func (b *ExecBackend) defaultDBPath(host string) string {
@@ -325,7 +334,7 @@ func (b *ExecBackend) Seed(ctx context.Context, rt registry.Route) error {
// cost is that instance's work. No CleanupHost here — the instance spawn
// reuses the dir moments later, and its exit path (or the next agent
// start's sweep) removes it.
- cmd, err := b.Cgroups.Start(host, b.tier(rt), func() *exec.Cmd {
+ cmd, err := b.spawnInCgroup(rt, func() *exec.Cmd {
cmd := exec.CommandContext(ctx, b.liveLink(host), argv...)
// The control-channel vars are agent-owned in BOTH directions here too
// (Start's discipline, see withoutVars), and here they are always
@@ -633,7 +642,7 @@ func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, strin
// carlos-exec@<host> and the sampler can account it. build may run
// twice — the manager's clone3-refusal fallback — so everything about
// the Cmd is configured inside it.
- cmd, err := b.Cgroups.Start(host, b.tier(rt), func() *exec.Cmd {
+ cmd, err := b.spawnInCgroup(rt, func() *exec.Cmd {
cmd := exec.Command(bin, "--socket", rt.Socket, "--db", dbPath)
cmd.Env = childEnv(childVars)
// The child logs through the agent's own stderr/stdout (journald
@@ -836,7 +845,7 @@ func (b *ExecBackend) StartSidecar(_ context.Context, rt registry.Route) (Proc,
// Same host cgroup as the instance: a sidecar's load is that instance's
// work, and attributing it elsewhere would be a lie in the other
// direction.
- cmd, err := b.Cgroups.Start(rt.Host, b.tier(rt), func() *exec.Cmd {
+ cmd, err := b.spawnInCgroup(rt, func() *exec.Cmd {
cmd := exec.Command(b.liveLink(rt.Host), "sidecar", "run")
cmd.Env = childEnv(fileEnv)
// Like the instance child: logs through the agent's own streams
diff --git a/internal/activator/backend_exec_test.go b/internal/activator/backend_exec_test.go
index b1f165250..6bc46a846 100644
--- a/internal/activator/backend_exec_test.go
+++ b/internal/activator/backend_exec_test.go
@@ -1,6 +1,9 @@
package activator
import (
+ "os"
+ "os/exec"
+ "path/filepath"
"testing"
"github.com/carlosframework/platform/internal/cgroup"
@@ -21,3 +24,35 @@ func TestExecBackendTierSeam(t *testing.T) {
t.Fatalf("set Tier: %v, want TierChildren", got)
}
}
+
+// TestSpawnInCgroupAppliesTheTier: the seam every spawn site uses really
+// writes the route's tier into the host cgroup — a regression that swapped
+// a site back to a bare Cgroups.Start(…, cgroup.Limit{}, …) compiles fine
+// and this is what catches the class. Real cgroup.SetupAt against a
+// temp-dir fixture; the spawn is /bin/true.
+func TestSpawnInCgroupAppliesTheTier(t *testing.T) {
+ root := t.TempDir()
+ for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} {
+ if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ m, err := cgroup.SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := ExecBackend{Cgroups: m, Tier: func(registry.Route) cgroup.Limit { return cgroup.TierChildren }}
+ rt := registry.Route{Host: "h.example.com"}
+ cmd, err := b.spawnInCgroup(rt, func() *exec.Cmd { return exec.Command("/bin/true") })
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = cmd.Wait()
+ got, err := os.ReadFile(filepath.Join(root, "carlos-exec@h.example.com", "memory.max"))
+ if err != nil {
+ t.Fatalf("memory.max not written: %v", err)
+ }
+ if string(got) != "1073741824" {
+ t.Errorf("memory.max = %q, want the children tier", got)
+ }
+}
diff --git a/internal/appconfig/syncexec.go b/internal/appconfig/syncexec.go
index 1c897955b..2987f1f5e 100644
--- a/internal/appconfig/syncexec.go
+++ b/internal/appconfig/syncexec.go
@@ -61,7 +61,7 @@ const configEnvSuffix = ".config.env"
// (pipelines design §4a): two routes of ONE app bound to different
// environments resolve to different manifests, and keying on (acct, app)
// alone would render whichever route was seen first into both files.
-func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv *ecdh.PrivateKey, dataDir string, bounce func(host string) error) error {
+func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv *ecdh.PrivateKey, dataDir string, bounce func(host string) error, appsDomain string) error {
routes, err := reg.List()
if err != nil {
return err
@@ -369,6 +369,20 @@ func SyncExec(ctx context.Context, reg *registry.Registry, s release.Store, priv
if r.m == nil && skipRemovals {
continue
}
+ // The #344 guard: an absent manifest removes the config file ONLY
+ // when the account it was looked up under agrees with the account
+ // the host itself names. A platform host's <sqid> label is not a
+ // moving part; the row's account column is (the domains reconcile
+ // stamps it from bucket listings, and a mis-stamp deleted a live
+ // tenant's delivered secrets on 2026-08-28). Held, loudly, like an
+ // unresolved group — the fix is the row's account, not this file.
+ if r.m == nil {
+ if sq := release.HostAccountLabel(rt.Host, appsDomain); sq != "" && sq != k.acct {
+ fmt.Printf(" %s: no config manifest under account %q, but the host names account %q — config file left as-is; fix the route row's account column (#344)\n",
+ rt.Host, k.acct, sq)
+ continue
+ }
+ }
// Namespace-collision guard, checked before EITHER shape can reach
// syncExecHost below (the single call site both shapes share): if a
// route "<rt.Host>.config" also lives on this box, its hand-staged
diff --git a/internal/appconfig/syncexec_test.go b/internal/appconfig/syncexec_test.go
index aab960ce0..372e83818 100644
--- a/internal/appconfig/syncexec_test.go
+++ b/internal/appconfig/syncexec_test.go
@@ -94,7 +94,7 @@ func TestSyncExecWritesConfigFileAndBouncesOnChange(t *testing.T) {
}
var bounced []string
bounce := func(h string) error { bounced = append(bounced, h); return nil }
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(filepath.Join(dd, "inst.example.com.config.env"))
@@ -112,7 +112,7 @@ func TestSyncExecWritesConfigFileAndBouncesOnChange(t *testing.T) {
}
// Second pass, nothing changed: no bounce.
bounced = nil
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec (second): %v", err)
}
if len(bounced) != 0 {
@@ -140,7 +140,7 @@ func TestSyncExecRemovesFileWhenManifestDeleted(t *testing.T) {
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
@@ -185,7 +185,7 @@ func TestSyncExecMassRemovalGuardRefusesWhenEveryGroupResolvesAbsent(t *testing.
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if _, err := os.Stat(pathA); err != nil {
@@ -230,7 +230,7 @@ func TestSyncExecMassRemovalGuardDoesNotBlockNormalConvergence(t *testing.T) {
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if _, err := os.Stat(pathA); err != nil {
@@ -282,7 +282,7 @@ func TestSyncExecRendersEachRoutesBoundEnvironment(t *testing.T) {
t.Fatalf("Upsert %s: %v", rt.Host, err)
}
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
want := map[string]string{
@@ -329,7 +329,7 @@ func TestSyncExecLeavesFileUntouchedWhenBoundEnvironmentIsMissing(t *testing.T)
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(path)
@@ -392,7 +392,7 @@ func TestSyncExecOverlayOnlyAppDropsStaleKeys(t *testing.T) {
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(path)
@@ -440,7 +440,7 @@ func TestSyncExecMassRemovalGuardIgnoresErroredGroups(t *testing.T) {
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} {
@@ -476,7 +476,7 @@ func TestSyncExecNeverTouchesInstanceEnvOrOutOfScopeRoutes(t *testing.T) {
t.Fatalf("Upsert %s: %v", rt.Host, err)
}
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if got, _ := os.ReadFile(own); string(got) != "CARLOS_ADMIN_TOKEN=tok\n" {
@@ -516,7 +516,7 @@ func TestSyncExecReapsOnlyItsOwnNamespace(t *testing.T) {
t.Fatalf("stage %s: %v", name, err)
}
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
for name, keep := range files {
@@ -564,7 +564,7 @@ func TestSyncExecReapsOnlyItsOwnNamespaceWithAddrShapeInTheMix(t *testing.T) {
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
for name, keep := range files {
@@ -592,7 +592,7 @@ func TestSyncExecEmptyRegistryRemovesNothing(t *testing.T) {
if err := os.WriteFile(path, []byte("X=1\n"), 0o600); err != nil {
t.Fatalf("stage: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if _, err := os.Stat(path); err != nil {
@@ -640,7 +640,7 @@ func TestSyncExecOptedInAddrRouteMaterializesFile(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
@@ -665,7 +665,7 @@ func TestSyncExecOptedInAddrRouteMaterializesFile(t *testing.T) {
t.Fatal(err)
}
time.Sleep(10 * time.Millisecond) // ensure a rewrite would move mtime
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec (second pass): %v", err)
}
after, err := os.Stat(path)
@@ -697,7 +697,7 @@ func TestSyncExecUnoptedAddrRouteIsSkipped(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if _, err := os.Stat(filepath.Join(dd, "demo.go.tito.io.config.env")); !os.IsNotExist(err) {
@@ -741,7 +741,7 @@ func TestSyncExecUnrecognizedExecDeliveryValueIgnoredNotFatal(t *testing.T) {
}
var syncErr error
out := captureStdout(t, func() {
- syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil })
+ syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "")
})
if syncErr != nil {
t.Fatalf("SyncExec: %v", syncErr)
@@ -790,7 +790,7 @@ func TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs(t *testing.T)
}
// First pass materializes normally — this is the file a transient
// blip must leave exactly as it is.
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec (initial): %v", err)
}
path := filepath.Join(dd, "demo.go.tito.io.config.env")
@@ -805,7 +805,7 @@ func TestSyncExecOptedInAddrRouteFetchFailureLogsAndLeavesFileAsIs(t *testing.T)
bounce := func(h string) error { bounced = append(bounced, h); return nil }
var syncErr error
out := captureStdout(t, func() {
- syncErr = SyncExec(ctx, reg, failing, pk, dd, bounce)
+ syncErr = SyncExec(ctx, reg, failing, pk, dd, bounce, "")
})
if syncErr != nil {
t.Fatalf("SyncExec (blip): %v", syncErr)
@@ -873,7 +873,7 @@ func TestSyncExecOptedInAddrRouteIsNeverBounced(t *testing.T) {
bounced = append(bounced, h)
return nil
}
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec (initial write): %v", err)
}
if len(bounced) != 0 {
@@ -884,7 +884,7 @@ func TestSyncExecOptedInAddrRouteIsNeverBounced(t *testing.T) {
if err := SetEnv(ctx, s, "bab", "titogo", map[string]string{"TITOGO_DEMO_BANNER": "2"}); err != nil {
t.Fatalf("SetEnv (change): %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec (changed content): %v", err)
}
if len(bounced) != 0 {
@@ -933,7 +933,7 @@ func TestSyncExecSocketRouteStillBouncesAlongsideOptedInAddrRoute(t *testing.T)
bounced = append(bounced, h)
return nil
}
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
if !slices.Contains(bounced, "inst.example.com") {
@@ -979,7 +979,7 @@ func TestSyncExecOptedInAddrRouteHonorsBoundEnvironment(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
@@ -1018,7 +1018,7 @@ func TestSyncExecOptedInAddrAppWithNoVarsConvergesToEmptyFile(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
@@ -1073,7 +1073,7 @@ func TestSyncExecMassRemovalGuardUnaffectedByOptedInAddrAppWithContent(t *testin
if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
bounced = append(bounced, h)
return nil
- }); err != nil {
+ }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
@@ -1128,7 +1128,7 @@ func TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec (opted in): %v", err)
}
path := filepath.Join(dd, "demo.go.tito.io.config.env")
@@ -1144,7 +1144,7 @@ func TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass(t *testing.T) {
bounce := func(h string) error { bounced = append(bounced, h); return nil }
var syncErr error
out := captureStdout(t, func() {
- syncErr = SyncExec(ctx, reg, s, pk, dd, bounce)
+ syncErr = SyncExec(ctx, reg, s, pk, dd, bounce, "")
})
if syncErr != nil {
t.Fatalf("SyncExec (opted out): %v", syncErr)
@@ -1170,7 +1170,7 @@ func TestSyncExecOptOutLeavesStaleFileButWarnsEveryPass(t *testing.T) {
// changed must repeat it, since nothing about the risk (stale
// plaintext secrets, forever) has gone away.
out2 := captureStdout(t, func() {
- syncErr = SyncExec(ctx, reg, s, pk, dd, bounce)
+ syncErr = SyncExec(ctx, reg, s, pk, dd, bounce, "")
})
if syncErr != nil {
t.Fatalf("SyncExec (opted out, second pass): %v", syncErr)
@@ -1205,7 +1205,7 @@ func TestSyncExecNoOptOutWarningWhenNeverOptedIn(t *testing.T) {
}
var syncErr error
out := captureStdout(t, func() {
- syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil })
+ syncErr = SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "")
})
if syncErr != nil {
t.Fatalf("SyncExec: %v", syncErr)
@@ -1245,7 +1245,7 @@ func TestSyncExecAddrFileCarriesHeaderSocketFileDoesNot(t *testing.T) {
t.Fatalf("Upsert %s: %v", rt.Host, err)
}
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
addrGot, err := os.ReadFile(filepath.Join(dd, "demo.go.tito.io.config.env"))
@@ -1299,7 +1299,7 @@ func TestSyncExecConvergesToEmptyFileWhenAllVarsUnset(t *testing.T) {
var bounced []string
bounce := func(h string) error { bounced = append(bounced, h); return nil }
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec (set): %v", err)
}
got, err := os.ReadFile(path)
@@ -1316,7 +1316,7 @@ func TestSyncExecConvergesToEmptyFileWhenAllVarsUnset(t *testing.T) {
t.Fatalf("Unset (all vars): %v", err)
}
bounced = nil
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec (unset): %v", err)
}
got, err = os.ReadFile(path)
@@ -1353,7 +1353,7 @@ func TestReapSparesLiveRoutesInstanceEnvWithConfigSuffix(t *testing.T) {
t.Fatalf("Upsert: %v", err)
}
out := captureStdout(t, func() {
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
})
@@ -1416,7 +1416,7 @@ func TestSyncExecRefusesConfigNamespaceCollision(t *testing.T) {
var bounced []string
bounce := func(h string) error { bounced = append(bounced, h); return nil }
out = captureStdout(t, func() {
- if err := SyncExec(ctx, reg, s, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
})
@@ -1487,7 +1487,7 @@ func TestSyncExecFetchFailureLeavesOtherGroupsConverging(t *testing.T) {
failing := &failingKeyStore{Store: s, failKey: Key("bab", "appa"), err: blipErr}
var bounced []string
bounce := func(h string) error { bounced = append(bounced, h); return nil }
- if err := SyncExec(ctx, reg, failing, pk, dd, bounce); err != nil {
+ if err := SyncExec(ctx, reg, failing, pk, dd, bounce, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(stalePath)
@@ -1708,7 +1708,7 @@ func TestSyncExecOptInResolvesPerEnvironment(t *testing.T) {
t.Fatalf("Upsert %s: %v", rt.Host, err)
}
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
assertConfigFile(t, dd, "connect.go.tito.io", tc.wantRelayFile)
@@ -1757,7 +1757,7 @@ func TestSyncExecEnvScopedOptInDeliversTheMergedBundle(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
got, err := os.ReadFile(filepath.Join(dd, "connect.go.tito.io"+configEnvSuffix))
@@ -1802,7 +1802,7 @@ func TestSyncExecEnvBundleFetchErrorLeavesFileAlone(t *testing.T) {
}); err != nil {
t.Fatalf("Upsert: %v", err)
}
- if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec: %v", err)
}
path := filepath.Join(dd, "connect.go.tito.io"+configEnvSuffix)
@@ -1814,7 +1814,7 @@ func TestSyncExecEnvBundleFetchErrorLeavesFileAlone(t *testing.T) {
blip := &failingKeyStore{Store: s, failKey: EnvironmentKey("bab", "titogo", "relay"),
err: errors.New("503 SlowDown")}
out := captureStdout(t, func() {
- if err := SyncExec(ctx, reg, blip, pk, dd, func(string) error { return nil }); err != nil {
+ if err := SyncExec(ctx, reg, blip, pk, dd, func(string) error { return nil }, ""); err != nil {
t.Fatalf("SyncExec (blip): %v", err)
}
})
@@ -1829,3 +1829,70 @@ func TestSyncExecEnvBundleFetchErrorLeavesFileAlone(t *testing.T) {
t.Fatalf("a fetch blip against the bound bundle must say so loudly and name the environment; got %q", out)
}
}
+
+// The #344 guard, from the 2026-08-28 incident: the domains reconcile
+// mis-stamped a route's account, the config sync keyed the manifest under
+// the stamped account, found nothing, and DELETED the tenant's delivered
+// config — secrets included — bouncing the instance into a crash loop. An
+// absent manifest may remove the file only when the lookup account agrees
+// with the account the host itself names; a platform host's <sqid> label
+// is not a moving part, the row's account column is.
+func TestSyncExecHoldsConfigWhenHostNamesADifferentAccount(t *testing.T) {
+ ctx, s, reg, dd, priv, _ := execKit(t)
+ pk, _ := ParsePrivateKey(priv, "test")
+ if err := reg.Upsert(registry.Route{
+ Host: "birthday-alarm.bde.oncarlos.test", Kind: "instance", App: "birthday-alarm",
+ Account: "bab", // the mis-stamp
+ Status: "ready", Socket: "/run/carlos/birthday.sock",
+ }); err != nil {
+ t.Fatalf("Upsert: %v", err)
+ }
+ // A second, healthy group so the mass-removal guard (which needs EVERY
+ // group absent) stays out of the way — this guard is per-route.
+ if err := reg.Upsert(registry.Route{
+ Host: "inst.example.com", Kind: "instance", App: "eleven", Account: "bab",
+ Status: "ready", Socket: "/run/carlos/inst.sock",
+ }); err != nil {
+ t.Fatalf("Upsert: %v", err)
+ }
+ if err := SetEnv(ctx, s, "bab", "eleven", map[string]string{"A": "1"}); err != nil {
+ t.Fatalf("SetEnv: %v", err)
+ }
+ path := filepath.Join(dd, "birthday-alarm.bde.oncarlos.test.config.env")
+ if err := os.WriteFile(path, []byte("BIRTHDAYALARM_INSTANCE_KEY=deadbeef\n"), 0o600); err != nil {
+ t.Fatalf("stage: %v", err)
+ }
+ var bounced []string
+ if err := SyncExec(ctx, reg, s, pk, dd, func(h string) error {
+ bounced = append(bounced, h)
+ return nil
+ }, "oncarlos.test"); err != nil {
+ t.Fatalf("SyncExec: %v", err)
+ }
+ if _, err := os.Stat(path); err != nil {
+ t.Fatalf("delivered config removed under a mis-stamped account: %v", err)
+ }
+ for _, h := range bounced {
+ if h == "birthday-alarm.bde.oncarlos.test" {
+ t.Fatalf("held route was bounced: %v", bounced)
+ }
+ }
+ // And the guard steps aside when the row and the host agree: the same
+ // absent manifest under the RIGHT account is a real deletion.
+ if err := reg.Upsert(registry.Route{
+ Host: "birthday-alarm.bab.oncarlos.test", Kind: "instance", App: "birthday-alarm",
+ Account: "bab", Status: "ready", Socket: "/run/carlos/birthday2.sock",
+ }); err != nil {
+ t.Fatalf("Upsert: %v", err)
+ }
+ agreed := filepath.Join(dd, "birthday-alarm.bab.oncarlos.test.config.env")
+ if err := os.WriteFile(agreed, []byte("OLD=1\n"), 0o600); err != nil {
+ t.Fatalf("stage: %v", err)
+ }
+ if err := SyncExec(ctx, reg, s, pk, dd, func(string) error { return nil }, "oncarlos.test"); err != nil {
+ t.Fatalf("SyncExec: %v", err)
+ }
+ if _, err := os.Stat(agreed); !os.IsNotExist(err) {
+ t.Fatalf("agreeing host's stale file survived, stat err = %v", err)
+ }
+}
diff --git a/internal/caps/render.go b/internal/caps/render.go
index 2cdadc905..c4c2f01ae 100644
--- a/internal/caps/render.go
+++ b/internal/caps/render.go
@@ -91,6 +91,10 @@ func RenderService(app string, s *Set, g Grants) ([]byte, error) {
"KillMode=control-group",
"TasksMax=1024",
"MemoryMax=1G",
+ // Throttle-and-reclaim before the kill, like the exec path's
+ // memory.high (internal/cgroup applyLimit): the same declared
+ // capability behaves the same on both backings.
+ "MemoryHigh=920M",
)
}
diff --git a/internal/caps/testdata/keymail.service.conf b/internal/caps/testdata/keymail.service.conf
index 766f71f78..facb583d8 100644
--- a/internal/caps/testdata/keymail.service.conf
+++ b/internal/caps/testdata/keymail.service.conf
@@ -8,6 +8,7 @@ RuntimeDirectoryPreserve=restart
KillMode=control-group
TasksMax=1024
MemoryMax=1G
+MemoryHigh=920M
DynamicUser=no
RestrictSUIDSGID=yes
User=carlos-keymail
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go
index b6a38a8dc..94ccaff70 100644
--- a/internal/cgroup/cgroup.go
+++ b/internal/cgroup/cgroup.go
@@ -360,10 +360,33 @@ func (m *Manager) biasOOM(host string, cmd *exec.Cmd) {
return
}
if err := os.WriteFile(oomScoreAdjPath(cmd.Process.Pid), []byte(ExecOOMScoreAdj), 0o644); err != nil {
+ // A child that crashed in its first milliseconds is a zombie by the
+ // time this write lands, and a zombie's oom_score_adj refuses the
+ // write (seen live: birthday-alarm's crash loop logged one of these
+ // per wake). A dead child needs no bias — say nothing.
+ if procGone(cmd.Process.Pid) {
+ return
+ }
m.logf("cgroup: %s: oom_score_adj for pid %d: %v — the agent and this child score alike at an edge-tree OOM", host, cmd.Process.Pid, err)
}
}
+// procGone reports whether pid is dead — absent from /proc, or a zombie
+// (state Z in /proc/<pid>/stat, third field after the parenthesised comm).
+func procGone(pid int) bool {
+ b, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat"))
+ if err != nil {
+ return true
+ }
+ // comm may contain spaces and parens; the state letter follows the LAST ')'.
+ s := string(b)
+ i := strings.LastIndexByte(s, ')')
+ if i < 0 || i+2 >= len(s) {
+ return false
+ }
+ return s[i+2] == 'Z'
+}
+
// CleanupHost removes host's per-host cgroup dir after its children exit.
// Nil-safe and rmdir-tolerant: while a sibling (instance vs sidecar) is
// still alive the dir is non-empty and the remove fails — the LAST exiter's
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go
index 418ecb905..2cb29f138 100644
--- a/internal/domains/reconcile.go
+++ b/internal/domains/reconcile.go
@@ -186,7 +186,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
if routes[i].App == "" {
continue // an app-less route (status page, bare proxy) has no account
}
- acct := resolveRouteAccount(routes[i], owners[routes[i].App])
+ acct := resolveRouteAccount(routes[i], owners[routes[i].App], cfg.AppsDomain)
if acct == "" || routes[i].Account == acct {
continue
}
@@ -712,19 +712,32 @@ type appKey struct{ acct, app string }
// - several do, and the row names none of them: leave it alone. Picking one
// would re-home somebody's app onto another tenant's bytes, which is far
// worse than a row that reconciliation declines to touch.
-func resolveRouteAccount(rt registry.Route, owners []string) string {
+func resolveRouteAccount(rt registry.Route, owners []string, appsDomain string) string {
if len(owners) == 0 {
if rt.Account == "" {
return OpsSqid // nothing shipped yet: ops, like a console-less box
}
return "" // the row says something; the bucket says nothing. Believe the row.
}
- if slices.Contains(owners, release.RouteAccount(rt)) {
- return release.RouteAccount(rt)
+ // A row that names one of the owners is right; keep it.
+ if release.ValidAccount(rt.Account) && slices.Contains(owners, rt.Account) {
+ return rt.Account
+ }
+ // The row names nothing (or something the bucket doesn't hold). Before
+ // any fallback: a platform host literally names its owner in its own
+ // labels — <name>.<sqid>.<apps domain>. #344 was the ops fallback
+ // winning this tie (a stray pipeline.json under ops vs the tenant's
+ // whole app), and the config sync then deleting the tenant's delivered
+ // config for the mis-stamped account.
+ if sq := release.HostAccountLabel(rt.Host, appsDomain); sq != "" && slices.Contains(owners, sq) {
+ return sq
}
if len(owners) == 1 {
return owners[0]
}
+ // Several owners and nothing that decides between them. NOT ops-if-
+ // present: an unstamped row already reads as ops at runtime, so
+ // stamping buys nothing — and a wrong stamp is what #344 cost.
log.Printf("domains: %s (app %q) names account %q, but objects exist under %v — leaving it alone",
rt.Host, rt.App, rt.Account, owners)
return ""
diff --git a/internal/domains/reconcile_test.go b/internal/domains/reconcile_test.go
index 8e94b7167..7629174cf 100644
--- a/internal/domains/reconcile_test.go
+++ b/internal/domains/reconcile_test.go
@@ -1386,3 +1386,51 @@ func TestStampHealthIsIdempotent(t *testing.T) {
t.Error("a pass with no AppsDomain wrote a stamp")
}
}
+
+// The #344 shape, live on the flagship 2026-08-28: a route with an EMPTY
+// account column, for an app whose name holds objects under ops (a stray
+// pipeline.json from an abandoned attempt) AND under the tenant that
+// actually ships it. The ops fallback used to win the tie; the host's own
+// sqid label — birthday-alarm.<sqid>.oncarlos.com literally names its
+// owner — must win instead. The mis-stamp deleted the tenant's delivered
+// config (syncexec keyed apps/bab/...) and the app died on every wake.
+func TestReconcileStampsTheHostsOwnAccountNotOpsOnATie(t *testing.T) {
+ sqidNum := int64(793481920157)
+ s, reg := dirStore(t), regWith(t)
+ sqid := writeAccountManifest(t, s, sqidNum, "Paulca")
+ if err := reg.Upsert(registry.Route{Host: "birthday-alarm." + sqid + ".oncarlos.test",
+ Kind: "instance", App: "birthday-alarm", Status: "ready",
+ Socket: "/run/carlos/birthday-alarm.sock"}); err != nil {
+ t.Fatal(err)
+ }
+ putKey(t, s, "apps/bab/birthday-alarm/pipeline.json") // the stray
+ putKey(t, s, "apps/"+sqid+"/birthday-alarm/config.json")
+ if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
+ t.Fatal(err)
+ }
+ if rt := mustGet(t, reg, "birthday-alarm."+sqid+".oncarlos.test"); rt.Account != sqid {
+ t.Errorf("account = %q, want %s — the ops fallback won a tie the host itself decides", rt.Account, sqid)
+ }
+}
+
+// The same tie on a host that names NO account (a custom domain): the pass
+// must leave the row alone rather than defaulting it onto ops — an
+// unstamped row already reads as ops at runtime, so stamping buys nothing
+// and a wrong stamp deletes config (#344).
+func TestReconcileLeavesAnAmbiguousCustomDomainUnstamped(t *testing.T) {
+ s, reg := dirStore(t), regWith(t)
+ sqid := writeAccountManifest(t, s, 793481920157, "Paulca")
+ if err := reg.Upsert(registry.Route{Host: "birthdays.example.com",
+ Kind: "instance", App: "birthday-alarm", Status: "ready",
+ Socket: "/run/carlos/birthdays.sock"}); err != nil {
+ t.Fatal(err)
+ }
+ putKey(t, s, "apps/bab/birthday-alarm/pipeline.json")
+ putKey(t, s, "apps/"+sqid+"/birthday-alarm/config.json")
+ if err := domains.Reconcile(ctx, s, reg, cfg(t)); err != nil {
+ t.Fatal(err)
+ }
+ if rt := mustGet(t, reg, "birthdays.example.com"); rt.Account != "" {
+ t.Errorf("account = %q, want unstamped — ambiguity is not ops's to win", rt.Account)
+ }
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 294f0ff27..bd8f84dbb 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -652,7 +652,7 @@ var configSyncMu sync.Mutex
func syncExecSerialized(ctx context.Context, reg *registry.Registry, store release.Store, priv *ecdh.PrivateKey, dd string, svc *activator.Service) error {
configSyncMu.Lock()
defer configSyncMu.Unlock()
- return appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc))
+ return appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc), os.Getenv("PLATFORM_APPS_DOMAIN"))
}
// reconcileMu serializes the two provisioning loops' Reconcile passes
diff --git a/internal/host/units/carlos-app@.service b/internal/host/units/carlos-app@.service
index 2fc9dde79..613eea27b 100644
--- a/internal/host/units/carlos-app@.service
+++ b/internal/host/units/carlos-app@.service
@@ -57,6 +57,9 @@ CapabilityBoundingSet=
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
MemoryDenyWriteExecute=yes
MemoryMax=512M
+# Throttle-and-reclaim before the kill — the same band the exec path's
+# memory.high gives (internal/cgroup applyLimit, 90% of max).
+MemoryHigh=460M
TasksMax=256
[Install]
diff --git a/internal/release/qualified.go b/internal/release/qualified.go
index 9ab574b10..b3fb46b00 100644
--- a/internal/release/qualified.go
+++ b/internal/release/qualified.go
@@ -182,3 +182,32 @@ func GetMigrationSource(ctx context.Context, s Store, key, legacy string) (body
}
return body, etag, true, nil
}
+
+// HostAccountLabel extracts the account sqid a PLATFORM host names in its
+// own labels: <name>.<sqid>.<apps domain> (and the canary shape
+// <canary>.<app>.<sqid>.<apps domain>) both put the sqid immediately
+// before the apps domain. "" when the host is not under the apps domain,
+// has no label pair before it, or the label isn't a valid account name —
+// callers treat "" as "the host names nobody", never as an error. This is
+// the one detail #344 turned on: a route row's account column is a moving
+// part (the domains reconcile stamps it from bucket listings), but a
+// platform host's own name is not.
+func HostAccountLabel(host, appsDomain string) string {
+ appsDomain = strings.TrimSuffix(strings.TrimSpace(appsDomain), ".")
+ if appsDomain == "" {
+ return ""
+ }
+ rest, ok := strings.CutSuffix(strings.ToLower(host), "."+strings.ToLower(appsDomain))
+ if !ok {
+ return ""
+ }
+ labels := strings.Split(rest, ".")
+ if len(labels) < 2 {
+ return ""
+ }
+ sq := labels[len(labels)-1]
+ if !ValidAccount(sq) {
+ return ""
+ }
+ return sq
+}