amadan
Merge pull request #336 from carlosframework/reliability
Memory pressure containment: edge wall, per-app caps, memwatch, module alarm
16 files changed,
+806
−17
.gitignore+1 −0infra/modules/README.md+1 −0infra/modules/region-host/main.tf+66 −0infra/modules/region-host/variables.tf+6 −0internal/activator/backend_exec.go+17 −3internal/activator/backend_exec_test.go+23 −0internal/cgroup/cgroup.go+100 −3internal/cgroup/cgroup_linux_test.go+7 −7internal/cgroup/cgroup_test.go+120 −2internal/host/agent.go+7 −0internal/host/host_test.go+8 −0internal/host/memtier.go+26 −0internal/host/memtier_test.go+38 −0internal/host/memwatch.go+208 −0internal/host/memwatch_test.go+155 −0internal/host/units/carlos-edge.service+23 −2
diff --git a/.gitignore b/.gitignore| index 6b209b585..42d71f76e 100644 |
| --- a/.gitignore |
| +++ b/.gitignore |
| @@ -8,3 +8,4 @@ platform |
| # amadan reuses one workspace per repo across jobs, so an untracked |
| # directory in the checkout is that step's cache — see the step for why. |
| .amadan-cache/ |
| +infra/modules/**/.terraform.lock.hcl |
diff --git a/infra/modules/README.md b/infra/modules/README.md| index f0c68eea7..1765debb2 100644 |
| --- a/infra/modules/README.md |
| +++ b/infra/modules/README.md |
| @@ -70,6 +70,7 @@ bucket, so a wrong-region wake fails loudly at the IAM layer. |
| | `steering_record_names` | Name bound on that grant, e.g. `["*.oncarlos.com"]`. Empty (default) grants A-record UPSERT/DELETE on the WHOLE zone, apex included — set it. | |
| | `ami_name_pattern` | AMI name filter (default `debian-13-arm64-*`). | |
| | `ami_owner` | AMI owner account (default `136693071363`, Debian). | |
| +| `alerts_email` | Email for the host's `MemAvailablePercent < 15% for 3 min` alarm (memory-pressure spec §2.4). The module creates a same-region SNS topic + subscription per host (an alarm cannot notify a topic in another region). Empty (default): no topic, no alarm. Set it only once the box is publishing — missing data is treated as breaching. The `cloudwatch:PutMetricData` grant on namespace `Carlos/Host` is unconditional, so the agent's memwatch duty publishes either way. | |
| | `enable_auto_recovery` | Arm the EC2 auto-recovery alarm (HA ladder rung 0) — a system-status-check failure recovers the instance onto healthy hardware, same instance id, EIP and volumes. Default `true`; set `false` only where an external supervisor owns recovery. | |
| Outputs: `eip_public_ip` (point your DNS here), `instance_id`, |
diff --git a/infra/modules/region-host/main.tf b/infra/modules/region-host/main.tf| index 407565c29..ae5afb52f 100644 |
| --- a/infra/modules/region-host/main.tf |
| +++ b/infra/modules/region-host/main.tf |
| @@ -288,6 +288,28 @@ resource "aws_iam_role_policy" "host_costexplorer" { |
| }) |
| } |
| +# The agent's memwatch duty (internal/host/memwatch.go) publishes |
| +# MemAvailablePercent and MemoryPressureSome60 once a minute. PutMetricData |
| +# has no resource ARN; the namespace condition is the whole bound — this |
| +# role can write Carlos/Host and nothing else. |
| +resource "aws_iam_role_policy" "host_metrics" { |
| + name = "${var.iam_name}-metrics" |
| + role = aws_iam_role.host.id |
| + policy = jsonencode({ |
| + Version = "2012-10-17" |
| + Statement = [ |
| + { |
| + Effect = "Allow" |
| + Action = ["cloudwatch:PutMetricData"] |
| + Resource = "*" |
| + Condition = { |
| + StringEquals = { "cloudwatch:namespace" = "Carlos/Host" } |
| + } |
| + }, |
| + ] |
| + }) |
| +} |
| + |
| resource "aws_iam_role_policy_attachment" "host_ssm" { |
| role = aws_iam_role.host.name |
| policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" |
| @@ -347,3 +369,47 @@ resource "aws_cloudwatch_metric_alarm" "host_auto_recover" { |
| } |
| tags = { Name = "${var.name}-auto-recover" } |
| } |
| + |
| +# Memory-pressure spec §2.4: the early warning the status checks can't give. |
| +# On 2026-08-28 the flagship thrashed for 15 minutes with EC2 status ok/ok; |
| +# MemAvailable would have crossed 15% minutes earlier. Missing data is |
| +# BREACHING: a box that stops reporting is a box in trouble (or an agent |
| +# rolled without CARLOS_HOST_METRICS — either way, look). Which is also |
| +# why the caller should set alerts_email only once the box is publishing |
| +# (spec §4): wiring it first buys days of ALARM mail about nothing. |
| +# |
| +# The topic lives here, in the host's region, because an alarm can only |
| +# notify a same-region topic — one account-wide topic passed to four |
| +# regional hosts would apply and then never deliver for three of them. |
| +resource "aws_sns_topic" "alerts" { |
| + count = var.alerts_email != "" ? 1 : 0 |
| + name = "${var.name}-alerts" |
| +} |
| + |
| +resource "aws_sns_topic_subscription" "alerts_email" { |
| + count = var.alerts_email != "" ? 1 : 0 |
| + topic_arn = aws_sns_topic.alerts[0].arn |
| + protocol = "email" |
| + endpoint = var.alerts_email |
| +} |
| + |
| +resource "aws_cloudwatch_metric_alarm" "host_memory_low" { |
| + count = var.alerts_email != "" ? 1 : 0 |
| + alarm_name = "${var.name}-memory-low" |
| + alarm_description = "MemAvailable below 15% for 3 minutes on ${var.name} — a tenant is eating the box; see the memory-pressure spec §4 for what to look at." |
| + namespace = "Carlos/Host" |
| + metric_name = "MemAvailablePercent" |
| + statistic = "Minimum" |
| + period = 60 |
| + evaluation_periods = 3 |
| + datapoints_to_alarm = 3 |
| + threshold = 15 |
| + comparison_operator = "LessThanThreshold" |
| + treat_missing_data = "breaching" |
| + alarm_actions = [aws_sns_topic.alerts[0].arn] |
| + ok_actions = [aws_sns_topic.alerts[0].arn] |
| + dimensions = { |
| + InstanceId = aws_instance.host.id |
| + } |
| + tags = { Name = "${var.name}-memory-low" } |
| +} |
diff --git a/infra/modules/region-host/variables.tf b/infra/modules/region-host/variables.tf| index e46144582..3f0a898c0 100644 |
| --- a/infra/modules/region-host/variables.tf |
| +++ b/infra/modules/region-host/variables.tf |
| @@ -108,3 +108,9 @@ variable "enable_cost_explorer" { |
| type = bool |
| default = false |
| } |
| + |
| +variable "alerts_email" { |
| + description = "Email that receives this host's memory alarm (memory-pressure spec §2.4). A CloudWatch alarm can only notify an SNS topic in its own region, so the module creates a regional topic + subscription per host rather than taking a topic ARN. Empty: no topic, no alarm; the PutMetricData grant is unconditional so the agent's memwatch duty can publish either way." |
| + type = string |
| + default = "" |
| +} |
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go| index ab062ede4..580e63fe9 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -71,6 +71,12 @@ type ExecBackend struct { |
| // metrics sampler can account it (exec-child accounting spec). Nil — no |
| // delegation: old unit file, tests, darwin — spawns exactly as before. |
| Cgroups *cgroup.Manager |
| + // Tier, when non-nil, picks the memory cap for a route's carlos-exec@ |
| + // cgroup (memory-pressure spec §2.3). The box wiring reads the staged |
| + // manifest of the live version; tests leave it nil and get the default |
| + // tier. Instance, seed and sidecar share the cgroup, so they share the |
| + // tier — a seed's first-boot cost is that instance's work. |
| + Tier func(registry.Route) cgroup.Limit |
| // Logs, when non-nil, gets each instance and seed child's stdout/stderr |
| // teed in line-wise for `carlos logs` (spec §5) — journald keeps every |
| // byte it got before. Nil (tests, a box without the shipping loop) |
| @@ -116,6 +122,14 @@ func (b *ExecBackend) teeStd(cmd *exec.Cmd, rt registry.Route) { |
| var _ Backend = (*ExecBackend)(nil) |
| +// tier is Tier with the nil default applied. |
| +func (b *ExecBackend) tier(rt registry.Route) cgroup.Limit { |
| + if b.Tier == nil { |
| + return cgroup.TierDefault |
| + } |
| + return b.Tier(rt) |
| +} |
| + |
| func (b *ExecBackend) liveDir() string { return filepath.Join(b.OptDir, "live") } |
| func (b *ExecBackend) liveLink(host string) string { return filepath.Join(b.liveDir(), host) } |
| func (b *ExecBackend) defaultDBPath(host string) string { |
| @@ -311,7 +325,7 @@ func (b *ExecBackend) Seed(ctx context.Context, rt registry.Route) error { |
| // cost is that instance's work. No CleanupHost here — the instance spawn |
| // reuses the dir moments later, and its exit path (or the next agent |
| // start's sweep) removes it. |
| - cmd, err := b.Cgroups.Start(host, func() *exec.Cmd { |
| + cmd, err := b.Cgroups.Start(host, b.tier(rt), func() *exec.Cmd { |
| cmd := exec.CommandContext(ctx, b.liveLink(host), argv...) |
| // The control-channel vars are agent-owned in BOTH directions here too |
| // (Start's discipline, see withoutVars), and here they are always |
| @@ -619,7 +633,7 @@ func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, strin |
| // carlos-exec@<host> and the sampler can account it. build may run |
| // twice — the manager's clone3-refusal fallback — so everything about |
| // the Cmd is configured inside it. |
| - cmd, err := b.Cgroups.Start(host, func() *exec.Cmd { |
| + cmd, err := b.Cgroups.Start(host, b.tier(rt), func() *exec.Cmd { |
| cmd := exec.Command(bin, "--socket", rt.Socket, "--db", dbPath) |
| cmd.Env = childEnv(childVars) |
| // The child logs through the agent's own stderr/stdout (journald |
| @@ -822,7 +836,7 @@ func (b *ExecBackend) StartSidecar(_ context.Context, rt registry.Route) (Proc, |
| // Same host cgroup as the instance: a sidecar's load is that instance's |
| // work, and attributing it elsewhere would be a lie in the other |
| // direction. |
| - cmd, err := b.Cgroups.Start(rt.Host, func() *exec.Cmd { |
| + cmd, err := b.Cgroups.Start(rt.Host, b.tier(rt), func() *exec.Cmd { |
| cmd := exec.Command(b.liveLink(rt.Host), "sidecar", "run") |
| cmd.Env = childEnv(fileEnv) |
| // Like the instance child: logs through the agent's own streams |
diff --git a/internal/activator/backend_exec_test.go b/internal/activator/backend_exec_test.go| new file mode 100644 |
| index 000000000..b1f165250 |
| --- /dev/null |
| +++ b/internal/activator/backend_exec_test.go |
| @@ -0,0 +1,23 @@ |
| +package activator |
| + |
| +import ( |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// TestExecBackendTierSeam: a nil Tier means the default tier, and a set |
| +// Tier is consulted with the route. The cgroup manager is nil here (no |
| +// delegation in a test), so this pins the seam's contract, not the write. |
| +func TestExecBackendTierSeam(t *testing.T) { |
| + var b ExecBackend |
| + rt := registry.Route{Host: "h.example.com"} |
| + if got := b.tier(rt); got != cgroup.TierDefault { |
| + t.Fatalf("nil Tier: %v, want TierDefault", got) |
| + } |
| + b.Tier = func(registry.Route) cgroup.Limit { return cgroup.TierChildren } |
| + if got := b.tier(rt); got != cgroup.TierChildren { |
| + t.Fatalf("set Tier: %v, want TierChildren", got) |
| + } |
| +} |
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go| index 9752a0cf0..58f5491df 100644 |
| --- a/internal/cgroup/cgroup.go |
| +++ b/internal/cgroup/cgroup.go |
| @@ -3,7 +3,9 @@ |
| // file), so the metrics sampler can report per-instance figures for routes |
| // that run as exec children of the agent — today those live inside the |
| // agent's own cgroup and can never show readings. Design: |
| -// docs/superpowers/specs/2026-08-07-exec-child-accounting-design.md. |
| +// docs/superpowers/specs/2026-08-07-exec-child-accounting-design.md. Since |
| +// the memory-pressure spec (2026-08-28) the same per-host cgroup also |
| +// carries the instance's memory cap (Limit, the tiers). |
| // |
| // The agent is NOT root: everything here works through ownership of the |
| // delegated subtree, never privilege. When delegation is absent (old unit |
| @@ -40,6 +42,50 @@ func IsExecName(name string) bool { |
| return strings.HasPrefix(name, execPrefix) && len(name) > len(execPrefix) |
| } |
| +// Limit is what Start writes into a host's cgroup before the child lands |
| +// in it. MemoryMax is bytes; zero means "write nothing" — the pre-spec |
| +// behaviour for a caller with no tier. memory.high is derived (90% of |
| +// max) rather than a second field: a tenant that is throttled before it is |
| +// killed is the whole point, and there is no honest reason for a caller to |
| +// pick a different band. |
| +type Limit struct { |
| + MemoryMax int64 |
| +} |
| + |
| +// The tiers are PLATFORM-chosen (memory-pressure spec §2.3), mirroring |
| +// what caps/render.go already writes for unit-backed apps: the carlos-app@ |
| +// template's 512M, and the 1G a forking family gets when its manifest |
| +// declares `children`. No number comes from a manifest. |
| +var ( |
| + TierDefault = Limit{MemoryMax: 512 << 20} |
| + TierChildren = Limit{MemoryMax: 1 << 30} |
| +) |
| + |
| +// highOf is memory.high for a given memory.max: 90%, so reclaim and |
| +// throttling start well before the kill. |
| +func highOf(max int64) int64 { return max / 10 * 9 } |
| + |
| +// ExecOOMScoreAdj is written to every exec child's /proc/<pid>/oom_score_adj |
| +// right after it starts. The kernel's OOM killer — the memcg one that fires |
| +// when carlos-edge.service's own MemoryMax is hit included — scores per |
| +// PROCESS by RSS, and the agent (~110 MB) outweighs any single tenant |
| +// process on the flagship today, so without this the outer wall's victim |
| +// would be the agent, and Restart=always would then cycle every awake |
| +// instance: precisely what OOMPolicy=continue exists to prevent. +500 adds |
| +// half the box's RAM to a child's score; the agent stays at 0 and is never |
| +// the first pick. Raising one's own-uid process's score needs no privilege |
| +// (lowering does), which is why this is done here rather than by a unit |
| +// directive. Grandchildren inherit the value at fork; a family that forks |
| +// in the microseconds between exec and this write keeps 0 for those early |
| +// forks — the safe direction is still "a tenant process before the agent" |
| +// for everything forked after. |
| +const ExecOOMScoreAdj = "500" |
| + |
| +// oomScoreAdjPath is a seam so a test can point the write at a temp dir. |
| +var oomScoreAdjPath = func(pid int) string { |
| + return filepath.Join("/proc", strconv.Itoa(pid), "oom_score_adj") |
| +} |
| + |
| // ParseSelfCgroup extracts the cgroup v2 path from /proc/self/cgroup |
| // content — the "0::" unified-hierarchy line — relative (no leading slash), |
| // ready to join under /sys/fs/cgroup. Not-ok when there is no v2 line (a |
| @@ -172,7 +218,8 @@ var startCmd = func(cmd *exec.Cmd) error { return cmd.Start() } |
| // a seed's first-boot cost and a sidecar's load ARE that instance's work. |
| func (m *Manager) hostDir(host string) string { return filepath.Join(m.root, ExecName(host)) } |
| -// Start runs build() and starts the result inside host's cgroup. Nil-safe: |
| +// Start runs build() and starts the result inside host's cgroup, with |
| +// limit applied to that cgroup first (memory-pressure spec §2.3). Nil-safe: |
| // with no manager (or any accounting failure) the child starts exactly as |
| // today — a spawn must never fail because accounting failed. build must |
| // return a fully configured, unstarted Cmd; it may be called twice, because |
| @@ -192,7 +239,7 @@ func (m *Manager) hostDir(host string) string { return filepath.Join(m.root, Exe |
| // mis-accounting in the safe direction. Note the PID write migrates only |
| // the child process itself — any grandchildren it has already forked by |
| // then stay charged to the agent's cgroup, not the child's. |
| -func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error) { |
| +func (m *Manager) Start(host string, limit Limit, build func() *exec.Cmd) (*exec.Cmd, error) { |
| if m == nil { |
| cmd := build() |
| return cmd, startCmd(cmd) |
| @@ -203,6 +250,7 @@ func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error) |
| cmd := build() |
| return cmd, startCmd(cmd) |
| } |
| + m.applyLimit(host, dir, limit) |
| if intoCgroupSupported && !m.fallback.Load() { |
| f, err := os.Open(dir) |
| if err != nil { |
| @@ -214,6 +262,7 @@ func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error) |
| err = startCmd(cmd) |
| f.Close() |
| if err == nil { |
| + m.biasOOM(host, cmd) |
| return cmd, nil |
| } |
| firstErr := err |
| @@ -235,6 +284,38 @@ func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error) |
| return m.startAndMove(host, build) |
| } |
| +// applyLimit writes limit into dir. Best-effort in the package's usual |
| +// sense: a refused write (a kernel without the memory controller here, a |
| +// control file that isn't ours) is one logged line and an uncapped child. |
| +// Written BEFORE the child is in the cgroup, so the cap holds from its |
| +// first page. memory.max first: if that one fails, high is pointless. |
| +// |
| +// The cap bounds the cgroup's TOTAL charge — anonymous memory plus the page |
| +// cache its reads pull in — not RSS. A tenant whose cache alone exceeds |
| +// memory.high lives under constant reclaim: slow, and nothing in the |
| +// journal says why. Hence the log line: the tier a host landed on is the |
| +// first thing to read when an app "got slow after the roll". |
| +// |
| +// memory.oom.group=1 makes an OOM kill inside this cgroup take the whole |
| +// family: a hub whose children outlive it would otherwise leave orphans |
| +// holding the socket while the activator respawns a fresh hub beside them. |
| +func (m *Manager) applyLimit(host, dir string, limit Limit) { |
| + if limit.MemoryMax <= 0 { |
| + return |
| + } |
| + if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(limit.MemoryMax, 10)), 0o644); err != nil { |
| + m.logf("cgroup: %s: memory.max: %v — child runs uncapped", host, err) |
| + return |
| + } |
| + if err := os.WriteFile(filepath.Join(dir, "memory.high"), []byte(strconv.FormatInt(highOf(limit.MemoryMax), 10)), 0o644); err != nil { |
| + m.logf("cgroup: %s: memory.high: %v — child is capped but not throttled first", host, err) |
| + } |
| + if err := os.WriteFile(filepath.Join(dir, "memory.oom.group"), []byte("1"), 0o644); err != nil { |
| + m.logf("cgroup: %s: memory.oom.group: %v — an OOM kill here takes one process, not the family", host, err) |
| + } |
| + m.logf("cgroup: %s: memory.max=%dM memory.high=%dM", host, limit.MemoryMax>>20, highOf(limit.MemoryMax)>>20) |
| +} |
| + |
| // startAndMove is the fallback spawn: plain start, then write the child's |
| // PID into the host cgroup. A failed move is logged and tolerated — the |
| // child runs (unaccounted), which beats not running. |
| @@ -259,13 +340,29 @@ func (m *Manager) startAndMove(host string, build func() *exec.Cmd) (*exec.Cmd, |
| if err := os.WriteFile(procs, []byte(strconv.Itoa(cmd.Process.Pid)), 0o644); err != nil { |
| m.logf("cgroup: %s: move pid %d: %v — child stays unaccounted", host, cmd.Process.Pid, err) |
| } |
| + m.biasOOM(host, cmd) |
| return cmd, nil |
| } |
| +// biasOOM writes ExecOOMScoreAdj for a just-started child. Best-effort: a |
| +// refused write is one line, and the child runs with the agent's score. |
| +func (m *Manager) biasOOM(host string, cmd *exec.Cmd) { |
| + if cmd == nil || cmd.Process == nil { |
| + return |
| + } |
| + if err := os.WriteFile(oomScoreAdjPath(cmd.Process.Pid), []byte(ExecOOMScoreAdj), 0o644); err != nil { |
| + m.logf("cgroup: %s: oom_score_adj for pid %d: %v — the agent and this child score alike at an edge-tree OOM", host, cmd.Process.Pid, err) |
| + } |
| +} |
| + |
| // CleanupHost removes host's per-host cgroup dir after its children exit. |
| // Nil-safe and rmdir-tolerant: while a sibling (instance vs sidecar) is |
| // still alive the dir is non-empty and the remove fails — the LAST exiter's |
| // call succeeds, and the next agent start's sweep catches anything missed. |
| +// (On real cgroupfs the control files applyLimit wrote don't count: rmdir |
| +// ignores interface files. In the temp-dir test fixture they are ordinary |
| +// files, so a test that caps a host and then expects CleanupHost to remove |
| +// the dir must delete them first — that failure has no production analogue.) |
| func (m *Manager) CleanupHost(host string) { |
| if m == nil { |
| return |
diff --git a/internal/cgroup/cgroup_linux_test.go b/internal/cgroup/cgroup_linux_test.go| index 05d902b58..5abaa2665 100644 |
| --- a/internal/cgroup/cgroup_linux_test.go |
| +++ b/internal/cgroup/cgroup_linux_test.go |
| @@ -21,7 +21,7 @@ func TestStartUsesCgroupFDFirst(t *testing.T) { |
| t.Fatal(err) |
| } |
| calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| - if _, err := m.Start("h.example.com", buildTrue); err != nil { |
| + if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil { |
| t.Fatal(err) |
| } |
| if len(*calls) != 1 { |
| @@ -56,7 +56,7 @@ func TestStartFallsBackAndLatches(t *testing.T) { |
| } |
| return nil |
| }) |
| - if _, err := m.Start("h.example.com", buildTrue); err != nil { |
| + if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil { |
| t.Fatal(err) |
| } |
| if len(*calls) != 2 { |
| @@ -70,7 +70,7 @@ func TestStartFallsBackAndLatches(t *testing.T) { |
| t.Fatal("refusal did not latch the fallback") |
| } |
| - if _, err := m.Start("h2.example.com", buildTrue); err != nil { |
| + if _, err := m.Start("h2.example.com", Limit{}, buildTrue); err != nil { |
| t.Fatal(err) |
| } |
| if len(*calls) != 3 { |
| @@ -97,7 +97,7 @@ func TestStartMergesIntoBuilderSysProcAttr(t *testing.T) { |
| cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} |
| return cmd |
| } |
| - if _, err := m.Start("h.example.com", build); err != nil { |
| + if _, err := m.Start("h.example.com", Limit{}, build); err != nil { |
| t.Fatal(err) |
| } |
| attr := (*calls)[0].SysProcAttr |
| @@ -126,7 +126,7 @@ func TestStartTransientClone3FailureDoesNotLatch(t *testing.T) { |
| } |
| return nil |
| }) |
| - if _, err := m.Start("h.example.com", buildTrue); err != nil { |
| + if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil { |
| t.Fatal(err) |
| } |
| if len(*calls) != 2 { |
| @@ -136,7 +136,7 @@ func TestStartTransientClone3FailureDoesNotLatch(t *testing.T) { |
| t.Fatal("a transient clone3 failure latched the fallback") |
| } |
| - if _, err := m.Start("h2.example.com", buildTrue); err != nil { |
| + if _, err := m.Start("h2.example.com", Limit{}, buildTrue); err != nil { |
| t.Fatal(err) |
| } |
| // The fake keeps refusing clone3 with ENODEV for h2 too, so the second |
| @@ -172,7 +172,7 @@ func TestStartJoinsBothErrorsOnTotalFailure(t *testing.T) { |
| } |
| return errRetry |
| }) |
| - _, err = m.Start("h.example.com", buildTrue) |
| + _, err = m.Start("h.example.com", Limit{}, buildTrue) |
| if err == nil { |
| t.Fatal("Start with two failing spawns succeeded, want error") |
| } |
diff --git a/internal/cgroup/cgroup_test.go b/internal/cgroup/cgroup_test.go| index ee4ffba26..b7858b343 100644 |
| --- a/internal/cgroup/cgroup_test.go |
| +++ b/internal/cgroup/cgroup_test.go |
| @@ -168,7 +168,7 @@ func buildTrue() *exec.Cmd { return exec.Command("/bin/true") } |
| func TestStartNilManagerSpawnsPlainly(t *testing.T) { |
| calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| var m *Manager |
| - cmd, err := m.Start("h.example.com", buildTrue) |
| + cmd, err := m.Start("h.example.com", Limit{}, buildTrue) |
| if err != nil || cmd == nil { |
| t.Fatalf("Start = (%v, %v), want started cmd", cmd, err) |
| } |
| @@ -187,7 +187,7 @@ func TestStartGenuineSpawnFailureDoesNotLatch(t *testing.T) { |
| t.Fatal(err) |
| } |
| fakeStart(t, func(*exec.Cmd) error { return errors.New("no such binary") }) |
| - if _, err := m.Start("h.example.com", buildTrue); err == nil { |
| + if _, err := m.Start("h.example.com", Limit{}, buildTrue); err == nil { |
| t.Fatal("Start with a failing spawn succeeded, want error") |
| } |
| if m.fallback.Load() { |
| @@ -267,3 +267,121 @@ func TestStartAndMoveToleratesBrokenSeam(t *testing.T) { |
| t.Fatal(err) |
| } |
| } |
| + |
| +// TestStartWritesMemoryLimit: a non-zero Limit lands in the host cgroup as |
| +// memory.max and memory.high (high = 90% of max, so a tenant is throttled |
| +// and reclaimed before it is killed). The temp-dir fixture has no kernel |
| +// behind it, so the assertion is on the bytes written. |
| +func TestStartWritesMemoryLimit(t *testing.T) { |
| + root := newDelegatedRoot(t) |
| + m, err := SetupAt(root, t.Logf) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + if _, err := m.Start("h.example.com", Limit{MemoryMax: 1 << 30}, buildTrue); err != nil { |
| + t.Fatal(err) |
| + } |
| + dir := m.hostDir("h.example.com") |
| + for file, want := range map[string]string{ |
| + "memory.max": "1073741824", |
| + "memory.high": "966367638", // max/10*9, integer arithmetic |
| + "memory.oom.group": "1", |
| + } { |
| + got, err := os.ReadFile(filepath.Join(dir, file)) |
| + if err != nil { |
| + t.Fatalf("%s not written: %v", file, err) |
| + } |
| + if string(got) != want { |
| + t.Errorf("%s = %q, want %q", file, got, want) |
| + } |
| + } |
| +} |
| + |
| +// TestStartZeroLimitWritesNothing: the zero Limit is "no cap" — the |
| +// pre-spec behaviour, byte for byte, for any caller that has no tier. |
| +func TestStartZeroLimitWritesNothing(t *testing.T) { |
| + root := newDelegatedRoot(t) |
| + m, err := SetupAt(root, t.Logf) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, file := range []string{"memory.max", "memory.high"} { |
| + if _, err := os.Stat(filepath.Join(m.hostDir("h.example.com"), file)); !os.IsNotExist(err) { |
| + t.Errorf("%s exists for a zero limit", file) |
| + } |
| + } |
| +} |
| + |
| +// TestStartLimitWriteFailureStillSpawns: a cgroup that refuses the limit |
| +// (read-only control file, an old kernel) costs one log line and an |
| +// uncapped child — never a failed spawn. Same discipline as every other |
| +// accounting failure in this package. |
| +func TestStartLimitWriteFailureStillSpawns(t *testing.T) { |
| + root := newDelegatedRoot(t) |
| + m, err := SetupAt(root, t.Logf) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + dir := m.hostDir("h.example.com") |
| + if err := os.MkdirAll(dir, 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + // A directory where the file should go: WriteFile fails with EISDIR. |
| + if err := os.Mkdir(filepath.Join(dir, "memory.max"), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + cmd, err := m.Start("h.example.com", TierDefault, buildTrue) |
| + if err != nil || cmd == nil { |
| + t.Fatalf("Start = (%v, %v), want a started cmd despite the limit failure", cmd, err) |
| + } |
| + // max failed, so high must NOT have been written: a high with no max |
| + // is a throttle with no wall, and the doc comment promises the short |
| + // circuit. |
| + if _, err := os.Stat(filepath.Join(dir, "memory.high")); !os.IsNotExist(err) { |
| + t.Errorf("memory.high written after memory.max failed: %v", err) |
| + } |
| +} |
| + |
| +// TestStartBiasesChildOOMScore: a started child gets ExecOOMScoreAdj |
| +// written to its oom_score_adj (redirected into a temp dir here), on both |
| +// spawn paths. Without it the agent — the largest process in the edge |
| +// tree — is the memcg OOM killer's first pick when the outer wall fires. |
| +func TestStartBiasesChildOOMScore(t *testing.T) { |
| + root := newDelegatedRoot(t) |
| + m, err := SetupAt(root, t.Logf) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + adj := t.TempDir() |
| + orig := oomScoreAdjPath |
| + oomScoreAdjPath = func(pid int) string { return filepath.Join(adj, strconv.Itoa(pid)) } |
| + t.Cleanup(func() { oomScoreAdjPath = orig }) |
| + // A real start: /bin/true gives the child a pid to write for. |
| + cmd, err := m.startAndMove("h.example.com", buildTrue) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + _ = cmd.Wait() |
| + got, err := os.ReadFile(filepath.Join(adj, strconv.Itoa(cmd.Process.Pid))) |
| + if err != nil { |
| + t.Fatalf("oom_score_adj not written: %v", err) |
| + } |
| + if string(got) != ExecOOMScoreAdj { |
| + t.Errorf("oom_score_adj = %q, want %q", got, ExecOOMScoreAdj) |
| + } |
| +} |
| + |
| +func TestTiers(t *testing.T) { |
| + if TierDefault.MemoryMax != 512<<20 { |
| + t.Errorf("TierDefault = %d, want 512M — the carlos-app@ template's number", TierDefault.MemoryMax) |
| + } |
| + if TierChildren.MemoryMax != 1<<30 { |
| + t.Errorf("TierChildren = %d, want 1G — caps/render.go's children tier", TierChildren.MemoryMax) |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 24d55b52f..294f0ff27 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -276,6 +276,10 @@ func RunAgent(args []string) { |
| Cache: cache, |
| Materialize: materializeHook(reg, dd), |
| Cgroups: cgm, |
| + // The memory tier for each spawned child's cgroup, read off the |
| + // staged manifest of the route's live version (memory-pressure |
| + // spec §2.3): 512M, or 1G for a family that declares children. |
| + Tier: memTier(stagedManifestOfLiveVersion(release.OptDir())), |
| // The scheduled-work verbs on every spawned instance's control |
| // socket write to the SAME registry this agent's converger and |
| // runner read (schedule spec §7.2): an app that asks for a |
| @@ -359,6 +363,9 @@ func RunAgent(args []string) { |
| go logshipLoop(logSink, status.Label(readMachineID())) |
| } |
| go statusLoop(reg) |
| + // The memory early-warning (memory-pressure spec §2.4): per box, gated |
| + // by CARLOS_HOST_METRICS — one log line and nothing else where unset. |
| + go memwatchLoop() |
| go domainsLoop(reg, peekAcmeCache(args)) |
| go instancesLoop(reg) |
| // The deployment-global election, armed BEFORE the loops it gates so a |
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index 2be92e3c5..a4d664c12 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -123,6 +123,14 @@ func TestBootstrapWritesUnitsToStagingDir(t *testing.T) { |
| // cgroup manager degrades to nil and every exec-backed tenant goes |
| // blind on the Instances page. |
| "Delegate=cpu memory io pids", |
| + // The outer wall (memory-pressure spec §2.2): the edge tree is |
| + // where every exec-backed tenant lives, so this is the one place a |
| + // runaway tenant meets a ceiling before the kernel starts thrashing. |
| + // OOMPolicy=continue is the line that keeps one tenant's OOM kill |
| + // from becoming a whole-edge restart (systemd's default is stop). |
| + "MemoryHigh=70%", |
| + "MemoryMax=80%", |
| + "OOMPolicy=continue", |
| } { |
| if !strings.Contains(string(edge), want) { |
| t.Errorf("carlos-edge.service: missing %q", want) |
diff --git a/internal/host/memtier.go b/internal/host/memtier.go| new file mode 100644 |
| index 000000000..5c877b777 |
| --- /dev/null |
| +++ b/internal/host/memtier.go |
| @@ -0,0 +1,26 @@ |
| +package host |
| + |
| +import ( |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// memTier builds the exec backend's Tier seam from a manifest reader — |
| +// on a box, stagedManifestOfLiveVersion, the same reader the caps pass |
| +// uses, so "what does this app declare" has one answer on the box. |
| +// |
| +// One bit decides it: capabilities.children → the 1G tier a forking family |
| +// gets on the unit path (caps/render.go); anything else → 512M, the |
| +// carlos-app@ template's number. A missing or unreadable sibling is the |
| +// default tier, not an error: the smaller cap is the safe direction, and |
| +// the caps pass already reports the unreadable sibling in its own advice. |
| +func memTier(load func(registry.Route) (*release.Manifest, error)) func(registry.Route) cgroup.Limit { |
| + return func(rt registry.Route) cgroup.Limit { |
| + m, err := load(rt) |
| + if err != nil || m == nil || m.Capabilities == nil || !m.Capabilities.Children { |
| + return cgroup.TierDefault |
| + } |
| + return cgroup.TierChildren |
| + } |
| +} |
diff --git a/internal/host/memtier_test.go b/internal/host/memtier_test.go| new file mode 100644 |
| index 000000000..d193bb7fd |
| --- /dev/null |
| +++ b/internal/host/memtier_test.go |
| @@ -0,0 +1,38 @@ |
| +package host |
| + |
| +import ( |
| + "errors" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/caps" |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// TestMemTier: the tier is a platform choice keyed on ONE manifest bit. |
| +// Everything that isn't "declares children" — no manifest, no |
| +// capabilities block, an unreadable sibling — is the default tier: the |
| +// safe direction is the smaller cap, and the caps pass will say why the |
| +// sibling is unreadable in its own advice. |
| +func TestMemTier(t *testing.T) { |
| + rt := registry.Route{Host: "h.example.com", App: "app"} |
| + cases := []struct { |
| + name string |
| + m *release.Manifest |
| + err error |
| + want cgroup.Limit |
| + }{ |
| + {"no manifest", nil, nil, cgroup.TierDefault}, |
| + {"unreadable", nil, errors.New("boom"), cgroup.TierDefault}, |
| + {"no capabilities", &release.Manifest{}, nil, cgroup.TierDefault}, |
| + {"capabilities, no children", &release.Manifest{Capabilities: &caps.Set{V: caps.Version}}, nil, cgroup.TierDefault}, |
| + {"children", &release.Manifest{Capabilities: &caps.Set{V: caps.Version, Children: true}}, nil, cgroup.TierChildren}, |
| + } |
| + for _, c := range cases { |
| + tier := memTier(func(registry.Route) (*release.Manifest, error) { return c.m, c.err }) |
| + if got := tier(rt); got != c.want { |
| + t.Errorf("%s: tier = %v, want %v", c.name, got, c.want) |
| + } |
| + } |
| +} |
diff --git a/internal/host/memwatch.go b/internal/host/memwatch.go| new file mode 100644 |
| index 000000000..54781fad8 |
| --- /dev/null |
| +++ b/internal/host/memwatch.go |
| @@ -0,0 +1,208 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "log" |
| + "net/http" |
| + "os" |
| + "strconv" |
| + "strings" |
| + "time" |
| + |
| + "github.com/aws/aws-sdk-go-v2/aws" |
| + awsconfig "github.com/aws/aws-sdk-go-v2/config" |
| + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" |
| + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" |
| +) |
| + |
| +// memwatch (memory-pressure spec §2.4) publishes two CloudWatch metrics a |
| +// minute so a box that is running out of memory pages someone BEFORE the |
| +// kernel starts thrashing — the state the flagship sat in for fifteen |
| +// minutes on 2026-08-28 with every AWS-side check green. The status |
| +// snapshot already carries memory to the bucket every five minutes, but a |
| +// bucket write alarms nobody; CloudWatch does. |
| +// |
| +// Carlos/Host MemAvailablePercent MemAvailable / MemTotal, /proc/meminfo |
| +// Carlos/Host MemoryPressureSome60 some avg60 from /proc/pressure/memory |
| +// |
| +// Both keyed by InstanceId (IMDS). The region-host module's alarm reads the |
| +// first; the second is the early warning — it climbs minutes before |
| +// MemAvailable hits the floor. |
| +// |
| +// Per-box, not a housekeeper duty: every box reports itself. Armed only by |
| +// CARLOS_HOST_METRICS=cloudwatch in host.env, so a Pi or a customer fleet |
| +// box never tries to reach an API it has no credentials for. |
| + |
| +const ( |
| + memwatchNamespace = "Carlos/Host" |
| + memwatchPeriod = time.Minute |
| +) |
| + |
| +// metricPutter is the one CloudWatch call this file makes. |
| +type metricPutter interface { |
| + PutMetricData(context.Context, *cloudwatch.PutMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) |
| +} |
| + |
| +type memwatch struct { |
| + put metricPutter |
| + instanceID string |
| + mem func() (*int64, *int64) // MemTotal, MemAvailable in bytes |
| + pressure func() (string, error) // /proc/pressure/memory content |
| + logf func(string, ...any) |
| + // lastPutErr is the text of the put failure last logged, "" while |
| + // puts succeed. Failures log on CHANGE (and recovery logs once), not |
| + // once a minute: a box rolled before its IAM grant lands would |
| + // otherwise write 1,440 identical lines a day. |
| + lastPutErr string |
| +} |
| + |
| +// memwatchArmed is the host.env gate, split out so the three answers it |
| +// can give are table-testable: armed, off (unset), or off with a value |
| +// this duty doesn't understand — the last is the one an operator wants to |
| +// see in the journal, because they typed it. |
| +func memwatchArmed(getenv func(string) string) (armed bool, why string) { |
| + switch v := strings.TrimSpace(getenv("CARLOS_HOST_METRICS")); v { |
| + case "cloudwatch": |
| + return true, "" |
| + case "": |
| + return false, "CARLOS_HOST_METRICS unset" |
| + default: |
| + return false, "CARLOS_HOST_METRICS=" + strconv.Quote(v) + ` (the only value is "cloudwatch")` |
| + } |
| +} |
| + |
| +// memwatchLoop arms the duty when host.env asks for it. One log line either |
| +// way; a box that can't resolve its instance id or build a client logs why |
| +// and stays quiet rather than retrying — the operator set the env, so a |
| +// silent duty is a bug they'd want to see at arm time, in the journal. |
| +// Process-lifetime, like statusLoop: nothing stops the ticker. |
| +func memwatchLoop() { |
| + if armed, why := memwatchArmed(os.Getenv); !armed { |
| + log.Printf("agent: memwatch disabled (%s)", why) |
| + return |
| + } |
| + ctx := context.Background() |
| + // Region comes from the default chain — AWS_REGION in host.env, else |
| + // IMDS. A compatible-store deployment that points AWS_REGION at its |
| + // own object store (internal/lease/endpoint.go's note) would aim the |
| + // CloudWatch client at nonsense, which is one more reason this duty |
| + // is opt-in per box rather than on wherever credentials resolve. |
| + cfg, err := awsconfig.LoadDefaultConfig(ctx) |
| + if err != nil { |
| + log.Printf("agent: memwatch disabled: load aws config: %v", err) |
| + return |
| + } |
| + id, err := imdsInstanceID() |
| + if err != nil { |
| + log.Printf("agent: memwatch disabled: instance id: %v", err) |
| + return |
| + } |
| + w := memwatch{ |
| + put: cloudwatch.NewFromConfig(cfg), |
| + instanceID: id, |
| + mem: readMemBytes, |
| + pressure: readPressureFile, |
| + logf: log.Printf, |
| + } |
| + log.Printf("agent: memwatch armed (%s, every %s, instance %s)", memwatchNamespace, memwatchPeriod, id) |
| + for now := range time.Tick(memwatchPeriod) { |
| + w.tick(ctx, now) |
| + } |
| +} |
| + |
| +// tick reads, builds, puts. A failed read skips the tick (a zero would look |
| +// like a full box); a failed put is logged when it first happens or changes, |
| +// and the next tick tries again. |
| +func (w *memwatch) tick(ctx context.Context, now time.Time) { |
| + total, avail := w.mem() |
| + if total == nil || avail == nil || *total <= 0 { |
| + w.logf("memwatch: /proc/meminfo unreadable this tick — skipped") |
| + return |
| + } |
| + availPct := float64(*avail) / float64(*total) * 100 |
| + psi, hasPSI := 0.0, false |
| + if s, err := w.pressure(); err == nil { |
| + psi, hasPSI = parsePressureSome60(s) |
| + } |
| + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) |
| + defer cancel() |
| + _, err := w.put.PutMetricData(ctx, memwatchData(w.instanceID, availPct, psi, hasPSI, now)) |
| + switch { |
| + case err != nil && err.Error() != w.lastPutErr: |
| + w.lastPutErr = err.Error() |
| + w.logf("memwatch: put: %v (logged once until it changes)", err) |
| + case err == nil && w.lastPutErr != "": |
| + w.lastPutErr = "" |
| + w.logf("memwatch: put recovered") |
| + } |
| +} |
| + |
| +// memwatchData is the pure half: the exact PutMetricData input for one |
| +// tick. Standard resolution — a one-minute period at standard resolution |
| +// is free-tier territory; high resolution is not and buys nothing for a |
| +// three-minute alarm. |
| +func memwatchData(instanceID string, availPct, psi float64, hasPSI bool, now time.Time) *cloudwatch.PutMetricDataInput { |
| + dims := []types.Dimension{{Name: aws.String("InstanceId"), Value: aws.String(instanceID)}} |
| + data := []types.MetricDatum{{ |
| + MetricName: aws.String("MemAvailablePercent"), |
| + Dimensions: dims, |
| + Timestamp: aws.Time(now), |
| + Unit: types.StandardUnitPercent, |
| + Value: aws.Float64(availPct), |
| + }} |
| + if hasPSI { |
| + data = append(data, types.MetricDatum{ |
| + MetricName: aws.String("MemoryPressureSome60"), |
| + Dimensions: dims, |
| + Timestamp: aws.Time(now), |
| + Unit: types.StandardUnitPercent, |
| + Value: aws.Float64(psi), |
| + }) |
| + } |
| + return &cloudwatch.PutMetricDataInput{Namespace: aws.String(memwatchNamespace), MetricData: data} |
| +} |
| + |
| +// imdsInstanceID asks the metadata service which instance this is — the |
| +// dimension the alarm keys on — through the same IMDSv2 token dance |
| +// readPlacementRegion uses (imdsToken/imdsLeaf). The IMDS uid-gate |
| +// (harden-host) lets the carlos user through by construction. |
| +func imdsInstanceID() (string, error) { |
| + client := &http.Client{Timeout: 2 * time.Second} |
| + token := imdsToken(client) |
| + if token == "" { |
| + return "", errors.New("no IMDSv2 token (not on EC2, or IMDS unreachable)") |
| + } |
| + id, code := imdsLeaf(client, token, "/latest/meta-data/instance-id") |
| + if code != http.StatusOK || id == "" { |
| + return "", errors.New("instance-id leaf did not answer 200") |
| + } |
| + return id, nil |
| +} |
| + |
| +// readPressureFile is the /proc/pressure/memory read; absent on a kernel |
| +// without PSI (CONFIG_PSI) — the tick then publishes MemAvailable alone. |
| +func readPressureFile() (string, error) { |
| + b, err := os.ReadFile("/proc/pressure/memory") |
| + return string(b), err |
| +} |
| + |
| +// parsePressureSome60 pulls "some ... avg60=N" out of /proc/pressure/memory. |
| +// Not-ok for any shape it doesn't recognise: a parse bug must not report a |
| +// calm box. |
| +func parsePressureSome60(s string) (float64, bool) { |
| + for _, line := range strings.Split(s, "\n") { |
| + fields := strings.Fields(line) |
| + if len(fields) == 0 || fields[0] != "some" { |
| + continue |
| + } |
| + for _, f := range fields[1:] { |
| + if v, ok := strings.CutPrefix(f, "avg60="); ok { |
| + n, err := strconv.ParseFloat(v, 64) |
| + return n, err == nil |
| + } |
| + } |
| + return 0, false |
| + } |
| + return 0, false |
| +} |
diff --git a/internal/host/memwatch_test.go b/internal/host/memwatch_test.go| new file mode 100644 |
| index 000000000..9068a4730 |
| --- /dev/null |
| +++ b/internal/host/memwatch_test.go |
| @@ -0,0 +1,155 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" |
| +) |
| + |
| +// TestParsePressureSome60 pins /proc/pressure/memory's shape: two lines, |
| +// "some" then "full", each with avg10/avg60/avg300/total. Only some/avg60 |
| +// is wanted. A kernel without PSI has no file at all — the caller handles |
| +// that — but a file with an unexpected shape reports not-ok rather than 0, |
| +// so a parse bug can't masquerade as a healthy box. |
| +func TestParsePressureSome60(t *testing.T) { |
| + good := "some avg10=0.00 avg60=12.50 avg300=3.10 total=171234\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=153521\n" |
| + if got, ok := parsePressureSome60(good); !ok || got != 12.5 { |
| + t.Fatalf("parse = (%v, %v), want (12.5, true)", got, ok) |
| + } |
| + for name, s := range map[string]string{ |
| + "empty": "", |
| + "full only": "full avg10=0.00 avg60=0.00 avg300=0.00 total=1\n", |
| + "no avg60": "some avg10=0.00 avg300=0.00 total=1\n", |
| + "not number": "some avg10=0.00 avg60=x avg300=0.00 total=1\n", |
| + } { |
| + if _, ok := parsePressureSome60(s); ok { |
| + t.Errorf("%s: parsed ok, want not-ok", name) |
| + } |
| + } |
| +} |
| + |
| +// TestMemwatchData: the tick's PutMetricData input. Two data points when |
| +// PSI is readable, one when it is not; namespace, dimension and units are |
| +// what the region-host module's alarm and IAM condition name — a drift |
| +// here is an alarm that never fires, silently. |
| +func TestMemwatchData(t *testing.T) { |
| + now := time.Date(2026, 8, 28, 11, 4, 0, 0, time.UTC) |
| + in := memwatchData("i-0123", 14.2, 30.5, true, now) |
| + if *in.Namespace != "Carlos/Host" { |
| + t.Fatalf("namespace %q", *in.Namespace) |
| + } |
| + if len(in.MetricData) != 2 { |
| + t.Fatalf("%d data points, want 2", len(in.MetricData)) |
| + } |
| + names := map[string]float64{} |
| + for _, d := range in.MetricData { |
| + names[*d.MetricName] = *d.Value |
| + if len(d.Dimensions) != 1 || *d.Dimensions[0].Name != "InstanceId" || *d.Dimensions[0].Value != "i-0123" { |
| + t.Errorf("%s: dimensions %+v", *d.MetricName, d.Dimensions) |
| + } |
| + if !d.Timestamp.Equal(now) { |
| + t.Errorf("%s: timestamp %v", *d.MetricName, d.Timestamp) |
| + } |
| + } |
| + if names["MemAvailablePercent"] != 14.2 || names["MemoryPressureSome60"] != 30.5 { |
| + t.Errorf("values %v", names) |
| + } |
| + if in := memwatchData("i-0123", 14.2, 0, false, now); len(in.MetricData) != 1 || *in.MetricData[0].MetricName != "MemAvailablePercent" { |
| + t.Errorf("without PSI: %+v", in.MetricData) |
| + } |
| +} |
| + |
| +type fakePutter struct { |
| + calls []*cloudwatch.PutMetricDataInput |
| +} |
| + |
| +func (f *fakePutter) PutMetricData(_ context.Context, in *cloudwatch.PutMetricDataInput, _ ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) { |
| + f.calls = append(f.calls, in) |
| + return &cloudwatch.PutMetricDataOutput{}, nil |
| +} |
| + |
| +// TestMemwatchTick: one tick, one call, built from the readers it is |
| +// handed. An unreadable meminfo is a skipped tick, not a zero. |
| +func TestMemwatchTick(t *testing.T) { |
| + f := &fakePutter{} |
| + w := memwatch{ |
| + put: f, |
| + instanceID: "i-0123", |
| + mem: func() (*int64, *int64) { tot, av := int64(4000), int64(1000); return &tot, &av }, |
| + pressure: func() (string, error) { |
| + return "some avg10=0 avg60=7.5 avg300=0 total=0\nfull avg10=0 avg60=0 avg300=0 total=0\n", nil |
| + }, |
| + logf: t.Logf, |
| + } |
| + w.tick(context.Background(), time.Unix(0, 0)) |
| + if len(f.calls) != 1 || len(f.calls[0].MetricData) != 2 { |
| + t.Fatalf("calls %+v", f.calls) |
| + } |
| + if v := *f.calls[0].MetricData[0].Value; v != 25 { |
| + t.Errorf("MemAvailablePercent = %v, want 25", v) |
| + } |
| + w.mem = func() (*int64, *int64) { return nil, nil } |
| + w.tick(context.Background(), time.Unix(0, 0)) |
| + if len(f.calls) != 1 { |
| + t.Errorf("a tick with unreadable meminfo must not put: %d calls", len(f.calls)) |
| + } |
| +} |
| + |
| +// TestMemwatchArmed: the gate's three answers. The unknown-value case is |
| +// the one that matters — an operator who typed CARLOS_HOST_METRICS=true |
| +// gets told what the only value is, in the journal, at arm time. |
| +func TestMemwatchArmed(t *testing.T) { |
| + for v, want := range map[string]bool{"cloudwatch": true, " cloudwatch ": true, "": false, "true": false} { |
| + armed, why := memwatchArmed(func(string) string { return v }) |
| + if armed != want { |
| + t.Errorf("%q: armed=%v, want %v", v, armed, want) |
| + } |
| + if !armed && why == "" { |
| + t.Errorf("%q: disabled with no reason", v) |
| + } |
| + } |
| + if _, why := memwatchArmed(func(string) string { return "true" }); !strings.Contains(why, `"cloudwatch"`) { |
| + t.Errorf("unknown value's reason %q does not name the only valid value", why) |
| + } |
| +} |
| + |
| +type failPutter struct{ err error } |
| + |
| +func (f *failPutter) PutMetricData(context.Context, *cloudwatch.PutMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) { |
| + return nil, f.err |
| +} |
| + |
| +// TestMemwatchPutFailureLogsOnChange: the same failure minute after minute |
| +// is ONE line; a different failure is another; recovery is one more. |
| +func TestMemwatchPutFailureLogsOnChange(t *testing.T) { |
| + var lines []string |
| + fp := &failPutter{err: errors.New("AccessDenied")} |
| + w := memwatch{ |
| + put: fp, |
| + instanceID: "i-0123", |
| + mem: func() (*int64, *int64) { tot, av := int64(4000), int64(1000); return &tot, &av }, |
| + pressure: func() (string, error) { return "", errors.New("no psi") }, |
| + logf: func(f string, a ...any) { lines = append(lines, f) }, |
| + } |
| + for i := 0; i < 3; i++ { |
| + w.tick(context.Background(), time.Unix(0, 0)) |
| + } |
| + if len(lines) != 1 { |
| + t.Fatalf("3 identical failures logged %d lines, want 1: %v", len(lines), lines) |
| + } |
| + fp.err = errors.New("Throttling") |
| + w.tick(context.Background(), time.Unix(0, 0)) |
| + if len(lines) != 2 { |
| + t.Fatalf("a changed failure did not log: %v", lines) |
| + } |
| + w.put = &fakePutter{} |
| + w.tick(context.Background(), time.Unix(0, 0)) |
| + w.tick(context.Background(), time.Unix(0, 0)) |
| + if len(lines) != 3 || !strings.Contains(lines[2], "recovered") { |
| + t.Fatalf("recovery logged wrong: %v", lines) |
| + } |
| +} |
diff --git a/internal/host/units/carlos-edge.service b/internal/host/units/carlos-edge.service| index ee57b7f40..018304cec 100644 |
| --- a/internal/host/units/carlos-edge.service |
| +++ b/internal/host/units/carlos-edge.service |
| @@ -54,8 +54,29 @@ MemoryDenyWriteExecute=yes |
| SystemCallFilter=@system-service |
| SystemCallErrorNumber=EPERM |
| SystemCallArchitectures=native |
| -# No MemoryMax: hibernating instances are this unit's children, so a cap here |
| -# would cap the whole set of awake instances at once. |
| +# The outer wall (memory-pressure spec 2026-08-28 §2.2). Exec-backed |
| +# instances are this unit's children, so these DO cap the whole set of |
| +# awake instances at once — that is now the point: on 2026-08-28 the |
| +# flagship thrashed for 15 minutes with no swap, no ceiling and an OOM |
| +# killer that never fired, because nothing bounded the tree. Percentages |
| +# so one file fits a t4g.micro and a t4g.medium; 80% leaves what the |
| +# box's other units (keymail, console, ssm, litestream, journald) use. |
| +# Past High the kernel reclaims the tree (page cache first, then zram); |
| +# at Max the memcg OOM killer picks the highest-scoring PROCESS in the |
| +# tree. By RSS alone that would be the agent itself (~110 MB against |
| +# 20-60 MB tenant processes), so the agent writes oom_score_adj=+500 on |
| +# every exec child it starts (internal/cgroup ExecOOMScoreAdj) — a tenant |
| +# process is always the first pick. Per-instance caps below this |
| +# (carlos-exec@<host> memory.max, internal/cgroup) are the inner walls. |
| +MemoryHigh=70% |
| +MemoryMax=80% |
| +# systemd's default OOMPolicy=stop STOPS the whole service when the kernel |
| +# OOM-kills ANY process in the unit's cgroup tree (memory.events is |
| +# hierarchical, delegation or not). Restart=always would bring the edge |
| +# back a second later and cycle every awake instance on the box. With |
| +# continue, one tenant's kill is one tenant's kill; the activator recycles |
| +# it on its next request. |
| +OOMPolicy=continue |
| TasksMax=1024 |
| # Exec-backed instances run as this unit's children. Delegation hands the |
| # unit its own cgroup subtree — chowned to User=carlos, unmanaged by systemd |