amadan

carlos / platform Public

host: memwatch duty — MemAvailablePercent + PSI to CloudWatch once a minute, gated by CARLOS_HOST_METRICS (memory-pressure spec §2.4)

Paul Campbell pushed by paul@keymail.dev 41260e5e0d09db0d58f695bbbb0744617a9c6536 parent f9d772b
5 files changed, +293 −13
  • go.mod +6 −5
  • go.sum +10 −8
  • internal/host/agent.go +3 −0
  • internal/host/memwatch.go +176 −0
  • internal/host/memwatch_test.go +98 −0
diff --git a/go.mod b/go.mod
index dd7e91d44..6f666b88e 100644
--- a/go.mod
+++ b/go.mod
@@ -3,16 +3,18 @@ module github.com/carlosframework/platform
go 1.26
require (
- github.com/aws/aws-sdk-go-v2 v1.43.6
+ github.com/aws/aws-sdk-go-v2 v1.44.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16
+ github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0
github.com/aws/aws-sdk-go-v2/service/costexplorer v1.67.5
github.com/aws/aws-sdk-go-v2/service/iam v1.58.2
github.com/aws/aws-sdk-go-v2/service/route53 v1.64.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0
github.com/aws/aws-sdk-go-v2/service/sesv2 v1.66.6
github.com/aws/aws-sdk-go-v2/service/sts v1.45.6
- github.com/aws/smithy-go v1.27.8
+ github.com/aws/smithy-go v1.28.1
github.com/benbjohnson/litestream v0.5.14
golang.org/x/crypto v0.52.0
golang.org/x/net v0.55.0
@@ -21,10 +23,9 @@ require (
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
- github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18 // indirect
- github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect
- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect
diff --git a/go.sum b/go.sum
index d8d482895..297f59edf 100644
--- a/go.sum
+++ b/go.sum
@@ -18,8 +18,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJ
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0 h1:wQlqotpyjYPjJz+Noh5bRu7Snmydk8SKC5Z6u1CR20Y=
github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.3.0/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M=
-github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ=
-github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00=
+github.com/aws/aws-sdk-go-v2 v1.44.0 h1:4IbaHhtzy+4h37z4JQyO9a2QsiCml3CNYHtq5hIHigo=
+github.com/aws/aws-sdk-go-v2 v1.44.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
@@ -30,14 +30,16 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBU
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18 h1:9vWXHtaepwoAl/UuKzxwgOoJDXPCC3hvgNMfcmdS2Tk=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18/go.mod h1:sKuUZ+MwUTuJbYvZ8pK0x10LvgcJK3Y4rmh63YBekwk=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 h1:lznzIOvvbqjfe8UAaciCRJgBgJsxuTROKlhZuXQWfv8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37/go.mod h1:otfkzyfQeMMLZAqX59GSXTL3o22BR/l6HFaRzzbWSqA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 h1:zCEORWo0eU0gDjG+IyApE/2B+ZGG1m+GU7B263XV8ds=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37/go.mod h1:i6c0PEl3TNOWxRbQ++KQcVenPWS/GoQeiklKhNuqzJ8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 h1:UIXlbijuB2XK1Kr57fo8iIxCuaSHJzwZ1uo+2tbEYIk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40/go.mod h1:wcEsL6jscjZjVUinb0Q5qD/GXOG1yT3GNfmT9HuDwzU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 h1:xLQVRDs2NddDmK9BEyh5KSlJ1Gpy5/GIJXrV6WcVGAE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40/go.mod h1:XRXnpFVFGLaEVK+olDdFIM1vNa04ETW452oFGEPUxAo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 h1:A3UAuCmx7LyUcrixBTzKJYYIUZ2yTvn6ZhT8PB+7APk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38/go.mod h1:1PDUYG9Z+JrbbsobsAZHjWOm9QBT/djiK3QbykTL5Z4=
+github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0 h1:RdWMeCVaJfu4hCPyRNVQMKMP/Y5J04zB5LF45DmHhRg=
+github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.68.0/go.mod h1:HfsU8PQ6mHewSJNS77IPAFMZZgFaYSHjAhSOAFXdu9M=
github.com/aws/aws-sdk-go-v2/service/costexplorer v1.67.5 h1:hiJJTfio1+i/xnfLwu6RsDjDzkGSiLux54LTzDK4MyA=
github.com/aws/aws-sdk-go-v2/service/costexplorer v1.67.5/go.mod h1:gy1JJMsby9XLbrwMhJcMu0IuMJF4DvDe4pD1wqMmwkk=
github.com/aws/aws-sdk-go-v2/service/iam v1.58.2 h1:/6iRcqrC6k1rMA6uCZMzFE9inOrBpNmhbrZ90X8qH50=
@@ -64,8 +66,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlV
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 h1:JvExZWabChDM0qJAirQYGfOYo0ndT3edXj+fqSPNjkE=
github.com/aws/aws-sdk-go-v2/service/sts v1.45.6/go.mod h1:XZcaQkV2cItp6yEkrwljyaPOf22RuX7T43jxap/FOmM=
-github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY=
-github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
+github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/benbjohnson/litestream v0.5.14 h1:pu2RCUfEs2MfAvxSttNg/kVVzJspqg2sX/pPW3Elzxc=
github.com/benbjohnson/litestream v0.5.14/go.mod h1:aN5OtdkcCz07xVjCbmMWOGUSIdPumpqBYtdyrzNMhOI=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 0a9825ef8..6b7e05792 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -353,6 +353,9 @@ func RunAgent(args []string) {
go logshipLoop(logSink, status.Label(readMachineID()))
}
go statusLoop(reg)
+ // The memory early-warning (memory-pressure spec §2.4): per box, gated
+ // by CARLOS_HOST_METRICS — one log line and nothing else where unset.
+ go memwatchLoop()
go domainsLoop(reg, peekAcmeCache(args))
go instancesLoop(reg)
// The deployment-global election, armed BEFORE the loops it gates so a
diff --git a/internal/host/memwatch.go b/internal/host/memwatch.go
new file mode 100644
index 000000000..bf2ac4a8a
--- /dev/null
+++ b/internal/host/memwatch.go
@@ -0,0 +1,176 @@
+package host
+
+import (
+ "context"
+ "log"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch"
+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
+)
+
+// memwatch (memory-pressure spec §2.4) publishes two CloudWatch metrics a
+// minute so a box that is running out of memory pages someone BEFORE the
+// kernel starts thrashing — the state the flagship sat in for fifteen
+// minutes on 2026-08-28 with every AWS-side check green. The status
+// snapshot already carries memory to the bucket every five minutes, but a
+// bucket write alarms nobody; CloudWatch does.
+//
+// Carlos/Host MemAvailablePercent MemAvailable / MemTotal, /proc/meminfo
+// Carlos/Host MemoryPressureSome60 some avg60 from /proc/pressure/memory
+//
+// Both keyed by InstanceId (IMDS). The region-host module's alarm reads the
+// first; the second is the early warning — it climbs minutes before
+// MemAvailable hits the floor.
+//
+// Per-box, not a housekeeper duty: every box reports itself. Armed only by
+// CARLOS_HOST_METRICS=cloudwatch in host.env, so a Pi or a customer fleet
+// box never tries to reach an API it has no credentials for.
+
+const (
+ memwatchNamespace = "Carlos/Host"
+ memwatchPeriod = time.Minute
+)
+
+// metricPutter is the one CloudWatch call this file makes.
+type metricPutter interface {
+ PutMetricData(context.Context, *cloudwatch.PutMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error)
+}
+
+type memwatch struct {
+ put metricPutter
+ instanceID string
+ mem func() (*int64, *int64) // MemTotal, MemAvailable in bytes
+ pressure func() (string, error) // /proc/pressure/memory content
+ logf func(string, ...any)
+}
+
+// memwatchLoop arms the duty when host.env asks for it. One log line either
+// way; a box that can't resolve its instance id or build a client logs why
+// and stays quiet rather than retrying — the operator set the env, so a
+// silent duty is a bug they'd want to see at arm time, in the journal.
+func memwatchLoop() {
+ if v := strings.TrimSpace(os.Getenv("CARLOS_HOST_METRICS")); v != "cloudwatch" {
+ if v != "" {
+ log.Printf("agent: memwatch disabled: CARLOS_HOST_METRICS=%q (the only value is \"cloudwatch\")", v)
+ } else {
+ log.Printf("agent: memwatch disabled (CARLOS_HOST_METRICS unset)")
+ }
+ return
+ }
+ ctx := context.Background()
+ cfg, err := awsconfig.LoadDefaultConfig(ctx)
+ if err != nil {
+ log.Printf("agent: memwatch disabled: load aws config: %v", err)
+ return
+ }
+ id, err := imdsInstanceID(ctx, cfg)
+ if err != nil {
+ log.Printf("agent: memwatch disabled: instance id: %v", err)
+ return
+ }
+ w := memwatch{
+ put: cloudwatch.NewFromConfig(cfg),
+ instanceID: id,
+ mem: readMemBytes,
+ pressure: readPressureFile,
+ logf: log.Printf,
+ }
+ log.Printf("agent: memwatch armed (%s, every %s, instance %s)", memwatchNamespace, memwatchPeriod, id)
+ t := time.NewTicker(memwatchPeriod)
+ defer t.Stop()
+ for now := range t.C {
+ w.tick(ctx, now)
+ }
+}
+
+// tick reads, builds, puts. A failed read skips the tick (a zero would look
+// like a full box); a failed put is one line and the next tick tries again.
+func (w *memwatch) tick(ctx context.Context, now time.Time) {
+ total, avail := w.mem()
+ if total == nil || avail == nil || *total <= 0 {
+ w.logf("memwatch: /proc/meminfo unreadable this tick — skipped")
+ return
+ }
+ availPct := float64(*avail) / float64(*total) * 100
+ psi, hasPSI := 0.0, false
+ if s, err := w.pressure(); err == nil {
+ psi, hasPSI = parsePressureSome60(s)
+ }
+ ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+ defer cancel()
+ if _, err := w.put.PutMetricData(ctx, memwatchData(w.instanceID, availPct, psi, hasPSI, now)); err != nil {
+ w.logf("memwatch: put: %v", err)
+ }
+}
+
+// memwatchData is the pure half: the exact PutMetricData input for one
+// tick. Standard resolution — a one-minute period at standard resolution
+// is free-tier territory; high resolution is not and buys nothing for a
+// three-minute alarm.
+func memwatchData(instanceID string, availPct, psi float64, hasPSI bool, now time.Time) *cloudwatch.PutMetricDataInput {
+ dims := []types.Dimension{{Name: aws.String("InstanceId"), Value: aws.String(instanceID)}}
+ data := []types.MetricDatum{{
+ MetricName: aws.String("MemAvailablePercent"),
+ Dimensions: dims,
+ Timestamp: aws.Time(now),
+ Unit: types.StandardUnitPercent,
+ Value: aws.Float64(availPct),
+ }}
+ if hasPSI {
+ data = append(data, types.MetricDatum{
+ MetricName: aws.String("MemoryPressureSome60"),
+ Dimensions: dims,
+ Timestamp: aws.Time(now),
+ Unit: types.StandardUnitPercent,
+ Value: aws.Float64(psi),
+ })
+ }
+ return &cloudwatch.PutMetricDataInput{Namespace: aws.String(memwatchNamespace), MetricData: data}
+}
+
+// imdsInstanceID asks the metadata service which instance this is — the
+// dimension the alarm keys on. The IMDS uid-gate (harden-host) lets the
+// carlos user through by construction.
+func imdsInstanceID(ctx context.Context, cfg aws.Config) (string, error) {
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ out, err := imds.NewFromConfig(cfg).GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
+ if err != nil {
+ return "", err
+ }
+ return out.InstanceID, nil
+}
+
+// readPressureFile is the /proc/pressure/memory read; absent on a kernel
+// without PSI (CONFIG_PSI) — the tick then publishes MemAvailable alone.
+func readPressureFile() (string, error) {
+ b, err := os.ReadFile("/proc/pressure/memory")
+ return string(b), err
+}
+
+// parsePressureSome60 pulls "some ... avg60=N" out of /proc/pressure/memory.
+// Not-ok for any shape it doesn't recognise: a parse bug must not report a
+// calm box.
+func parsePressureSome60(s string) (float64, bool) {
+ for _, line := range strings.Split(s, "\n") {
+ fields := strings.Fields(line)
+ if len(fields) == 0 || fields[0] != "some" {
+ continue
+ }
+ for _, f := range fields[1:] {
+ if v, ok := strings.CutPrefix(f, "avg60="); ok {
+ n, err := strconv.ParseFloat(v, 64)
+ return n, err == nil
+ }
+ }
+ return 0, false
+ }
+ return 0, false
+}
diff --git a/internal/host/memwatch_test.go b/internal/host/memwatch_test.go
new file mode 100644
index 000000000..24ef0e436
--- /dev/null
+++ b/internal/host/memwatch_test.go
@@ -0,0 +1,98 @@
+package host
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/service/cloudwatch"
+)
+
+// TestParsePressureSome60 pins /proc/pressure/memory's shape: two lines,
+// "some" then "full", each with avg10/avg60/avg300/total. Only some/avg60
+// is wanted. A kernel without PSI has no file at all — the caller handles
+// that — but a file with an unexpected shape reports not-ok rather than 0,
+// so a parse bug can't masquerade as a healthy box.
+func TestParsePressureSome60(t *testing.T) {
+ good := "some avg10=0.00 avg60=12.50 avg300=3.10 total=171234\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=153521\n"
+ if got, ok := parsePressureSome60(good); !ok || got != 12.5 {
+ t.Fatalf("parse = (%v, %v), want (12.5, true)", got, ok)
+ }
+ for name, s := range map[string]string{
+ "empty": "",
+ "full only": "full avg10=0.00 avg60=0.00 avg300=0.00 total=1\n",
+ "no avg60": "some avg10=0.00 avg300=0.00 total=1\n",
+ "not number": "some avg10=0.00 avg60=x avg300=0.00 total=1\n",
+ } {
+ if _, ok := parsePressureSome60(s); ok {
+ t.Errorf("%s: parsed ok, want not-ok", name)
+ }
+ }
+}
+
+// TestMemwatchData: the tick's PutMetricData input. Two data points when
+// PSI is readable, one when it is not; namespace, dimension and units are
+// what the region-host module's alarm and IAM condition name — a drift
+// here is an alarm that never fires, silently.
+func TestMemwatchData(t *testing.T) {
+ now := time.Date(2026, 8, 28, 11, 4, 0, 0, time.UTC)
+ in := memwatchData("i-0123", 14.2, 30.5, true, now)
+ if *in.Namespace != "Carlos/Host" {
+ t.Fatalf("namespace %q", *in.Namespace)
+ }
+ if len(in.MetricData) != 2 {
+ t.Fatalf("%d data points, want 2", len(in.MetricData))
+ }
+ names := map[string]float64{}
+ for _, d := range in.MetricData {
+ names[*d.MetricName] = *d.Value
+ if len(d.Dimensions) != 1 || *d.Dimensions[0].Name != "InstanceId" || *d.Dimensions[0].Value != "i-0123" {
+ t.Errorf("%s: dimensions %+v", *d.MetricName, d.Dimensions)
+ }
+ if !d.Timestamp.Equal(now) {
+ t.Errorf("%s: timestamp %v", *d.MetricName, d.Timestamp)
+ }
+ }
+ if names["MemAvailablePercent"] != 14.2 || names["MemoryPressureSome60"] != 30.5 {
+ t.Errorf("values %v", names)
+ }
+ if in := memwatchData("i-0123", 14.2, 0, false, now); len(in.MetricData) != 1 || *in.MetricData[0].MetricName != "MemAvailablePercent" {
+ t.Errorf("without PSI: %+v", in.MetricData)
+ }
+}
+
+type fakePutter struct {
+ calls []*cloudwatch.PutMetricDataInput
+}
+
+func (f *fakePutter) PutMetricData(_ context.Context, in *cloudwatch.PutMetricDataInput, _ ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) {
+ f.calls = append(f.calls, in)
+ return &cloudwatch.PutMetricDataOutput{}, nil
+}
+
+// TestMemwatchTick: one tick, one call, built from the readers it is
+// handed. An unreadable meminfo is a skipped tick, not a zero.
+func TestMemwatchTick(t *testing.T) {
+ f := &fakePutter{}
+ w := memwatch{
+ put: f,
+ instanceID: "i-0123",
+ mem: func() (*int64, *int64) { tot, av := int64(4000), int64(1000); return &tot, &av },
+ pressure: func() (string, error) {
+ return "some avg10=0 avg60=7.5 avg300=0 total=0\nfull avg10=0 avg60=0 avg300=0 total=0\n", nil
+ },
+ logf: t.Logf,
+ }
+ w.tick(context.Background(), time.Unix(0, 0))
+ if len(f.calls) != 1 || len(f.calls[0].MetricData) != 2 {
+ t.Fatalf("calls %+v", f.calls)
+ }
+ if v := *f.calls[0].MetricData[0].Value; v != 25 {
+ t.Errorf("MemAvailablePercent = %v, want 25", v)
+ }
+ w.mem = func() (*int64, *int64) { return nil, nil }
+ w.tick(context.Background(), time.Unix(0, 0))
+ if len(f.calls) != 1 {
+ t.Errorf("a tick with unreadable meminfo must not put: %d calls", len(f.calls))
+ }
+}