amadan

carlos / platform Public

region-host: cloudwatch:PutMetricData on Carlos/Host + MemAvailablePercent<15% alarm behind alerts_topic_arn (memory-pressure spec §2.4)

Paul Campbell pushed by paul@keymail.dev ae41882f9322010b1baaab0387f91b73dbcf9182 parent 41260e5
4 files changed, +94 −0
  • infra/modules/README.md +1 −0
  • infra/modules/region-host/.terraform.lock.hcl +39 −0
  • infra/modules/region-host/main.tf +48 −0
  • infra/modules/region-host/variables.tf +6 −0

CI passed — run details

diff --git a/infra/modules/README.md b/infra/modules/README.md
index f0c68eea7..a788ee4fe 100644
--- a/infra/modules/README.md
+++ b/infra/modules/README.md
@@ -70,6 +70,7 @@ bucket, so a wrong-region wake fails loudly at the IAM layer.
| `steering_record_names` | Name bound on that grant, e.g. `["*.oncarlos.com"]`. Empty (default) grants A-record UPSERT/DELETE on the WHOLE zone, apex included — set it. |
| `ami_name_pattern` | AMI name filter (default `debian-13-arm64-*`). |
| `ami_owner` | AMI owner account (default `136693071363`, Debian). |
+| `alerts_topic_arn` | SNS topic for the host's `MemAvailablePercent < 15% for 3 min` alarm (memory-pressure spec §2.4). Empty (default): no alarm. The `cloudwatch:PutMetricData` grant on namespace `Carlos/Host` is unconditional, so the agent's memwatch duty publishes either way. |
| `enable_auto_recovery` | Arm the EC2 auto-recovery alarm (HA ladder rung 0) — a system-status-check failure recovers the instance onto healthy hardware, same instance id, EIP and volumes. Default `true`; set `false` only where an external supervisor owns recovery. |
Outputs: `eip_public_ip` (point your DNS here), `instance_id`,
diff --git a/infra/modules/region-host/.terraform.lock.hcl b/infra/modules/region-host/.terraform.lock.hcl
new file mode 100644
index 000000000..f81693a7c
--- /dev/null
+++ b/infra/modules/region-host/.terraform.lock.hcl
@@ -0,0 +1,39 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/hashicorp/aws" {
+ version = "6.62.0"
+ constraints = ">= 6.0.0"
+ hashes = [
+ "h1:5x/4mMlIqSyeMJQ8tD0A+FTNynpdHW8IA7F2zqrgpwU=",
+ "h1:Etlx1tUvlB7sqFWPciSn9Yz1g50ctM+dGlyVMpu720I=",
+ "h1:FBnaqFZDf3DdVb4SVpCPoT+TZQUSChNmmDViEGPV1rw=",
+ "h1:MRIBAtFWiQAyo4kpbBUtIvxlSmpQ9Eel0nLMYCb6MH4=",
+ "h1:MeMP80kzq1meAJQ0l+kU6KVW1S+4EhIRSy6q3L++LPU=",
+ "h1:N8W8KgcjlG1kb9mPapfieH/vYzyNrJBsb4RS0axwg5Y=",
+ "h1:OB5obEZKuaX4gQ7DYxYvAMaf4I6v2keeKOPfOpFMkgI=",
+ "h1:WnhkO4yQc0QvVgx/xJFI+UbM1y8BC8yvApoDUlhb5XY=",
+ "h1:fN9PvxrT2/rAiT86Hen0vicyMbKXAZG6VC/TEP9ZpD8=",
+ "h1:kRrdLje5ab/tmObt9lOKOi2KCyIB8B3xZCKhcfdS2K0=",
+ "h1:l5VASLhVAOCr2Q+7ywGqWb+JSEIJ5UjilOMBjFMdQ7U=",
+ "h1:mugvZRK3/kSysUmpt664tMNpOZtGbQd6nGKaM4p3kwo=",
+ "h1:nY4ct0BTUQ7se8gbnzXsPCN2tiP4mZsGnEw2juSpXNs=",
+ "h1:qRFHk1ksyfMj2KOrs9pEntzagM+Tzqm+qznJfxVrJj0=",
+ "h1:szVx4GPJr2hJt9t82VilK2Gu/N9WR/KbdeLZ9wb0Nek=",
+ "zh:072d542e40ca0b8c5e081c9f834e3e41aa2ad31c01219522f314685d5c482823",
+ "zh:0b2643473f5bb154d724e64e75632814d74acda5fd81b6488d0ed13f413152b2",
+ "zh:108b3e175886e45c4e955f1ef323aa849dfd82fc450108238be2ab7c0d1f0c2e",
+ "zh:3586ca03a5d07e40a67b01b7fdebc6f7636a42c2a2f6ab88635ffca8f6928dfe",
+ "zh:47bd626153ee94e6b45533c7c02f579dc6d586d6d9580e03a040643f647be50b",
+ "zh:49aafbe6f665b3c4c97522905fd09229f2758a782c7cfc508280607030134ef8",
+ "zh:6e3397227c3f8f3becc04f95ebcb977d7012f025c9921f9ef5e603fe6d5da665",
+ "zh:70f4478658a13eeb57725e3b424329494a0ffbd78ac2734effea4e60d802cfa2",
+ "zh:774d4357447f94a5d4981c7569f85cd140ac577974174c5e7b477d834c6bfb43",
+ "zh:7971ef7bc532ab0edb3ee739ac3010b46321ec8541f873956a20dabbd6a0f189",
+ "zh:8ec18fba906468a9e6ce15329aa098dfb45ab1b9dca446a8fc2079a0d6ef590c",
+ "zh:98328661edd4dfc4e4782fd03a7010854218d3fbf35435837ac33e7ee8aafe7e",
+ "zh:9cbbf5033116f72749ebd4cf9add6aadf7e9e4bc7704eec86a8f4c139c16c120",
+ "zh:9faf944474f9233ddbb3606bc495570d4b952fea21f09c28676f9847d619e50f",
+ "zh:d0636d6de8a7b0b0bcc2add80556ee7118981d738998d4772558b9de0df99af1",
+ ]
+}
diff --git a/infra/modules/region-host/main.tf b/infra/modules/region-host/main.tf
index c145aac8f..d8f94946b 100644
--- a/infra/modules/region-host/main.tf
+++ b/infra/modules/region-host/main.tf
@@ -254,6 +254,28 @@ resource "aws_iam_role_policy" "host_costexplorer" {
})
}
+# The agent's memwatch duty (internal/host/memwatch.go) publishes
+# MemAvailablePercent and MemoryPressureSome60 once a minute. PutMetricData
+# has no resource ARN; the namespace condition is the whole bound — this
+# role can write Carlos/Host and nothing else.
+resource "aws_iam_role_policy" "host_metrics" {
+ name = "${var.iam_name}-metrics"
+ role = aws_iam_role.host.id
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Effect = "Allow"
+ Action = ["cloudwatch:PutMetricData"]
+ Resource = "*"
+ Condition = {
+ StringEquals = { "cloudwatch:namespace" = "Carlos/Host" }
+ }
+ },
+ ]
+ })
+}
+
resource "aws_iam_role_policy_attachment" "host_ssm" {
role = aws_iam_role.host.name
policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
@@ -313,3 +335,29 @@ resource "aws_cloudwatch_metric_alarm" "host_auto_recover" {
}
tags = { Name = "${var.name}-auto-recover" }
}
+
+# Memory-pressure spec §2.4: the early warning the status checks can't give.
+# On 2026-08-28 the flagship thrashed for 15 minutes with EC2 status ok/ok;
+# MemAvailable would have crossed 15% minutes earlier. Missing data is
+# BREACHING: a box that stops reporting is a box in trouble (or an agent
+# rolled without CARLOS_HOST_METRICS — either way, look).
+resource "aws_cloudwatch_metric_alarm" "host_memory_low" {
+ count = var.alerts_topic_arn != "" ? 1 : 0
+ alarm_name = "${var.name}-memory-low"
+ alarm_description = "MemAvailable below 15% for 3 minutes on ${var.name} — a tenant is eating the box; see the memory-pressure spec §4 for what to look at."
+ namespace = "Carlos/Host"
+ metric_name = "MemAvailablePercent"
+ statistic = "Minimum"
+ period = 60
+ evaluation_periods = 3
+ datapoints_to_alarm = 3
+ threshold = 15
+ comparison_operator = "LessThanThreshold"
+ treat_missing_data = "breaching"
+ alarm_actions = [var.alerts_topic_arn]
+ ok_actions = [var.alerts_topic_arn]
+ dimensions = {
+ InstanceId = aws_instance.host.id
+ }
+ tags = { Name = "${var.name}-memory-low" }
+}
diff --git a/infra/modules/region-host/variables.tf b/infra/modules/region-host/variables.tf
index e46144582..78d3dbc2f 100644
--- a/infra/modules/region-host/variables.tf
+++ b/infra/modules/region-host/variables.tf
@@ -108,3 +108,9 @@ variable "enable_cost_explorer" {
type = bool
default = false
}
+
+variable "alerts_topic_arn" {
+ description = "SNS topic that receives this host's memory alarm (memory-pressure spec §2.4). Empty: no alarm is created; the PutMetricData grant is unconditional so the agent's memwatch duty can publish either way."
+ type = string
+ default = ""
+}