amadan

eleven / messenger Public

infra+server: prod hibernation bucket + fleet-backup exclusion for hibernating hosts (#550) (#658)

infra/prod/hibernate.tf (applied): the eleven-prod-instances bucket
(versioned, 14d noncurrent, BPA) + scoped activator-prod IAM user — the
router's credentials for parking hibernated instance state.

deploy/fleet-backup.sh: skip any registry host with hibernate=1 — the
activator's embedded Litestream owns those DBs' replication under the
lease; a second replicator on the same WAL would fight it for
checkpointing. Tolerates the column not existing yet.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 123672443a441e8fbeea8695882e3c792cde5a1e parent 99315ef
2 files changed, +95 −1
  • deploy/fleet-backup.sh +17 −1
  • infra/prod/hibernate.tf +78 −0
diff --git a/deploy/fleet-backup.sh b/deploy/fleet-backup.sh
index caf13a8..b7dfa62 100755
--- a/deploy/fleet-backup.sh
+++ b/deploy/fleet-backup.sh
@@ -46,11 +46,27 @@ fi
# 2. The config, regenerated from what's actually in /data. One replica prefix
# per database file; trash and sidecar session dirs are not databases.
shopt -s nullglob
+# Hibernating instances are excluded: the activator's embedded Litestream
+# owns their replication (to the prod-instances bucket, under the lease) —
+# a second replicator on the same WAL would fight it for checkpointing.
+hibernating=""
+if [ -f /data/registry.db ] && command -v sqlite3 >/dev/null; then
+ hibernating=$(sqlite3 /data/registry.db \
+ "SELECT host FROM routes WHERE hibernate = 1" 2>/dev/null || true)
+fi
+skip_hibernating() {
+ local base; base=$(basename "$1" .db)
+ for h in $hibernating; do [ "$h" = "$base" ] && return 0; done
+ return 1
+}
dbs=()
if [ $# -gt 0 ]; then
for name in "$@"; do dbs+=("/data/$name"); done
else
- for f in /data/*.db; do dbs+=("$f"); done
+ for f in /data/*.db; do
+ skip_hibernating "$f" && { echo "skipping $f (hibernating — activator owns its replication)"; continue; }
+ dbs+=("$f")
+ done
fi
[ ${#dbs[@]} -gt 0 ] || { echo "no databases matched" >&2; exit 1; }
diff --git a/infra/prod/hibernate.tf b/infra/prod/hibernate.tf
new file mode 100644
index 0000000..1dfe198
--- /dev/null
+++ b/infra/prod/hibernate.tf
@@ -0,0 +1,78 @@
+# 🤖 Production instance-hibernation state (issue #550 Part 4 v2 on the real
+# fleet): the bucket activator-managed instances park in, and the router
+# box's scoped credentials for it. Separate from the fleet-backup bucket —
+# hibernation's embedded Litestream owns each managed DB's replication
+# (deploy/fleet-backup.sh excludes those hosts; two replicators on one WAL
+# would fight over checkpointing).
+
+resource "aws_s3_bucket" "prod_instances" {
+ bucket = "eleven-prod-instances-${local.account_id}"
+}
+
+resource "aws_s3_bucket_versioning" "prod_instances" {
+ bucket = aws_s3_bucket.prod_instances.id
+
+ versioning_configuration {
+ status = "Enabled"
+ }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "prod_instances" {
+ bucket = aws_s3_bucket.prod_instances.id
+
+ rule {
+ id = "expire-noncurrent-14d"
+ status = "Enabled"
+
+ filter {}
+
+ noncurrent_version_expiration {
+ noncurrent_days = 14
+ }
+ }
+}
+
+resource "aws_s3_bucket_public_access_block" "prod_instances" {
+ bucket = aws_s3_bucket.prod_instances.id
+
+ block_public_acls = true
+ block_public_policy = true
+ ignore_public_acls = true
+ restrict_public_buckets = true
+}
+
+# The router's activator credentials: object rw + version listing (the wake
+# path restores, the lease CASes, sweepers may prune) on this bucket only.
+# Key minted by hand, lives in the router's env — never in tofu state.
+resource "aws_iam_user" "activator_prod" {
+ name = "activator-prod"
+}
+
+data "aws_iam_policy_document" "activator_prod" {
+ statement {
+ sid = "ListProdInstances"
+ actions = ["s3:ListBucket", "s3:ListBucketVersions"]
+ resources = [aws_s3_bucket.prod_instances.arn]
+ }
+
+ statement {
+ sid = "ReadWriteProdInstanceObjects"
+ actions = [
+ "s3:GetObject",
+ "s3:PutObject",
+ "s3:DeleteObject",
+ "s3:DeleteObjectVersion",
+ ]
+ resources = ["${aws_s3_bucket.prod_instances.arn}/*"]
+ }
+}
+
+resource "aws_iam_user_policy" "activator_prod" {
+ name = "activator-prod-instances-bucket"
+ user = aws_iam_user.activator_prod.name
+ policy = data.aws_iam_policy_document.activator_prod.json
+}
+
+output "prod_instances_bucket" {
+ value = aws_s3_bucket.prod_instances.bucket
+}