amadan

eleven / ocho Public

infra: prod hardening + fleet-backup staging (#550) (#637)

infra/prod (applied): CloudTrail (all-region, log validation, 90d retention)
and the fleet-backup bucket — versioned, 14d noncurrent retention, Block
Public Access — with a litestream-fleet IAM user scoped to object-rw on that
one bucket. Separate tofu state from dev/ so a dev teardown can never touch
prod backups.

deploy/fleet-backup.sh: idempotent installer for the router box — downloads
litestream, regenerates /etc/litestream.yml from /data/*.db, installs the
lchat-litestream unit (lchat user, sandboxed). Canary-first via args.
Expects /data/litestream.env (creds minted outside tofu state).
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell c3bb167b8a8a46a1e62926a8ec18229cd72e2acf parent 20257d7
6 files changed, +338 −0
  • deploy/fleet-backup.sh +98 −0
  • infra/prod/.terraform.lock.hcl +39 −0
  • infra/prod/backup.tf +74 −0
  • infra/prod/cloudtrail.tf +77 −0
  • infra/prod/main.tf +41 −0
  • infra/prod/outputs.tf +9 −0
diff --git a/deploy/fleet-backup.sh b/deploy/fleet-backup.sh
new file mode 100755
index 0000000..fefadff
--- /dev/null
+++ b/deploy/fleet-backup.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# 🤖 Fleet DB backup via Litestream (issue #550: continuous replication, not
+# snapshots — closes the "instance DBs have no off-box backup" gap).
+#
+# Runs ON the router box, as root, idempotent — re-run any time (e.g. after
+# add-instance) to pick up new databases; it regenerates the config and
+# restarts the replicator. Everything replicated is instance ciphertext plus
+# routing metadata the box already holds, so an off-box copy is
+# threat-model-free (the E2EE rule).
+#
+# Prerequisites:
+# /data/litestream.env — AWS creds for the scoped litestream-fleet IAM user
+# (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY; object-rw on the fleet
+# bucket only — minted per infra/prod/backup.tf, key NOT in tofu state).
+#
+# Usage:
+# fleet-backup.sh # replicate every /data/*.db
+# fleet-backup.sh test.luan.chat.db # canary: just the named DB file(s)
+#
+# Verify after first run:
+# /opt/lchat/litestream databases -config /etc/litestream.yml
+# journalctl -u lchat-litestream -n 20
+# Restore drill (to a scratch path, never over the live file):
+# AWS creds in env, then:
+# /opt/lchat/litestream restore -config /etc/litestream.yml -o /tmp/drill.db /data/<host>.db
+set -euo pipefail
+
+LITESTREAM_VERSION="0.5.14"
+BUCKET="eleven-prod-fleet-backup-427880590669"
+REGION="eu-west-1"
+BIN=/opt/lchat/litestream
+CONF=/etc/litestream.yml
+ENVFILE=/data/litestream.env
+UNIT=/etc/systemd/system/lchat-litestream.service
+
+[ -f "$ENVFILE" ] || { echo "missing $ENVFILE (AWS creds for litestream-fleet) — mint per infra/prod/backup.tf" >&2; exit 1; }
+
+# 1. The binary (skip when the right version is already installed).
+if ! "$BIN" version 2>/dev/null | grep -q "$LITESTREAM_VERSION"; then
+ echo "installing litestream v$LITESTREAM_VERSION"
+ curl -fsSL "https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-amd64.tar.gz" \
+ | tar -xzO litestream > "$BIN.tmp"
+ chmod 0755 "$BIN.tmp" && mv "$BIN.tmp" "$BIN"
+fi
+
+# 2. The config, regenerated from what's actually in /data. One replica prefix
+# per database file; trash and sidecar session dirs are not databases.
+shopt -s nullglob
+dbs=()
+if [ $# -gt 0 ]; then
+ for name in "$@"; do dbs+=("/data/$name"); done
+else
+ for f in /data/*.db; do dbs+=("$f"); done
+fi
+[ ${#dbs[@]} -gt 0 ] || { echo "no databases matched" >&2; exit 1; }
+
+{
+ echo "# generated by deploy/fleet-backup.sh — re-run the script, don't edit"
+ echo "dbs:"
+ for f in "${dbs[@]}"; do
+ [ -f "$f" ] || { echo "warning: $f missing, skipped" >&2; continue; }
+ printf ' - path: %s\n replica:\n type: s3\n bucket: %s\n path: fleet/%s\n region: %s\n' \
+ "$f" "$BUCKET" "$(basename "$f")" "$REGION"
+ done
+} > "$CONF"
+chmod 0644 "$CONF"
+echo "wrote $CONF (${#dbs[@]} databases)"
+
+# 3. The unit. Runs as the lchat user (owns /data), same hardening posture as
+# the other units; litestream is a pure outbound client.
+cat > "$UNIT" <<'EOF'
+[Unit]
+Description=Litestream fleet DB replication (eleven)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+User=lchat
+Group=lchat
+EnvironmentFile=/data/litestream.env
+ExecStart=/opt/lchat/litestream replicate -config /etc/litestream.yml
+Restart=always
+RestartSec=5
+NoNewPrivileges=true
+ProtectSystem=strict
+ReadWritePaths=/data
+ProtectHome=true
+PrivateTmp=true
+MemoryMax=512M
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable --now lchat-litestream
+systemctl --no-pager --lines=0 status lchat-litestream || true
+echo "done — verify with: journalctl -u lchat-litestream -n 20"
diff --git a/infra/prod/.terraform.lock.hcl b/infra/prod/.terraform.lock.hcl
new file mode 100644
index 0000000..9d87350
--- /dev/null
+++ b/infra/prod/.terraform.lock.hcl
@@ -0,0 +1,39 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/hashicorp/aws" {
+ version = "6.54.0"
+ constraints = "~> 6.0"
+ hashes = [
+ "h1:+fuoXxQEOoT6LtTlxiz+2JQz29pOev6T3f+EnrKJQdU=",
+ "h1:/YoIrgxHWluPcxV7VkPDQXGUTWGLHyWrFZ0p4YQtD/k=",
+ "h1:2iBfjJWoE/mLz4Nlk3k3B01jmmrZ3nt6jUjcOBfdsIg=",
+ "h1:CGF4EHrHHvRlA4zDygUK3bEvleYHVzVO8B/77MpXN2U=",
+ "h1:HA9VzSihsNsOH9/43H3WRQ2B6ra+pWrJpntHrePtLro=",
+ "h1:KbJ2hmW/5X2UUbtkx5+NkSbqlypbQz9heJO9kJZuHVs=",
+ "h1:P1B4qs1A7J5fsd4nWIvNxSd95/hMOluCNZIgOUR7jdY=",
+ "h1:TgNW1ixmV3WyGXOnsX0ojjqjqk7kvLPiPOrwMHSPj9g=",
+ "h1:U0h5Zv8SDKQqWRG2PzOutGwynnNeotoc/Qi790KhlX8=",
+ "h1:dR1gLh4Jb58GSGTaTFJ5cRBmWVDILybvTqgsJ1FMPxQ=",
+ "h1:hqzkTh+nLtxUcXVbPspshZZ6jKD0Kv1ULWpc+LovkMQ=",
+ "h1:nXG/JZ187e+FJsHcgdQEpqEA0nGZCysU6+i0Rv2OLK0=",
+ "h1:nhC7R1QTAbieBYvXYnME2qVlzUTQenswXzRhMGthKw4=",
+ "h1:pJuq/1c7MVniTzmZz9+/iroIxv1C9c6NfZD4qf1uvvM=",
+ "h1:q5ygK0xlXRNjazU/a66Q72ijPmpFWOAtquYUIYEAFXo=",
+ "zh:03efcbe4eb2b94e8e1dc10e3fcc35c3dcc2c5ed03e8a4f14b027aa26d2b391d8",
+ "zh:081e2ccd4719f894ef2577c65949e4944a11ceb45ec55413db5482232c1d6dc3",
+ "zh:276572fc0da8b7e99680626d0399f9c1845b71ea66df1e30f4c4cabeee3220da",
+ "zh:67a0703538c6ea969a8436d3740866e31acadff3b3ff48d24941295ac6b1dbf4",
+ "zh:81e9e9c2e33366a8b0085e79542911a289e5e00df347f6a22960d5f54aa1d6b7",
+ "zh:8df21a1e549f49f18127bc997df4da596cf69b6f6d75b5f1a5ffdf351a01037d",
+ "zh:8e9d5c5c892634b7ab8a10a9a47e581662f976d17b49a2292d51b17e2260c5c0",
+ "zh:9b58601ba5f64c8594b0f05da934bf58d6564eb800bff86ce746733ebaa3ab36",
+ "zh:a5ffd086c56b5812ccec7632cd4053bcb4b0552790c99a9cf4b12dea24390ba8",
+ "zh:af87d40420bdd1c935c7ccd729b31f97f819f8576f9897b57cf6356a4fbda880",
+ "zh:d09bd4c2e6d0a8cfc2699a25a535d07eb4b31d8a71f912b6943ab978e3c8b37c",
+ "zh:d832d91ab28d0fea6dc7ac3fd313f75cc6ab7d90d63cdcf7ba649f6e70898b15",
+ "zh:e9d81182bb0bf5b8c13a0eb6029597ec2fc0d56914eea564004f6ae84529ed52",
+ "zh:eb8fe7a194c4f9b3d86366371773631052667e22136424ce725517775ef5b6fe",
+ "zh:f0b9c91fe2708200c198405ca758c100ab06ca0b4e39db5cabbaa739b6231a16",
+ ]
+}
diff --git a/infra/prod/backup.tf b/infra/prod/backup.tf
new file mode 100644
index 0000000..b590f8d
--- /dev/null
+++ b/infra/prod/backup.tf
@@ -0,0 +1,74 @@
+# 🤖 The fleet-backup bucket: Litestream on the router box streams every
+# instance DB (plus home/push/cloud/registry) here, one prefix per database.
+# Versioning is the ransomware posture — an overwrite or delete with the
+# box's credentials leaves noncurrent versions recoverable for 14 days, and
+# the box's IAM user (below) cannot touch versioning, lifecycle, or the
+# bucket itself.
+
+resource "aws_s3_bucket" "fleet_backup" {
+ bucket = "eleven-prod-fleet-backup-${local.account_id}"
+}
+
+resource "aws_s3_bucket_versioning" "fleet_backup" {
+ bucket = aws_s3_bucket.fleet_backup.id
+
+ versioning_configuration {
+ status = "Enabled"
+ }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "fleet_backup" {
+ bucket = aws_s3_bucket.fleet_backup.id
+
+ rule {
+ id = "expire-noncurrent-14d"
+ status = "Enabled"
+
+ filter {}
+
+ noncurrent_version_expiration {
+ noncurrent_days = 14
+ }
+ }
+}
+
+resource "aws_s3_bucket_public_access_block" "fleet_backup" {
+ bucket = aws_s3_bucket.fleet_backup.id
+
+ block_public_acls = true
+ block_public_policy = true
+ ignore_public_acls = true
+ restrict_public_buckets = true
+}
+
+# The router box's credentials: object read/write/delete + list on the fleet
+# bucket, nothing else in the account. Mint the access key by hand
+# (aws iam create-access-key --user-name litestream-fleet) — keys never land
+# in tofu state.
+resource "aws_iam_user" "litestream_fleet" {
+ name = "litestream-fleet"
+}
+
+data "aws_iam_policy_document" "litestream_fleet" {
+ statement {
+ sid = "ListFleetBucket"
+ actions = ["s3:ListBucket"]
+ resources = [aws_s3_bucket.fleet_backup.arn]
+ }
+
+ statement {
+ sid = "ReadWriteFleetObjects"
+ actions = [
+ "s3:GetObject",
+ "s3:PutObject",
+ "s3:DeleteObject",
+ ]
+ resources = ["${aws_s3_bucket.fleet_backup.arn}/*"]
+ }
+}
+
+resource "aws_iam_user_policy" "litestream_fleet" {
+ name = "litestream-fleet-bucket"
+ user = aws_iam_user.litestream_fleet.name
+ policy = data.aws_iam_policy_document.litestream_fleet.json
+}
diff --git a/infra/prod/cloudtrail.tf b/infra/prod/cloudtrail.tf
new file mode 100644
index 0000000..e03dead
--- /dev/null
+++ b/infra/prod/cloudtrail.tf
@@ -0,0 +1,77 @@
+# 🤖 Account hardening: an all-region CloudTrail with log-file validation,
+# writing to its own locked-down bucket (90-day retention — enough to
+# investigate, not an unbounded cost). Management events only, which keeps
+# the trail in the free tier.
+
+resource "aws_s3_bucket" "trail_logs" {
+ bucket = "eleven-cloudtrail-${local.account_id}"
+}
+
+resource "aws_s3_bucket_public_access_block" "trail_logs" {
+ bucket = aws_s3_bucket.trail_logs.id
+
+ block_public_acls = true
+ block_public_policy = true
+ ignore_public_acls = true
+ restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "trail_logs" {
+ bucket = aws_s3_bucket.trail_logs.id
+
+ rule {
+ id = "expire-90d"
+ status = "Enabled"
+
+ filter {}
+
+ expiration {
+ days = 90
+ }
+ }
+}
+
+data "aws_iam_policy_document" "trail_logs" {
+ statement {
+ sid = "CloudTrailAclCheck"
+ actions = ["s3:GetBucketAcl"]
+ resources = [aws_s3_bucket.trail_logs.arn]
+
+ principals {
+ type = "Service"
+ identifiers = ["cloudtrail.amazonaws.com"]
+ }
+ }
+
+ statement {
+ sid = "CloudTrailWrite"
+ actions = ["s3:PutObject"]
+ resources = ["${aws_s3_bucket.trail_logs.arn}/AWSLogs/${local.account_id}/*"]
+
+ principals {
+ type = "Service"
+ identifiers = ["cloudtrail.amazonaws.com"]
+ }
+
+ condition {
+ test = "StringEquals"
+ variable = "s3:x-amz-acl"
+ values = ["bucket-owner-full-control"]
+ }
+ }
+}
+
+resource "aws_s3_bucket_policy" "trail_logs" {
+ bucket = aws_s3_bucket.trail_logs.id
+ policy = data.aws_iam_policy_document.trail_logs.json
+}
+
+resource "aws_cloudtrail" "main" {
+ name = "eleven-account-trail"
+ s3_bucket_name = aws_s3_bucket.trail_logs.id
+ is_multi_region_trail = true
+ include_global_service_events = true
+ enable_log_file_validation = true
+
+ depends_on = [aws_s3_bucket_policy.trail_logs]
+}
diff --git a/infra/prod/main.tf b/infra/prod/main.tf
new file mode 100644
index 0000000..dca4fef
--- /dev/null
+++ b/infra/prod/main.tf
@@ -0,0 +1,41 @@
+# 🤖 Production-facing resources in the AWS account: the fleet-backup bucket
+# the router box replicates instance DBs into (all ciphertext — the E2EE rule
+# is what makes off-box replication threat-model-free), plus account
+# hardening (CloudTrail). Separate state from dev/ so a dev teardown can
+# never touch prod backups.
+
+terraform {
+ required_version = ">= 1.10"
+
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 6.0"
+ }
+ }
+
+ backend "s3" {
+ bucket = "eleven-tofu-state-427880590669"
+ key = "prod/terraform.tfstate"
+ region = "eu-west-1"
+ use_lockfile = true
+ }
+}
+
+provider "aws" {
+ region = "eu-west-1"
+
+ default_tags {
+ tags = {
+ project = "eleven"
+ env = "prod"
+ managed = "opentofu"
+ }
+ }
+}
+
+data "aws_caller_identity" "current" {}
+
+locals {
+ account_id = data.aws_caller_identity.current.account_id
+}
diff --git a/infra/prod/outputs.tf b/infra/prod/outputs.tf
new file mode 100644
index 0000000..c69563c
--- /dev/null
+++ b/infra/prod/outputs.tf
@@ -0,0 +1,9 @@
+# 🤖
+
+output "fleet_backup_bucket" {
+ value = aws_s3_bucket.fleet_backup.bucket
+}
+
+output "cloudtrail_bucket" {
+ value = aws_s3_bucket.trail_logs.bucket
+}