| 1 | #!/usr/bin/env bash |
| 2 | # 🤖 Fleet DB backup via Litestream (issue #550: continuous replication, not |
| 3 | # snapshots — closes the "instance DBs have no off-box backup" gap). |
| 4 | # |
| 5 | # Runs ON the router box, as root, idempotent — re-run any time (e.g. after |
| 6 | # add-instance) to pick up new databases; it regenerates the config and |
| 7 | # restarts the replicator. Everything replicated is instance ciphertext plus |
| 8 | # routing metadata the box already holds, so an off-box copy is |
| 9 | # threat-model-free (the E2EE rule). |
| 10 | # |
| 11 | # Prerequisites: |
| 12 | # /data/litestream.env — AWS creds for the scoped litestream-fleet IAM user |
| 13 | # (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY; object-rw on the fleet |
| 14 | # bucket only — minted per infra/prod/backup.tf, key NOT in tofu state). |
| 15 | # |
| 16 | # Usage: |
| 17 | # fleet-backup.sh # replicate every /data/*.db |
| 18 | # fleet-backup.sh test.luan.chat.db # canary: just the named DB file(s) |
| 19 | # |
| 20 | # Verify after first run: |
| 21 | # /opt/lchat/litestream databases -config /etc/litestream.yml |
| 22 | # journalctl -u lchat-litestream -n 20 |
| 23 | # Restore drill (to a scratch path, never over the live file): |
| 24 | # AWS creds in env, then: |
| 25 | # /opt/lchat/litestream restore -config /etc/litestream.yml -o /tmp/drill.db /data/<host>.db |
| 26 | set -euo pipefail |
| 27 | |
| 28 | LITESTREAM_VERSION="0.5.14" |
| 29 | BUCKET="eleven-prod-fleet-backup-427880590669" |
| 30 | REGION="eu-west-1" |
| 31 | BIN=/opt/lchat/litestream |
| 32 | CONF=/etc/litestream.yml |
| 33 | ENVFILE=/data/litestream.env |
| 34 | UNIT=/etc/systemd/system/lchat-litestream.service |
| 35 | |
| 36 | [ -f "$ENVFILE" ] || { echo "missing $ENVFILE (AWS creds for litestream-fleet) — mint per infra/prod/backup.tf" >&2; exit 1; } |
| 37 | |
| 38 | # 1. The binary (skip when the right version is already installed). |
| 39 | if ! "$BIN" version 2>/dev/null | grep -q "$LITESTREAM_VERSION"; then |
| 40 | echo "installing litestream v$LITESTREAM_VERSION" |
| 41 | curl -fsSL "https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-x86_64.tar.gz" \ |
| 42 | | tar -xzO litestream > "$BIN.tmp" |
| 43 | chmod 0755 "$BIN.tmp" && mv "$BIN.tmp" "$BIN" |
| 44 | fi |
| 45 | |
| 46 | # 2. The config, regenerated from what's actually in /data. One replica prefix |
| 47 | # per database file; trash and sidecar session dirs are not databases. |
| 48 | shopt -s nullglob |
| 49 | # Hibernating instances are excluded: the activator's embedded Litestream |
| 50 | # owns their replication (to the prod-instances bucket, under the lease) — |
| 51 | # a second replicator on the same WAL would fight it for checkpointing. |
| 52 | hibernating="" |
| 53 | if [ -f /data/registry.db ] && command -v sqlite3 >/dev/null; then |
| 54 | hibernating=$(sqlite3 /data/registry.db \ |
| 55 | "SELECT host FROM routes WHERE hibernate = 1" 2>/dev/null || true) |
| 56 | fi |
| 57 | skip_hibernating() { |
| 58 | local base; base=$(basename "$1" .db) |
| 59 | for h in $hibernating; do [ "$h" = "$base" ] && return 0; done |
| 60 | return 1 |
| 61 | } |
| 62 | dbs=() |
| 63 | if [ $# -gt 0 ]; then |
| 64 | for name in "$@"; do dbs+=("/data/$name"); done |
| 65 | else |
| 66 | for f in /data/*.db; do |
| 67 | skip_hibernating "$f" && { echo "skipping $f (hibernating — activator owns its replication)"; continue; } |
| 68 | dbs+=("$f") |
| 69 | done |
| 70 | fi |
| 71 | [ ${#dbs[@]} -gt 0 ] || { echo "no databases matched" >&2; exit 1; } |
| 72 | |
| 73 | { |
| 74 | echo "# generated by deploy/fleet-backup.sh — re-run the script, don't edit" |
| 75 | # Cadence: this replicator is a BACKUP, not failover (the activator owns |
| 76 | # failover for hibernating hosts), so every timer below trades recency for |
| 77 | # S3 request volume. Litestream's defaults (sync 1s, L1 compaction 30s, L0 |
| 78 | # retention LISTs every 15s) burned ~113k Tier-1 requests in 14h across the |
| 79 | # fleet with near-zero writes — the retention/compaction monitors LIST per |
| 80 | # database per tick whether or not anything changed. |
| 81 | cat <<'YAML' |
| 82 | # RPO: at most ~60s of a space's ciphertext is lost if the box dies — for a |
| 83 | # backup of family-chat instances that is plenty; idle databases upload |
| 84 | # nothing regardless (an idle sync tick makes no S3 calls). |
| 85 | sync-interval: 60s |
| 86 | # Compaction ladder, ~10x the 30s/5m/1h defaults with the same geometric |
| 87 | # shape (restore read-amplification stays bounded). L1's tick is the request |
| 88 | # driver — each level's monitor costs up to two LISTs per database per tick. |
| 89 | levels: |
| 90 | - interval: 5m |
| 91 | - interval: 30m |
| 92 | - interval: 3h |
| 93 | # Snapshots stay daily — the gentle direction here is LESS often, not the |
| 94 | # hourly one might reach for: each snapshot uploads a full database image. |
| 95 | # Retention 72h keeps ~3 daily snapshots so one bad newest snapshot never |
| 96 | # strands the restore chain. v0.5 enforces snapshot retention on the snapshot |
| 97 | # tick itself (there is no separate retention-check knob), so retention |
| 98 | # sweeps are daily too. |
| 99 | snapshot: |
| 100 | interval: 24h |
| 101 | retention: 72h |
| 102 | # The L0 retention monitor is the worst idle offender: every tick LISTs L1 |
| 103 | # (and usually L0) per database even with zero writes, and its 15s default |
| 104 | # exists for litestream's VFS read replicas, which we don't run. L0 files |
| 105 | # only become reclaimable after an L1 compaction, so hourly sweeps delete |
| 106 | # exactly the same files — just 240x fewer LISTs. l0-retention is the minimum |
| 107 | # age before a compacted L0 file may go; 15m costs pennies of storage and |
| 108 | # keeps recent increments around for point-in-time restores. |
| 109 | l0-retention: 15m |
| 110 | l0-retention-check-interval: 1h |
| 111 | YAML |
| 112 | echo "dbs:" |
| 113 | for f in "${dbs[@]}"; do |
| 114 | [ -f "$f" ] || { echo "warning: $f missing, skipped" >&2; continue; } |
| 115 | printf ' - path: %s\n replica:\n type: s3\n bucket: %s\n path: fleet/%s\n region: %s\n' \ |
| 116 | "$f" "$BUCKET" "$(basename "$f")" "$REGION" |
| 117 | done |
| 118 | } > "$CONF" |
| 119 | chmod 0644 "$CONF" |
| 120 | echo "wrote $CONF (${#dbs[@]} databases)" |
| 121 | |
| 122 | # 3. The unit. Runs as the lchat user (owns /data), same hardening posture as |
| 123 | # the other units; litestream is a pure outbound client. |
| 124 | cat > "$UNIT" <<'EOF' |
| 125 | [Unit] |
| 126 | Description=Litestream fleet DB replication (eleven) |
| 127 | After=network-online.target |
| 128 | Wants=network-online.target |
| 129 | |
| 130 | [Service] |
| 131 | User=lchat |
| 132 | Group=lchat |
| 133 | EnvironmentFile=/data/litestream.env |
| 134 | ExecStart=/opt/lchat/litestream replicate -config /etc/litestream.yml |
| 135 | Restart=always |
| 136 | RestartSec=5 |
| 137 | NoNewPrivileges=true |
| 138 | ProtectSystem=strict |
| 139 | ReadWritePaths=/data |
| 140 | ProtectHome=true |
| 141 | PrivateTmp=true |
| 142 | MemoryMax=512M |
| 143 | |
| 144 | [Install] |
| 145 | WantedBy=multi-user.target |
| 146 | EOF |
| 147 | |
| 148 | systemctl daemon-reload |
| 149 | systemctl enable lchat-litestream |
| 150 | systemctl restart lchat-litestream |
| 151 | systemctl --no-pager --lines=0 status lchat-litestream || true |
| 152 | echo "done — verify with: journalctl -u lchat-litestream -n 20" |
| 153 | |
amadan