| 1 | #!/usr/bin/env bash |
| 2 | # 🤖 Public SSH access ledger — the box-side collector (docs/status.md § The |
| 3 | # SSH access ledger). Every 5 minutes (lchat-ssh-ledger.timer) this reads NEW |
| 4 | # sshd journal events, groups accepted connections into operator sessions, and |
| 5 | # appends closed sessions to the public ledger the status site serves at |
| 6 | # ssh.html. Installed on the box at exactly /opt/lchat/ssh-ledger-collect.sh |
| 7 | # (the unit's ExecStart points there). |
| 8 | # |
| 9 | # ============================ PRIVACY — NON-NEGOTIABLE ======================== |
| 10 | # Everything this script publishes is PUBLIC, forever (the bucket is |
| 11 | # versioned). The rules: |
| 12 | # - METADATA ONLY, NEVER CONTENT. Who (operator handle via public-key |
| 13 | # fingerprint), when (start/end of the accepted-connection window), how |
| 14 | # many connections, and wireguard-or-public source. No commands, no |
| 15 | # command output, no session content — nothing here even reads it. |
| 16 | # - PUBLISH ON CLOSE ONLY. A session row is emitted only after the group has |
| 17 | # been quiet for >= QUIET_S — the public page must never be a live |
| 18 | # "an operator is connected right now" beacon for an attacker to watch. |
| 19 | # - Key FINGERPRINTS are public-key material (safe to publish); client IPs |
| 20 | # are read only to pick the source word and never reach the JSON. |
| 21 | # ============================================================================== |
| 22 | # |
| 23 | # This script is DELIBERATELY STANDALONE from deploy/status-publish.sh (the |
| 24 | # box's copy of that script has diverged from main and must not be forced to |
| 25 | # reconcile for this feature): it keeps its own state, writes its own outputs, |
| 26 | # and uploads them itself with the same status-publisher creds. |
| 27 | # |
| 28 | # The human "why" half lives in the repo: web/status/ssh/*.json entry files, |
| 29 | # PR-reviewed (review is the redaction gate), land in $PAGES_DIR/ssh/ via the |
| 30 | # normal viewer-page scp step. This script merges them into one |
| 31 | # ssh-entries.json so the page fetches exactly two JSON files. |
| 32 | # |
| 33 | # Usage: |
| 34 | # ssh-ledger-collect.sh # what the timer runs |
| 35 | # |
| 36 | # Test mode (no journalctl, no upload, no box state): point every input at a |
| 37 | # fixture — a file of journalctl -o json lines — and pin "now": |
| 38 | # SSH_LEDGER_JOURNAL_FILE=events.jsonl SSH_LEDGER_STATE_DIR=/tmp/state \ |
| 39 | # SSH_LEDGER_PAGES_DIR=/tmp/pages SSH_LEDGER_NOW=<epoch> ./ssh-ledger-collect.sh |
| 40 | # |
| 41 | # Prerequisites on the box: jq (apt install jq) + the aws CLI and |
| 42 | # /data/status.env creds already there for the status publisher. |
| 43 | set -euo pipefail |
| 44 | |
| 45 | STATE_DIR=${SSH_LEDGER_STATE_DIR:-/data/ssh-ledger} |
| 46 | PAGES_DIR=${SSH_LEDGER_PAGES_DIR:-/opt/lchat/status-pages} |
| 47 | SSH_UNIT=${SSH_LEDGER_UNIT:-ssh} # Ubuntu's sshd unit is ssh.service |
| 48 | BUCKET=${STATUS_BUCKET:-eleven-status-427880590669} |
| 49 | REGION=${STATUS_REGION:-eu-west-1} |
| 50 | STATUS_ENV=${STATUS_ENV_FILE:-/data/status.env} |
| 51 | WG_PREFIX="172.22.22." # the wg0 admin subnet (/24) |
| 52 | |
| 53 | # Grouping windows. JOIN_S: reconnects are routine while working (a dropped |
| 54 | # terminal, a second window, scp beside a shell), so accepts within 15 minutes |
| 55 | # of the previous one are one session, not many rows. QUIET_S: a group only |
| 56 | # publishes after 20 quiet minutes — longer than the join window on purpose, |
| 57 | # so a row can never appear while its session could still legally grow, and |
| 58 | # (the publish-on-close rule) the page never advertises a live connection. |
| 59 | JOIN_S=900 |
| 60 | QUIET_S=1200 |
| 61 | |
| 62 | NOW=${SSH_LEDGER_NOW:-$(date -u +%s)} |
| 63 | CURSOR="$STATE_DIR/cursor" |
| 64 | PENDING="$STATE_DIR/pending.tsv" # open groups: user fp start last connections source |
| 65 | SESSIONS="$PAGES_DIR/ssh-sessions.json" |
| 66 | ENTRIES="$PAGES_DIR/ssh-entries.json" |
| 67 | OPERATORS="$PAGES_DIR/ssh-operators.json" |
| 68 | |
| 69 | command -v jq >/dev/null 2>&1 || { echo "jq missing (apt install jq)" >&2; exit 1; } |
| 70 | mkdir -p "$STATE_DIR" "$PAGES_DIR" |
| 71 | |
| 72 | TMP=$(mktemp -d) |
| 73 | trap 'rm -rf "$TMP"' EXIT |
| 74 | |
| 75 | # iso <epoch> — GNU date on the box; the -r fallback keeps the test mode |
| 76 | # runnable on a BSD/macOS dev machine. |
| 77 | iso() { date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -r "$1" +%Y-%m-%dT%H:%M:%SZ; } |
| 78 | |
| 79 | # atomic_write <src> <dest> — temp beside the target + mv, so a reader (or a |
| 80 | # mid-write upload) never sees a half-written JSON. Same-directory temp keeps |
| 81 | # the rename atomic (never across filesystems). |
| 82 | atomic_write() { cp "$1" "$2.tmp.$$" && mv "$2.tmp.$$" "$2"; } |
| 83 | |
| 84 | # ---- 1. read new sshd events ------------------------------------------------- |
| 85 | # journalctl's --cursor-file persists our read position in $STATE_DIR (not the |
| 86 | # journal itself), so it survives journald rotation: rotation drops old |
| 87 | # entries, never the cursor, and the next run resumes exactly where this one |
| 88 | # stopped. First run (no cursor yet) backfills from the start of the journal. |
| 89 | if [ -n "${SSH_LEDGER_JOURNAL_FILE:-}" ]; then |
| 90 | cat "$SSH_LEDGER_JOURNAL_FILE" > "$TMP/journal.jsonl" |
| 91 | else |
| 92 | journalctl -u "$SSH_UNIT" -o json --cursor-file "$CURSOR" --no-pager > "$TMP/journal.jsonl" || true |
| 93 | fi |
| 94 | |
| 95 | # One TSV line per accepted-publickey login: epoch-seconds, user, client IP, |
| 96 | # key fingerprint. Password/other auth is off on this box, so "Accepted |
| 97 | # publickey" is the complete accept vocabulary; anything else in the sshd |
| 98 | # journal (disconnects, probes, scanner noise) is deliberately ignored. |
| 99 | jq -r 'select((.MESSAGE // "") | startswith("Accepted publickey for ")) |
| 100 | | [(.__REALTIME_TIMESTAMP | tonumber / 1000000 | floor), .MESSAGE] | @tsv' \ |
| 101 | "$TMP/journal.jsonl" > "$TMP/accepts.tsv" |
| 102 | |
| 103 | # ---- 2. fold accepts into pending groups ------------------------------------- |
| 104 | # pending.tsv holds every not-yet-published group. More than one group per |
| 105 | # (user, fingerprint) can coexist briefly: a reconnect 16 minutes after the |
| 106 | # last (past JOIN_S) starts a new group while the old one is still inside its |
| 107 | # QUIET_S wait — the old row publishes a few minutes later on its own clock. |
| 108 | [ -f "$PENDING" ] || : > "$PENDING" |
| 109 | cp "$PENDING" "$TMP/pending.tsv" |
| 110 | |
| 111 | msg_re='^Accepted publickey for ([^ ]+) from ([0-9a-fA-F:.]+) port [0-9]+ ssh2: [^ ]+ (SHA256:[A-Za-z0-9+/=]+)' |
| 112 | while IFS=$'\t' read -r ts msg; do |
| 113 | [[ "$msg" =~ $msg_re ]] || continue |
| 114 | user="${BASH_REMATCH[1]}" ip="${BASH_REMATCH[2]}" fp="${BASH_REMATCH[3]}" |
| 115 | # Source word only — the IP itself dies here (see PRIVACY above). A public |
| 116 | # accept should be rare now port 22 is wireguard-only, which is exactly why |
| 117 | # it must publish loudly when it happens. |
| 118 | case "$ip" in "$WG_PREFIX"*) src=wireguard ;; *) src=public ;; esac |
| 119 | # Join the newest open group for this (user, fp) if this accept is within |
| 120 | # JOIN_S of its last connection; otherwise open a new group. awk rewrites |
| 121 | # the whole (tiny) pending file per event — clarity over cleverness. |
| 122 | awk -v u="$user" -v f="$fp" -v ts="$ts" -v src="$src" -v join="$JOIN_S" ' |
| 123 | BEGIN { OFS="\t"; joined=0 } |
| 124 | $1==u && $2==f && !joined && ts-$4 <= join && ts >= $4 { |
| 125 | $4=ts; $5=$5+1 |
| 126 | if (src=="public") $6="public" # one public hop marks the whole group |
| 127 | joined=1 |
| 128 | } |
| 129 | { print } |
| 130 | END { if (!joined) print u, f, ts, ts, 1, src } |
| 131 | ' "$TMP/pending.tsv" > "$TMP/pending.next" && mv "$TMP/pending.next" "$TMP/pending.tsv" |
| 132 | done < "$TMP/accepts.tsv" |
| 133 | |
| 134 | # ---- 3. publish groups that have gone quiet ---------------------------------- |
| 135 | # Split pending into "still open" (kept for next run) and "quiet >= QUIET_S" |
| 136 | # (emitted below). This is the publish-on-close rule from the header. |
| 137 | awk -v now="$NOW" -v quiet="$QUIET_S" 'now - $4 >= quiet' "$TMP/pending.tsv" > "$TMP/closed.tsv" |
| 138 | awk -v now="$NOW" -v quiet="$QUIET_S" 'now - $4 < quiet' "$TMP/pending.tsv" > "$TMP/open.tsv" |
| 139 | |
| 140 | # Operator attribution by fingerprint — web/status/ssh-operators.json, staged |
| 141 | # with the viewer pages. An unmapped fingerprint publishes as "unknown key": |
| 142 | # render-worthy on purpose (an unexplained key on the box is itself a signal), |
| 143 | # never silently dropped. |
| 144 | [ -f "$OPERATORS" ] || echo '{}' > "$OPERATORS" |
| 145 | |
| 146 | echo '[]' > "$TMP/new.json" |
| 147 | while IFS=$'\t' read -r user fp start last conns src; do |
| 148 | [ -n "$user" ] || continue |
| 149 | jq --arg id "$user+$fp+$(iso "$start")" \ |
| 150 | --arg fp "$fp" --arg start "$(iso "$start")" --arg end "$(iso "$last")" \ |
| 151 | --argjson conns "$conns" --arg src "$src" \ |
| 152 | --slurpfile ops "$OPERATORS" \ |
| 153 | '. + [{id: $id, operator: ($ops[0][$fp] // "unknown key"), |
| 154 | fingerprint: $fp, start: $start, end: $end, |
| 155 | connections: $conns, source: $src}]' \ |
| 156 | "$TMP/new.json" > "$TMP/new.next" && mv "$TMP/new.next" "$TMP/new.json" |
| 157 | done < "$TMP/closed.tsv" |
| 158 | |
| 159 | # Append to the cumulative ledger, deduped on id (user+fingerprint+start), so |
| 160 | # a re-run over replayed journal lines can never double-publish a session. A |
| 161 | # missing ledger starts fresh; a CORRUPT one aborts the run — never clobber |
| 162 | # published history with an empty file. |
| 163 | if [ -f "$SESSIONS" ]; then |
| 164 | jq -e . "$SESSIONS" >/dev/null || { echo "corrupt $SESSIONS — refusing to overwrite" >&2; exit 1; } |
| 165 | else |
| 166 | echo '{"sessions": []}' > "$SESSIONS" |
| 167 | fi |
| 168 | jq --arg now "$(iso "$NOW")" --slurpfile add "$TMP/new.json" \ |
| 169 | '{generated: $now, sessions: ((.sessions // []) + $add[0] | unique_by(.id) | sort_by(.start))}' \ |
| 170 | "$SESSIONS" > "$TMP/sessions.json" |
| 171 | atomic_write "$TMP/sessions.json" "$SESSIONS" |
| 172 | atomic_write "$TMP/open.tsv" "$PENDING" |
| 173 | |
| 174 | # ---- 4. merge the human why-entries ------------------------------------------ |
| 175 | # web/status/ssh/*.json (one file per entry — parallel PRs never conflict) |
| 176 | # merge into one array so the page fetches exactly two JSONs. A malformed |
| 177 | # entry fails the run loudly rather than silently dropping someone's "why". |
| 178 | echo '[]' > "$TMP/entries.json" |
| 179 | for f in "$PAGES_DIR"/ssh/*.json; do |
| 180 | [ -e "$f" ] || continue # no entries staged yet — publish an empty array |
| 181 | jq -e . "$f" >/dev/null 2>&1 || { echo "malformed why-entry $f — fix or revert it" >&2; exit 1; } |
| 182 | jq --slurpfile e "$f" '. + $e' "$TMP/entries.json" > "$TMP/entries.next" && \ |
| 183 | mv "$TMP/entries.next" "$TMP/entries.json" |
| 184 | done |
| 185 | jq 'sort_by(.date) | reverse' "$TMP/entries.json" > "$TMP/entries.sorted" |
| 186 | atomic_write "$TMP/entries.sorted" "$ENTRIES" |
| 187 | |
| 188 | # ---- 5. upload ---------------------------------------------------------------- |
| 189 | # Standalone by design (see header): this script uploads its own outputs with |
| 190 | # the same scoped status-publisher creds status-publish.sh uses. Test mode |
| 191 | # (fixture journal) never uploads. Re-uploading the page + operator map every |
| 192 | # run self-heals a missed copy, same as the main publisher. |
| 193 | if [ -z "${SSH_LEDGER_JOURNAL_FILE:-}" ]; then |
| 194 | command -v aws >/dev/null 2>&1 || { echo "aws CLI missing — collected, not uploaded" >&2; exit 1; } |
| 195 | if [ -z "${AWS_ACCESS_KEY_ID:-}" ] && [ -f "$STATUS_ENV" ]; then |
| 196 | set -a; . "$STATUS_ENV"; set +a |
| 197 | fi |
| 198 | up() { aws s3 cp "$1" "s3://$BUCKET/$2" --region "$REGION" \ |
| 199 | --content-type "$3" --cache-control "$4" --no-progress >/dev/null; } |
| 200 | up "$SESSIONS" ssh-sessions.json "application/json" "max-age=60" |
| 201 | up "$ENTRIES" ssh-entries.json "application/json" "max-age=60" |
| 202 | [ -f "$PAGES_DIR/ssh.html" ] && up "$PAGES_DIR/ssh.html" ssh.html "text/html; charset=utf-8" "max-age=300" |
| 203 | [ -f "$OPERATORS" ] && up "$OPERATORS" ssh-operators.json "application/json" "max-age=300" |
| 204 | echo "ssh ledger published to s3://$BUCKET ($(iso "$NOW"))" |
| 205 | fi |
| 206 | |
amadan