| 1 | #!/usr/bin/env bash |
| 2 | # 🤖 Public fleet hardening posture — the box-side collector (docs/status.md § |
| 3 | # The fleet hardening dashboard). Every 5 minutes (lchat-hardening.timer) this |
| 4 | # publishes, per box in the mesh, whether SSH is locked down: the LOCAL box |
| 5 | # reads its own OS state authoritatively (ufw / sshd / systemd / wireguard), |
| 6 | # and each PEER box is probed from the outside — a connect-and-banner-grab :22 |
| 7 | # check, never a login. It writes hardening.json and uploads it + hardening.html |
| 8 | # to the public status bucket the site serves at hardening.html. Installed on |
| 9 | # the box at exactly /opt/lchat/hardening-posture.sh (the unit's ExecStart |
| 10 | # points there). |
| 11 | # |
| 12 | # ============================ WHAT THIS PUBLISHES ============================ |
| 13 | # Everything here is PUBLIC, forever (the bucket is versioned). Two design |
| 14 | # points make that safe, and one is the whole reason the page exists: |
| 15 | # - NO HOSTNAMES. A box is named by its whimsical, hash-derived label |
| 16 | # ("hypatia-whistling", the SAME label fleet.json shows) and a coarse region |
| 17 | # ("EU · Finland"). A peer's real hostname is read only to point the :22 |
| 18 | # probe at it and dies in this process — it never reaches the JSON, matching |
| 19 | # the status site's no-hostnames rule (deploy/status-publish.sh's header). |
| 20 | # - UNKNOWN IS LOUD, NEVER A GUESSED PASS. Any signal the collector can't read |
| 21 | # is emitted as "unknown" (a null ssh_internet_facing, or an "unknown" |
| 22 | # sub-state) and the page renders it as a warn state — never silently as |
| 23 | # hardened. An unreadable firewall must look unresolved, not safe. |
| 24 | # - THE POINT: an unhardened box surfaces publicly. If a mesh box is serving |
| 25 | # SSH to the internet, its :22 probe reads back an "SSH-…" banner and it |
| 26 | # sorts to the TOP of the page. Transparency, not a security control. |
| 27 | # |
| 28 | # ---------------------- The peer :22 probe is VANTAGE-DEPENDENT -------------- |
| 29 | # An external probe sees the fleet from THIS box's viewpoint only: a peer's |
| 30 | # firewall might allow this box but deny everyone else, or vice-versa. So the |
| 31 | # probe is classified conservatively — only a returned SSH banner is a CONFIDENT |
| 32 | # "exposed": |
| 33 | # - reads back "SSH-…" -> exposed (true) [confident BAD] |
| 34 | # - clean refusal / connect times out -> not reachable (false) [good] |
| 35 | # - connects but no SSH banner, or anything that isn't a clean refusal |
| 36 | # -> inconclusive (null) [UNKNOWN — loud warn, |
| 37 | # neither a pass nor a confirmed fail] |
| 38 | # The AUTHORITATIVE signal is a box self-reading its own ufw once it runs this |
| 39 | # collector; the probe is a best-effort outside check that makes a blatantly |
| 40 | # open box impossible to miss. The page says as much in its caveat. |
| 41 | # ============================================================================ |
| 42 | # |
| 43 | # This script is DELIBERATELY STANDALONE from deploy/status-publish.sh (the |
| 44 | # box's copy of that script has diverged from main and must not be forced to |
| 45 | # reconcile for this feature): it keeps its own state, writes its own outputs, |
| 46 | # and uploads them itself with the same status-publisher creds. It reads the |
| 47 | # mesh membership from the hibernation bucket exactly as status-publish.sh's |
| 48 | # mesh_peer_boxes does, so the two never need to share code. |
| 49 | # |
| 50 | # Usage: |
| 51 | # hardening-posture.sh # what the timer runs |
| 52 | # |
| 53 | # Test mode (no ufw/sshd/systemd/ip, no S3, no :22 probe, no upload): point |
| 54 | # every input at a fixture and pin "now". The command seams eval a shell command |
| 55 | # (so a fixture is just `cat file` printing pre-captured tool output): |
| 56 | # HARDENING_UFW_CMD='cat ufw.txt' # replaces `ufw status` |
| 57 | # HARDENING_SSHD_CMD='cat sshd-T.txt' # replaces `sshd -T` |
| 58 | # HARDENING_SYSTEMCTL_CMD='cat au.txt' # replaces `systemctl is-active unattended-upgrades` |
| 59 | # HARDENING_IP_CMD='cat wg0.txt' # replaces `ip link show wg0` |
| 60 | # HARDENING_ROUTER_ENV=router.env # LCHAT_MESH_SELF / LCHAT_BOX_REGION seams |
| 61 | # HARDENING_MEMBERS_FILE=members.tsv # replaces the S3 listing + stats + :22 probe |
| 62 | # HARDENING_NOW=<epoch> HARDENING_NO_UPLOAD=1 \ |
| 63 | # HARDENING_STATE_DIR=/tmp/state HARDENING_PAGES_DIR=/tmp/pages ./hardening-posture.sh |
| 64 | # |
| 65 | # HARDENING_MEMBERS_FILE is a TSV, one mesh member per line, four columns: |
| 66 | # <peerURL> <TAB> <label> <TAB> <region> <TAB> <probe> |
| 67 | # where <probe> is the peer's :22 verdict — serving-ssh | not-reachable | |
| 68 | # inconclusive (aliases reachable | filtered-or-closed | unknown are accepted). |
| 69 | # It is consulted only for PEERS; the row whose <peerURL> equals LCHAT_MESH_SELF |
| 70 | # is the local box and uses the LOCAL OS reads instead. Setting |
| 71 | # HARDENING_MEMBERS_FILE also forces no-upload, so a fixture run is always inert. |
| 72 | # |
| 73 | # Prerequisites on the box: jq (apt install jq) + the aws CLI and |
| 74 | # /data/status.env creds already there for the status publisher. The creds must |
| 75 | # also be able to READ the hibernation bucket (LCHAT_HIBERNATE_BUCKET) — the |
| 76 | # same read status-publish.sh's fleet page already relies on. |
| 77 | set -euo pipefail |
| 78 | |
| 79 | STATE_DIR=${HARDENING_STATE_DIR:-/data/hardening} |
| 80 | PAGES_DIR=${HARDENING_PAGES_DIR:-/opt/lchat/status-pages} |
| 81 | # The PUBLIC status bucket — upload target for hardening.json + hardening.html, |
| 82 | # the same bucket (and creds) status-publish.sh and ssh-ledger-collect.sh use. |
| 83 | BUCKET=${STATUS_BUCKET:-eleven-status-427880590669} |
| 84 | REGION=${STATUS_REGION:-eu-west-1} |
| 85 | STATUS_ENV=${STATUS_ENV_FILE:-/data/status.env} |
| 86 | # The router's own env — read (same file the router loads) only for the mesh's |
| 87 | # bucket/prefix/region and this box's own identity/region. Absent |
| 88 | # LCHAT_HIBERNATE_BUCKET just means the mesh isn't configured here: the page |
| 89 | # then shows this one box, byte-consistent with a single-box fleet. |
| 90 | ROUTER_ENV=${HARDENING_ROUTER_ENV:-/data/router.env} |
| 91 | # Staleness cutoff for a membership object — three missed beats, mirroring |
| 92 | # mesh.go's meshStaleAfter (3 × meshHeartbeatInterval, 60s). A box whose |
| 93 | # heartbeat has aged past this is presumed dead and dropped, so the page never |
| 94 | # lists a long-gone box (its S3 membership key lingers until lifecycle prunes |
| 95 | # it, but its LastModified stops advancing). |
| 96 | STALE_SECONDS=${HARDENING_MESH_STALE_SECONDS:-180} |
| 97 | # Probe budget: PROBE_TIMEOUT bounds the whole connect+banner-read; BANNER_READ |
| 98 | # is how long we wait for the SSH banner once connected (kept comfortably under |
| 99 | # PROBE_TIMEOUT so a fast connect always leaves room to read). |
| 100 | PROBE_TIMEOUT=${HARDENING_PROBE_TIMEOUT:-6} |
| 101 | BANNER_READ=${HARDENING_BANNER_READ:-5} |
| 102 | |
| 103 | OUT="$PAGES_DIR/hardening.json" |
| 104 | |
| 105 | command -v jq >/dev/null 2>&1 || { echo "jq missing (apt install jq)" >&2; exit 1; } |
| 106 | mkdir -p "$STATE_DIR" "$PAGES_DIR" |
| 107 | |
| 108 | TMP=$(mktemp -d) |
| 109 | trap 'rm -rf "$TMP"' EXIT |
| 110 | |
| 111 | NOW=${HARDENING_NOW:-$(date -u +%s)} |
| 112 | |
| 113 | # iso <epoch> — GNU date on the box; the -r fallback keeps test mode runnable on |
| 114 | # a BSD/macOS dev machine. |
| 115 | iso() { date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -r "$1" +%Y-%m-%dT%H:%M:%SZ; } |
| 116 | NOW_ISO=$(iso "$NOW") |
| 117 | |
| 118 | # atomic_write <src> <dest> — temp beside the target + mv, so a reader (or a |
| 119 | # mid-write upload) never sees a half-written JSON. Same-directory temp keeps |
| 120 | # the rename atomic (never across filesystems). |
| 121 | atomic_write() { cp "$1" "$2.tmp.$$" && mv "$2.tmp.$$" "$2"; } |
| 122 | |
| 123 | # env_str <file> <VAR> — one string value from a dotenv file (bucket names, |
| 124 | # prefixes, the mesh self URL). Empty when the file or var is absent. Mirrors |
| 125 | # status-publish.sh's env_str exactly so the two read router.env identically. |
| 126 | env_str() { grep -E "^$2=" "$1" 2>/dev/null | head -n1 | cut -d= -f2- || true; } |
| 127 | |
| 128 | # b64url_decode reverses encodeHeartbeatKey (internal/activator/heartbeat.go): |
| 129 | # url-safe alphabet, padding stripped — re-add it (len mod 4 tells how much) |
| 130 | # before the standard decoder. Same helper, same reason, as status-publish.sh. |
| 131 | b64url_decode() { |
| 132 | local s |
| 133 | s=$(printf '%s' "$1" | tr -- '-_' '+/') |
| 134 | case $(( ${#s} % 4 )) in |
| 135 | 2) s="${s}==" ;; |
| 136 | 3) s="${s}=" ;; |
| 137 | esac |
| 138 | printf '%s' "$s" | base64 -d 2>/dev/null |
| 139 | } |
| 140 | |
| 141 | # The local-OS reads are behind command seams (HARDENING_*_CMD): eval a shell |
| 142 | # command so a test can point each at a `cat fixture` printing pre-captured tool |
| 143 | # output. Real mode runs the actual tool. These vars are operator/test set — the |
| 144 | # same trust boundary as this root script itself. |
| 145 | run_ufw() { if [ -n "${HARDENING_UFW_CMD:-}" ]; then eval "$HARDENING_UFW_CMD"; else ufw status 2>/dev/null; fi; } |
| 146 | run_sshd() { if [ -n "${HARDENING_SSHD_CMD:-}" ]; then eval "$HARDENING_SSHD_CMD"; else sshd -T 2>/dev/null; fi; } |
| 147 | run_autoupd(){ if [ -n "${HARDENING_SYSTEMCTL_CMD:-}" ]; then eval "$HARDENING_SYSTEMCTL_CMD"; else systemctl is-active unattended-upgrades 2>/dev/null; fi; } |
| 148 | run_ip_wg() { if [ -n "${HARDENING_IP_CMD:-}" ]; then eval "$HARDENING_IP_CMD"; else ip link show wg0 2>/dev/null; fi; } |
| 149 | |
| 150 | # host_of <peerURL> — the bare host a :22 probe dials, scheme + port + any path |
| 151 | # stripped. IPv6 literals ([::1]:80) are handled so a colon in the address is |
| 152 | # never mistaken for the port separator. |
| 153 | host_of() { |
| 154 | local u=${1#*://} # drop scheme:// |
| 155 | u=${u%%/*} # drop /path |
| 156 | case "$u" in |
| 157 | \[*\]*) u=${u%%\]*}; printf '%s' "${u#\[}" ;; # [ipv6]:port -> ipv6 |
| 158 | *:*) printf '%s' "${u%:*}" ;; # host:port -> host |
| 159 | *) printf '%s' "$u" ;; |
| 160 | esac |
| 161 | } |
| 162 | |
| 163 | # box_label — this box's stable, whimsical, anonymous name ("hypatia-whistling"), |
| 164 | # an exact port of deploy/status-publish.sh's box_label (and meshstats.go's Go |
| 165 | # twin): sha256 of /etc/machine-id, first 16 hex chars split into two 8-char |
| 166 | # slices indexing the names/actions lists. The lists MUST stay identical to |
| 167 | # those two — same hash, same label, whichever side computes it — so a peer's |
| 168 | # self-published label and this box's local one agree. Falls back to `hostname` |
| 169 | # when the machine-id file is unreadable, matching bash's twin byte-for-byte. |
| 170 | box_label() { |
| 171 | local names=(ada alan grace marie nikola leonardo cleopatra homer sappho |
| 172 | darwin einstein curie turing hopper tesla gandalf frodo sherlock watson |
| 173 | ahab ishmael gatsby scout atticus hermione matilda mowgli merlin arthur |
| 174 | morgana beowulf odysseus penelope achilles athena apollo freya thor loki |
| 175 | amelia boudicca cyrano quixote sancho ophelia viola rosalind puck oberon |
| 176 | titania darcy pip scrooge nemo phileas zorro heidi galileo kepler newton |
| 177 | archimedes hypatia confucius rumi frida vincent wolfgang ludwig clara) |
| 178 | local actions=(stargazing juggling whistling painting sailing gardening |
| 179 | baking skating humming doodling fishing knitting strolling yodeling |
| 180 | beekeeping fencing waltzing sketching brewing climbing cartwheeling |
| 181 | birdwatching pondering inventing exploring composing conducting |
| 182 | spelunking kayaking orienteering puppeteering tinkering) |
| 183 | local idfile=${HARDENING_MACHINE_ID_FILE:-/etc/machine-id} |
| 184 | local seed |
| 185 | seed=$( { cat "$idfile" 2>/dev/null || hostname; } | sha256sum | cut -c1-16) |
| 186 | local n=$(( 16#${seed:0:8} % ${#names[@]} )) |
| 187 | local a=$(( 16#${seed:8:8} % ${#actions[@]} )) |
| 188 | printf '%s-%s' "${names[$n]}" "${actions[$a]}" |
| 189 | } |
| 190 | |
| 191 | # ---- 1. the LOCAL box's posture, from OS state (root reads) ------------------- |
| 192 | # The local box answers about itself authoritatively — never self-probe :22 (it |
| 193 | # would hairpin through the host's own NAT and read nothing meaningful). |
| 194 | |
| 195 | ufw_out=$(run_ufw || true) |
| 196 | first_line=$(printf '%s\n' "$ufw_out" | head -n1) |
| 197 | case "$first_line" in *"Status: active"*) firewall_active=true ;; *) firewall_active=false ;; esac |
| 198 | |
| 199 | # firewall_ssh_scope: hardened ("wireguard-only") iff there is NO bare |
| 200 | # "22/tcp ... ALLOW ... Anywhere" rule (public port 22) AND there IS a "22/tcp |
| 201 | # ... on wg0 ... ALLOW" rule (SSH reachable only over the admin tunnel). |
| 202 | # "internet-facing" the moment a bare 22 allow exists. Anything else — ufw |
| 203 | # inactive/unreadable, or active but with no recognizable 22 rule at all — is |
| 204 | # "unknown": we can't PROVE wireguard-only, and the contract says unproven reads |
| 205 | # loud, never as a silent pass. Matches both `ufw status` (ALLOW) and |
| 206 | # `ufw status verbose` (ALLOW IN) by keying on the "ALLOW" substring. |
| 207 | bare22=$(printf '%s\n' "$ufw_out" | awk ' |
| 208 | /^22(\/tcp)?[ \t]/ && /ALLOW/ && /Anywhere/ && !/wg0/ { f=1 } END { print f+0 }') |
| 209 | wg22=$(printf '%s\n' "$ufw_out" | awk ' |
| 210 | /^22(\/tcp)?[ \t]+on[ \t]+wg0/ && /ALLOW/ { f=1 } END { print f+0 }') |
| 211 | if [ "$firewall_active" != true ] || [ -z "$ufw_out" ]; then |
| 212 | firewall_ssh_scope=unknown |
| 213 | elif [ "$bare22" = 1 ]; then |
| 214 | firewall_ssh_scope=internet-facing |
| 215 | elif [ "$wg22" = 1 ]; then |
| 216 | firewall_ssh_scope=wireguard-only |
| 217 | else |
| 218 | firewall_ssh_scope=unknown |
| 219 | fi |
| 220 | |
| 221 | # root_login / password_auth from sshd's EFFECTIVE config (`sshd -T`) — the |
| 222 | # resolved values, not whatever a config file happens to say. Missing line |
| 223 | # (sshd -T failed, e.g. not root) -> unknown, never assumed safe. |
| 224 | sshd_out=$(run_sshd || true) |
| 225 | root_login=$(printf '%s\n' "$sshd_out" | awk '$1=="permitrootlogin"{print $2; exit}') |
| 226 | [ -n "$root_login" ] || root_login=unknown |
| 227 | password_auth=$(printf '%s\n' "$sshd_out" | awk '$1=="passwordauthentication"{print $2; exit}') |
| 228 | [ -n "$password_auth" ] || password_auth=unknown |
| 229 | |
| 230 | # auto_updates: unattended-upgrades active/inactive; anything else (failed, |
| 231 | # absent, systemctl missing) -> unknown. |
| 232 | au_out=$(printf '%s' "$(run_autoupd || true)" | tr -d '[:space:]') |
| 233 | case "$au_out" in |
| 234 | active) auto_updates=active ;; |
| 235 | inactive) auto_updates=inactive ;; |
| 236 | *) auto_updates=unknown ;; |
| 237 | esac |
| 238 | |
| 239 | # wireguard: wg0 present and up. `ip link show wg0` is empty/non-zero when the |
| 240 | # device is absent; a wireguard interface reports its admin state via the UP |
| 241 | # flag (often "state UNKNOWN" for L3 virtual links), so match the UP flag or a |
| 242 | # literal "state UP". |
| 243 | wg_out=$(run_ip_wg || true) |
| 244 | if [ -z "$wg_out" ]; then |
| 245 | wireguard=absent |
| 246 | elif printf '%s' "$wg_out" | grep -qE 'state UP|[<,]UP[,>]'; then |
| 247 | wireguard=up |
| 248 | else |
| 249 | wireguard=down |
| 250 | fi |
| 251 | |
| 252 | # Fold the local firewall scope into the shared verdict fields every box row |
| 253 | # carries: ssh_internet_facing is a TRISTATE — true (exposed), false (hardened), |
| 254 | # or JSON null (unknown -> the page's loud "not reported"). |
| 255 | case "$firewall_ssh_scope" in |
| 256 | wireguard-only) self_facing=false; self_detail="wireguard-only" ;; |
| 257 | internet-facing) self_facing=true; self_detail="port 22 open to the internet" ;; |
| 258 | *) self_facing=null; self_detail="firewall status not readable" ;; |
| 259 | esac |
| 260 | |
| 261 | SELF=$(env_str "$ROUTER_ENV" LCHAT_MESH_SELF) |
| 262 | SELF=$(printf '%s' "$SELF" | tr -d '[:space:]'); SELF=${SELF%/} # trim like meshFromEnv |
| 263 | SELF_LABEL=$(box_label) |
| 264 | SELF_REGION=$(env_str "$ROUTER_ENV" LCHAT_BOX_REGION) |
| 265 | |
| 266 | # ---- 2. enumerate the mesh + probe peers ------------------------------------- |
| 267 | # enumerate_members prints one TSV row per live mesh member: |
| 268 | # <peerURL> <TAB> <label> <TAB> <region> <TAB> <probe-hint> |
| 269 | # The probe-hint is empty in real mode (the loop runs the real :22 probe); a |
| 270 | # fixture supplies it directly. A member's label/region come from its |
| 271 | # self-published stats object (the same one fleet.json reads). |
| 272 | enumerate_members() { |
| 273 | if [ -n "${HARDENING_MEMBERS_FILE:-}" ]; then |
| 274 | cat "$HARDENING_MEMBERS_FILE" |
| 275 | return |
| 276 | fi |
| 277 | local bucket prefix region mesh_prefix |
| 278 | bucket=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_BUCKET) |
| 279 | prefix=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_PREFIX) |
| 280 | [ -n "$bucket" ] && [ -n "$prefix" ] || return # mesh off -> no peers, just self |
| 281 | region=$(env_str "$ROUTER_ENV" LCHAT_HIBERNATE_REGION); region=${region:-eu-west-1} |
| 282 | # Layout per hc.meshPrefix() (activator.go): membership at |
| 283 | # "<prefix>/mesh/members/<peer>", stats at "<prefix>/mesh/members-stats/<peer>". |
| 284 | mesh_prefix="${prefix}/mesh/members" |
| 285 | |
| 286 | # `aws s3 ls` (high-level), never `aws s3api list-objects*` — the latter |
| 287 | # crashes on this fleet's aws-cli/Python combination (a botocore/argparse |
| 288 | # regression), same reason status-publish.sh gives. `s3 ls` prints |
| 289 | # "YYYY-MM-DD HH:MM:SS SIZE KEY"; the key is the b64url leaf. |
| 290 | # |
| 291 | # Members are enumerated from the "-stats/" SIBLING prefix, not members/ |
| 292 | # itself: the status-publisher credential is deliberately scoped to |
| 293 | # "*-stats/*" in the hibernation bucket (infra/prod/status.tf) so it can |
| 294 | # never touch instance replicas, and a members/ listing comes back |
| 295 | # AccessDenied — silently empty here, which published a peer-less page on |
| 296 | # the collector's first real run (2026-07-19). Both prefixes carry the same |
| 297 | # b64url(peerURL) leaf at the same heartbeat cadence, so the stats listing |
| 298 | # is the same membership signal — read exactly as status-publish.sh's |
| 299 | # mesh_peer_boxes reads it. |
| 300 | local listing cutoff line date_part time_part key peer statsfile label reg |
| 301 | listing=$(aws s3 ls "s3://$bucket/${mesh_prefix}-stats/" --region "$region" 2>/dev/null) || true |
| 302 | [ -n "$listing" ] || return |
| 303 | cutoff=$(date -u -d "-${STALE_SECONDS} seconds" '+%Y-%m-%d %H:%M:%S') |
| 304 | |
| 305 | while IFS= read -r line; do |
| 306 | [ -n "$line" ] || continue |
| 307 | date_part=$(printf '%s' "$line" | awk '{print $1}') |
| 308 | time_part=$(printf '%s' "$line" | awk '{print $2}') |
| 309 | key=$(printf '%s' "$line" | awk '{print $4}') |
| 310 | [ -n "$key" ] || continue |
| 311 | [ "$date_part $time_part" '>' "$cutoff" ] || continue # stale -> dead box, skip |
| 312 | peer=$(b64url_decode "$key"); [ -n "$peer" ] || continue |
| 313 | # Join the sibling stats object for label + region (same values fleet.json |
| 314 | # shows). Absent/garbage stats -> empty; the loop labels it "unidentified". |
| 315 | statsfile="$TMP/stats-$(printf '%s' "$key" | tr '/+=' '___').json" |
| 316 | label=""; reg="" |
| 317 | if aws s3 cp "s3://$bucket/${mesh_prefix}-stats/$key" "$statsfile" --region "$region" >/dev/null 2>&1; then |
| 318 | label=$(jq -r '.label // empty' "$statsfile" 2>/dev/null || true) |
| 319 | reg=$(jq -r '.region // empty' "$statsfile" 2>/dev/null || true) |
| 320 | fi |
| 321 | printf '%s\t%s\t%s\t\n' "$peer" "$label" "$reg" |
| 322 | done <<< "$listing" |
| 323 | } |
| 324 | |
| 325 | # probe_ssh22 <host> — connect-and-banner-grab of :22 from THIS box (an outside |
| 326 | # viewpoint for a peer). Echoes "<verdict>\t<banner>". See the VANTAGE-DEPENDENT |
| 327 | # note above: only a returned "SSH-…" banner is a confident "exposed"; a clean |
| 328 | # refusal or a connect timeout is "not reachable" (good); a completed connection |
| 329 | # with no SSH banner — or anything else that isn't a clean refusal — is |
| 330 | # "inconclusive" (loud unknown, never a pass). No bytes are sent; no login is |
| 331 | # attempted. Prefer bash /dev/tcp for the connect + banner read. |
| 332 | probe_ssh22() { |
| 333 | local host=$1 out banner |
| 334 | [ -n "$host" ] || { printf 'inconclusive\t'; return; } |
| 335 | # Resolve first so an NXDOMAIN reads as "inconclusive" (couldn't probe), |
| 336 | # distinct from a resolvable host that cleanly refuses :22. |
| 337 | getent hosts "$host" >/dev/null 2>&1 || { printf 'inconclusive\t'; return; } |
| 338 | out=$(timeout "$PROBE_TIMEOUT" bash -c ' |
| 339 | h=$1 |
| 340 | exec 3<>/dev/tcp/"$h"/22 2>/dev/null || { printf "REFUSED"; exit 0; } |
| 341 | b="" |
| 342 | IFS= read -r -n 60 -t "'"$BANNER_READ"'" b <&3 2>/dev/null || true |
| 343 | exec 3>&- 2>/dev/null || true |
| 344 | if [ -n "$b" ]; then printf "BANNER\t%s" "$b"; else printf "NOBANNER"; fi |
| 345 | ' _ "$host" 2>/dev/null) || out="" |
| 346 | case "$out" in |
| 347 | BANNER*) |
| 348 | banner=${out#BANNER}; banner=${banner#$'\t'} |
| 349 | banner=$(printf '%s' "$banner" | tr -cd '[:print:]' | cut -c1-48) |
| 350 | case "$banner" in |
| 351 | SSH-*) printf 'serving-ssh\t%s' "$banner" ;; |
| 352 | *) printf 'inconclusive\t' ;; # answered, but not an SSH banner |
| 353 | esac ;; |
| 354 | NOBANNER) printf 'inconclusive\t' ;; # connected, silent — vantage-dependent |
| 355 | REFUSED) printf 'not-reachable\t' ;; # clean refusal — good |
| 356 | *) printf 'not-reachable\t' ;; # empty = connect timed out — good |
| 357 | esac |
| 358 | } |
| 359 | |
| 360 | # classify_probe <verdict> [banner] -> sets FACING (true/false/null) + DETAIL. |
| 361 | # Inconclusive maps to null so the page renders it loud, per the contract. |
| 362 | classify_probe() { |
| 363 | case "$1" in |
| 364 | serving-ssh|reachable) |
| 365 | FACING=true |
| 366 | if [ -n "${2:-}" ]; then DETAIL="port 22 serving SSH ($2)"; else DETAIL="port 22 serving SSH"; fi ;; |
| 367 | not-reachable|filtered-or-closed) |
| 368 | FACING=false; DETAIL="port 22 not reachable" ;; |
| 369 | *) |
| 370 | FACING=null; DETAIL="port 22 filtered/inconclusive" ;; |
| 371 | esac |
| 372 | } |
| 373 | |
| 374 | # peer_posture <label> — a peer's own self-published posture row, if fresh. |
| 375 | # Every box running this collector uploads its self row to |
| 376 | # "posture/<label>.json" in the status bucket (see the upload section), so any |
| 377 | # box can read back every other box's AUTHORITATIVE local read instead of |
| 378 | # stopping at the outside :22 hint. This is also what lets N collector boxes |
| 379 | # share the one hardening.json without flip-flopping viewpoints: whichever box |
| 380 | # uploads last, it built the same merged rows from the same self-reports. |
| 381 | # Freshness gates on the row's own `generated` stamp — a box that stopped |
| 382 | # collecting (or died) falls back to the probe view within POSTURE_STALE_SECONDS. |
| 383 | # Test seam: HARDENING_POSTURE_DIR serves "<label>.json" files instead of S3. |
| 384 | POSTURE_STALE_SECONDS=${HARDENING_POSTURE_STALE_SECONDS:-900} |
| 385 | peer_posture() { |
| 386 | local label=$1 f gen now age |
| 387 | [ -n "$label" ] || return 0 |
| 388 | f="$TMP/posture-$label.json" |
| 389 | if [ -n "${HARDENING_POSTURE_DIR:-}" ]; then |
| 390 | cat "$HARDENING_POSTURE_DIR/$label.json" 2>/dev/null || true |
| 391 | return 0 |
| 392 | fi |
| 393 | aws s3 cp "s3://$BUCKET/posture/$label.json" "$f" --region "$REGION" >/dev/null 2>&1 || return 0 |
| 394 | gen=$(jq -r '.generated // empty' "$f" 2>/dev/null) || true |
| 395 | [ -n "$gen" ] || return 0 |
| 396 | now=$(date -u +%s) |
| 397 | age=$(( now - $(date -u -d "$gen" +%s 2>/dev/null || echo 0) )) |
| 398 | [ "$age" -ge 0 ] && [ "$age" -le "$POSTURE_STALE_SECONDS" ] || return 0 |
| 399 | cat "$f" |
| 400 | } |
| 401 | |
| 402 | # ---- 3. build the boxes array ------------------------------------------------ |
| 403 | echo '[]' > "$TMP/boxes.json" |
| 404 | add_box() { # $1 = a JSON object string |
| 405 | jq --argjson b "$1" '. + [$b]' "$TMP/boxes.json" > "$TMP/boxes.next" \ |
| 406 | && mv "$TMP/boxes.next" "$TMP/boxes.json" |
| 407 | } |
| 408 | |
| 409 | self_added=0 |
| 410 | add_self() { |
| 411 | local obj |
| 412 | obj=$(jq -n \ |
| 413 | --arg label "$SELF_LABEL" --arg region "$SELF_REGION" \ |
| 414 | --argjson facing "$self_facing" --arg detail "$self_detail" \ |
| 415 | --arg root "$root_login" --arg pass "$password_auth" \ |
| 416 | --arg auto "$auto_updates" --arg wg "$wireguard" \ |
| 417 | --argjson fwactive "$firewall_active" \ |
| 418 | '{label:$label, is_self:true, ssh_internet_facing:$facing, detail:$detail, |
| 419 | root_login:$root, password_auth:$pass, auto_updates:$auto, wireguard:$wg, |
| 420 | firewall_active:$fwactive, source:"local"} |
| 421 | + (if $region == "" then {} else {region:$region} end)') |
| 422 | add_box "$obj" |
| 423 | # The same row, stamped, becomes this box's self-report for PEERS to merge |
| 424 | # (posture/<label>.json — see peer_posture). Uploaded in section 4. |
| 425 | printf '%s' "$obj" | jq --arg now "$NOW_ISO" '. + {generated:$now}' > "$TMP/self-posture.json" |
| 426 | self_added=1 |
| 427 | } |
| 428 | |
| 429 | # Split each TSV row by tab MANUALLY (not `read -r a b c d`): tab is IFS |
| 430 | # whitespace, so read collapses consecutive tabs and an empty column (a peer |
| 431 | # with no region) would silently shift the later fields left. Parameter |
| 432 | # expansion on a fixed 4-field (3-tab) row preserves every empty. |
| 433 | while IFS= read -r row; do |
| 434 | [ -n "$row" ] || continue |
| 435 | peer=${row%%$'\t'*}; rest=${row#*$'\t'} |
| 436 | label=${rest%%$'\t'*}; rest=${rest#*$'\t'} |
| 437 | region=${rest%%$'\t'*}; hint=${rest#*$'\t'} |
| 438 | if [ -n "$SELF" ] && [ "$peer" = "$SELF" ]; then |
| 439 | add_self |
| 440 | continue |
| 441 | fi |
| 442 | # A peer: probe its :22 (a fixture hint wins in test mode). |
| 443 | if [ -n "$hint" ]; then |
| 444 | verdict=$hint; banner="" |
| 445 | else |
| 446 | probe_out=$(probe_ssh22 "$(host_of "$peer")") |
| 447 | verdict=${probe_out%%$'\t'*}; banner=${probe_out#*$'\t'} |
| 448 | fi |
| 449 | classify_probe "$verdict" "$banner" |
| 450 | # Prefer the peer's own fresh self-report over the probe-only row — but the |
| 451 | # probe still has veto power the loud way: an SSH banner WE saw from outside |
| 452 | # overrides whatever the peer claims about itself (a compromised or |
| 453 | # misconfigured box must not be able to self-report its exposure away). |
| 454 | # Every other field (root login, wireguard, firewall…) only the box itself |
| 455 | # can know, so the self-report carries them verbatim. |
| 456 | posture=$(peer_posture "$label") |
| 457 | if [ -n "$posture" ]; then |
| 458 | obj=$(printf '%s' "$posture" | jq \ |
| 459 | --argjson pfacing "$FACING" --arg pdetail "$DETAIL" \ |
| 460 | 'del(.generated) |
| 461 | + {is_self:false, source:"self-reported"} |
| 462 | + (if $pfacing == true then {ssh_internet_facing:true, detail:$pdetail} else {} end)') |
| 463 | add_box "$obj" |
| 464 | continue |
| 465 | fi |
| 466 | [ -n "$label" ] || label="unidentified box" |
| 467 | obj=$(jq -n \ |
| 468 | --arg label "$label" --arg region "$region" \ |
| 469 | --argjson facing "$FACING" --arg detail "$DETAIL" \ |
| 470 | '{label:$label, is_self:false, ssh_internet_facing:$facing, detail:$detail, |
| 471 | root_login:"unknown", auto_updates:"unknown", wireguard:"unknown", |
| 472 | source:"probe"} |
| 473 | + (if $region == "" then {} else {region:$region} end)') |
| 474 | add_box "$obj" |
| 475 | done < <(enumerate_members) |
| 476 | |
| 477 | # Self may not have appeared in the membership listing (mesh off, or this box's |
| 478 | # own heartbeat hasn't landed yet) — always publish the local box's own row. |
| 479 | [ "$self_added" = 1 ] || add_self |
| 480 | |
| 481 | # Sort problems to the TOP: internet-facing (true) first, then inconclusive |
| 482 | # (null), then hardened (false); ties broken by label for a stable order. The |
| 483 | # loud rows a viewer needs to see never hide below the calm ones. |
| 484 | jq --arg now "$NOW_ISO" \ |
| 485 | '{generated: $now, |
| 486 | boxes: sort_by( |
| 487 | (if .ssh_internet_facing == true then 0 |
| 488 | elif .ssh_internet_facing == null then 1 |
| 489 | else 2 end), |
| 490 | (.label // ""))}' \ |
| 491 | "$TMP/boxes.json" > "$TMP/hardening.json" |
| 492 | atomic_write "$TMP/hardening.json" "$OUT" |
| 493 | |
| 494 | # ---- 4. upload --------------------------------------------------------------- |
| 495 | # Standalone by design (see header): uploads its own outputs with the same |
| 496 | # scoped status-publisher creds status-publish.sh uses. A fixture run (members |
| 497 | # fixture) or an explicit HARDENING_NO_UPLOAD=1 never uploads. Re-uploading the |
| 498 | # page every run self-heals a missed copy, same as the other publishers. |
| 499 | upload=1 |
| 500 | [ "${HARDENING_NO_UPLOAD:-}" = "1" ] && upload=0 |
| 501 | [ -n "${HARDENING_MEMBERS_FILE:-}" ] && upload=0 |
| 502 | if [ "$upload" = 1 ]; then |
| 503 | command -v aws >/dev/null 2>&1 || { echo "aws CLI missing — collected, not uploaded" >&2; exit 1; } |
| 504 | if [ -z "${AWS_ACCESS_KEY_ID:-}" ] && [ -f "$STATUS_ENV" ]; then |
| 505 | set -a; . "$STATUS_ENV"; set +a |
| 506 | fi |
| 507 | up() { aws s3 cp "$1" "s3://$BUCKET/$2" --region "$REGION" \ |
| 508 | --content-type "$3" --cache-control "$4" --no-progress >/dev/null; } |
| 509 | up "$OUT" hardening.json "application/json" "max-age=60" |
| 510 | # The self-report peers merge (peer_posture) — short cache: it's read |
| 511 | # box-to-box via s3 cp, so the cache header only matters if a human fetches it. |
| 512 | [ -f "$TMP/self-posture.json" ] && up "$TMP/self-posture.json" "posture/$SELF_LABEL.json" "application/json" "max-age=60" |
| 513 | [ -f "$PAGES_DIR/hardening.html" ] && up "$PAGES_DIR/hardening.html" hardening.html "text/html; charset=utf-8" "max-age=300" |
| 514 | echo "hardening posture published to s3://$BUCKET ($NOW_ISO)" |
| 515 | fi |
| 516 | |
amadan