amadan
server: encode the root-SSH lockdown as bootstrap --ssh-noroot (#1300)
The hardening recipe's last step — the PermitRootLogin-no drop-in — existed only as prose in deploy/README.md step 3, which is how kepler missed it: its sshd still reports prohibit-password on the hardening dashboard, and with root SSH already unreachable there is no root shell left to fix it by hand. lchat bootstrap --ssh-noroot converges the drop-in (/etc/ssh/sshd_config.d/10-lchat-noroot.conf), validates the merged config with sshd -t (restoring the prior state on failure), and reloads sshd — reachable over WireGuard via the already-allowlisted sudo -n /opt/lchat/lchat. It refuses until the lchat-operators group has members so a fresh box can never shut root out before operator accounts exist, and every full bootstrap run applies the same converge once they do, so a box that missed the manual step picks it up on the next provision pass. Self-hosters without operator setup are untouched (the group gate).
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
5 files changed,
+237
−10
bootstrap.go+114 −0bootstrap_test.go+104 −0deploy/README.md+7 −3docs/instance-router.md+11 −6docs/planet-scale-guide/operations.html+1 −1
diff --git a/bootstrap.go b/bootstrap.go| index acf9136..e03fb19 100644 |
| --- a/bootstrap.go |
| +++ b/bootstrap.go |
| @@ -438,6 +438,84 @@ func registerLinksApex(host, registryPath string) { |
| fmt.Printf("bootstrap: short-link apex %s -> %s\n", host, linksSocket) |
| } |
| +// --- root-SSH lockdown ------------------------------------------------------- |
| + |
| +// The drop-in that turns root SSH off for good — the last step of operator |
| +// setup (docs/instance-router.md § Hardening), encoded here because a box that |
| +// misses the manual step has no root shell left to fix it with; the allowlisted |
| +// `sudo -n /opt/lchat/lchat bootstrap --ssh-noroot` is the remaining root- |
| +// capable path. The 10- prefix matters: sshd config is first-match-wins, so |
| +// this must sort before the distro's 50-*.conf (prohibit-password) to win. |
| +const noRootConfPath = "/etc/ssh/sshd_config.d/10-lchat-noroot.conf" |
| + |
| +const noRootConf = `# Installed by lchat bootstrap (--ssh-noroot). Root SSH is bootstrap-only on |
| +# fleet boxes: operators are named accounts over WireGuard whose root capability |
| +# is the sudoers allowlist (deploy/sudoers-operators). Undo via console/rescue |
| +# only — see docs/instance-router.md § Hardening. |
| +PermitRootLogin no |
| +` |
| + |
| +// operatorsExist reports whether the lchat-operators group has members — the |
| +// "someone other than root can still get in" gate for the lockdown, so a fresh |
| +// box can never shut root out before operator accounts exist. Parsed from the |
| +// group file directly: os/user resolves groups but can't list their members. |
| +func operatorsExist(groupFile string) bool { |
| + b, err := os.ReadFile(groupFile) |
| + if err != nil { |
| + return false |
| + } |
| + for _, line := range strings.Split(string(b), "\n") { |
| + if f := strings.Split(line, ":"); len(f) == 4 && f[0] == "lchat-operators" && strings.TrimSpace(f[3]) != "" { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// sshdCheck validates the merged sshd config (`sshd -t`); a var so tests can |
| +// stand in for the daemon binary. |
| +var sshdCheck = func() error { |
| + sshd, err := exec.LookPath("sshd") |
| + if err != nil { |
| + sshd = "/usr/sbin/sshd" // sudo's PATH may omit sbin; the fleet's lives here |
| + } |
| + if out, err := exec.Command(sshd, "-t").CombinedOutput(); err != nil { |
| + return fmt.Errorf("sshd -t: %v: %s", err, strings.TrimSpace(string(out))) |
| + } |
| + return nil |
| +} |
| + |
| +// applyNoRootSSH converges the drop-in: write, validate the merged config, |
| +// reload sshd — restoring the prior state if validation fails, so a bad write |
| +// can never leave config sshd would refuse on its next real restart. An |
| +// already-correct file is a quiet no-op with no reload. Reload only re-reads |
| +// config: existing sessions (including the one running this) are untouched, |
| +// and named-account logins are unaffected — only new root logins are refused. |
| +func applyNoRootSSH(confPath string) (changed bool, err error) { |
| + prev, prevErr := os.ReadFile(confPath) |
| + if prevErr == nil && string(prev) == noRootConf { |
| + return false, nil |
| + } |
| + if err := os.WriteFile(confPath, []byte(noRootConf), 0o644); err != nil { |
| + return false, err |
| + } |
| + if err := sshdCheck(); err != nil { |
| + if prevErr == nil { |
| + os.WriteFile(confPath, prev, 0o644) |
| + } else { |
| + os.Remove(confPath) |
| + } |
| + return false, err |
| + } |
| + // Ubuntu names the unit ssh; other distros sshd. |
| + if err := systemctl("reload", "ssh"); err != nil { |
| + if err2 := systemctl("reload", "sshd"); err2 != nil { |
| + return false, fmt.Errorf("%v; %v", err, err2) |
| + } |
| + } |
| + return true, nil |
| +} |
| + |
| // runBootstrap installs the systemd units + runtime dir and starts the router, so |
| // provisioning a fresh box is one command after the binary is copied to |
| // /opt/lchat/lchat. Idempotent — safe to re-run. See docs/instance-router.md. |
| @@ -449,8 +527,30 @@ func runBootstrap(args []string) { |
| collectorsOnly := fs.Bool("collectors-only", false, "refresh ONLY the status collectors (scripts, units, viewer pages) + enable their timers — no service user, no control token, no router restart") |
| linksApex := fs.String("links-apex", "", "make this box the short-link apex for this host (starts the redirector and routes the host at it); without it, a box that isn't already routed as the apex doesn't run the redirector at all") |
| noUpdateTimer := fs.Bool("no-update-timer", false, "ensure the hourly auto-updater is DISABLED instead of enabled (for boxes that must never track a release channel, e.g. the dev mesh)") |
| + sshNoRoot := fs.Bool("ssh-noroot", false, "ONLY converge the root-SSH lockdown (PermitRootLogin no drop-in + sshd reload) — refused until the lchat-operators group has members") |
| _ = fs.Parse(args) |
| + // --ssh-noroot is the scoped, allowlist-reachable form of the hardening |
| + // recipe's last step: on a box where root SSH is already unreachable (keys |
| + // pulled) but sshd still says prohibit-password, an operator flips the |
| + // config itself over WireGuard — no root shell involved. Refused until |
| + // operator accounts exist so a fresh box can't shut root out first. |
| + if *sshNoRoot { |
| + if !operatorsExist("/etc/group") { |
| + die("bootstrap: refusing to disable root SSH — the lchat-operators group has no members yet; finish operator setup first (docs/instance-router.md § Hardening)") |
| + } |
| + changed, err := applyNoRootSSH(noRootConfPath) |
| + if err != nil { |
| + die("bootstrap: root-SSH lockdown: %v", err) |
| + } |
| + if changed { |
| + fmt.Println("bootstrap: root SSH disabled (PermitRootLogin no; sshd reloaded) — confirm: sshd -T | grep permitrootlogin") |
| + } else { |
| + fmt.Println("bootstrap: root-SSH lockdown already in place — nothing to do") |
| + } |
| + return |
| + } |
| + |
| // --collectors-only is the no-root-shell refresh path: with the fleet |
| // hardened there's no root scp to drop new collector scripts/pages, so an |
| // operator runs `sudo -n /opt/lchat/lchat bootstrap --collectors-only` over |
| @@ -548,6 +648,20 @@ func runBootstrap(args []string) { |
| // enable their 5-minute timers (safe everywhere — a pass with no /data/status.env |
| // creds is a near no-op). |
| enableCollectorTimers() |
| + // Converge the root-SSH lockdown on boxes that completed operator setup, so |
| + // a box that missed the manual step (kepler, 2026-07) picks it up on the |
| + // next provision pass. A box with no operator accounts — a fresh bootstrap, |
| + // or a self-hoster who never ran operator setup — is skipped: shutting root |
| + // out there could shut everyone out. |
| + if operatorsExist("/etc/group") { |
| + changed, err := applyNoRootSSH(noRootConfPath) |
| + if err != nil { |
| + die("bootstrap: root-SSH lockdown: %v", err) |
| + } |
| + if changed { |
| + fmt.Println(" root SSH disabled (PermitRootLogin no; sshd reloaded)") |
| + } |
| + } |
| autoUpdate := "auto-update" |
| if *noUpdateTimer { |
| autoUpdate = "NO auto-update" |
diff --git a/bootstrap_test.go b/bootstrap_test.go| index a31dac9..5878513 100644 |
| --- a/bootstrap_test.go |
| +++ b/bootstrap_test.go |
| @@ -380,3 +380,107 @@ func assertFileMode(t *testing.T, path string, want os.FileMode) { |
| t.Errorf("%s mode = %o, want %o", path, got, want) |
| } |
| } |
| + |
| +// TestOperatorsExist pins the lockdown gate: only a group file whose |
| +// lchat-operators entry actually lists members opens it — a missing file, |
| +// missing group, or empty member list must all read as "no operators". |
| +func TestOperatorsExist(t *testing.T) { |
| + cases := map[string]struct { |
| + content string |
| + want bool |
| + }{ |
| + "members": {"root:x:0:\nlchat-operators:x:990:paul,james\n", true}, |
| + "one member": {"lchat-operators:x:990:paul\n", true}, |
| + "empty list": {"lchat-operators:x:990:\n", false}, |
| + "blank list": {"lchat-operators:x:990: \n", false}, |
| + "no group": {"root:x:0:\nlchat:x:999:\n", false}, |
| + "name is prefix only": {"lchat-operators-old:x:991:paul\n", false}, |
| + } |
| + for name, tc := range cases { |
| + p := filepath.Join(t.TempDir(), "group") |
| + if err := os.WriteFile(p, []byte(tc.content), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := operatorsExist(p); got != tc.want { |
| + t.Errorf("%s: operatorsExist = %v, want %v", name, got, tc.want) |
| + } |
| + } |
| + if operatorsExist(filepath.Join(t.TempDir(), "missing")) { |
| + t.Error("missing group file should read as no operators") |
| + } |
| +} |
| + |
| +// TestNoRootDropIn guards the drop-in contract: the directive itself, and the |
| +// filename ordering that makes it effective — sshd config is first-match-wins, |
| +// so the file must sort before the distro's 50-*.conf (prohibit-password). |
| +func TestNoRootDropIn(t *testing.T) { |
| + if !strings.Contains(noRootConf, "PermitRootLogin no\n") { |
| + t.Error("drop-in missing its PermitRootLogin no directive") |
| + } |
| + base := filepath.Base(noRootConfPath) |
| + if base >= "50-" { |
| + t.Errorf("drop-in %s sorts after 50-*.conf — it would lose first-match to the distro default", base) |
| + } |
| + if filepath.Dir(noRootConfPath) != "/etc/ssh/sshd_config.d" { |
| + t.Errorf("drop-in in %s, want /etc/ssh/sshd_config.d", filepath.Dir(noRootConfPath)) |
| + } |
| +} |
| + |
| +// TestApplyNoRootSSH covers the converge flow with the exec seams stubbed: |
| +// fresh write reloads sshd, an already-correct file is a silent no-op, and a |
| +// failed validation restores exactly the prior state (file absent or prior |
| +// content) so sshd's next real restart can never hit config it would refuse. |
| +func TestApplyNoRootSSH(t *testing.T) { |
| + oldCheck, oldSysctl := sshdCheck, systemctl |
| + defer func() { sshdCheck, systemctl = oldCheck, oldSysctl }() |
| + |
| + var reloads []string |
| + systemctl = func(args ...string) error { |
| + reloads = append(reloads, strings.Join(args, " ")) |
| + return nil |
| + } |
| + sshdCheck = func() error { return nil } |
| + |
| + p := filepath.Join(t.TempDir(), "10-lchat-noroot.conf") |
| + changed, err := applyNoRootSSH(p) |
| + if err != nil || !changed { |
| + t.Fatalf("fresh apply: changed=%v err=%v, want true, nil", changed, err) |
| + } |
| + if b, _ := os.ReadFile(p); string(b) != noRootConf { |
| + t.Errorf("drop-in content = %q", b) |
| + } |
| + if len(reloads) != 1 || reloads[0] != "reload ssh" { |
| + t.Errorf("reloads = %v, want [reload ssh]", reloads) |
| + } |
| + |
| + // Second pass: already converged — no write, no reload. |
| + reloads = nil |
| + if changed, err = applyNoRootSSH(p); err != nil || changed { |
| + t.Fatalf("idempotent apply: changed=%v err=%v, want false, nil", changed, err) |
| + } |
| + if len(reloads) != 0 { |
| + t.Errorf("idempotent apply reloaded sshd: %v", reloads) |
| + } |
| + |
| + // Validation failure over an existing (stale) file restores it verbatim. |
| + stale := "PermitRootLogin prohibit-password\n" |
| + if err := os.WriteFile(p, []byte(stale), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + sshdCheck = func() error { return os.ErrInvalid } |
| + if _, err = applyNoRootSSH(p); err == nil { |
| + t.Fatal("apply with failing sshd -t should error") |
| + } |
| + if b, _ := os.ReadFile(p); string(b) != stale { |
| + t.Errorf("failed validation left %q, want prior content restored", b) |
| + } |
| + |
| + // Validation failure on a fresh box removes the file entirely. |
| + p2 := filepath.Join(t.TempDir(), "10-lchat-noroot.conf") |
| + if _, err = applyNoRootSSH(p2); err == nil { |
| + t.Fatal("apply with failing sshd -t should error") |
| + } |
| + if _, statErr := os.Stat(p2); !os.IsNotExist(statErr) { |
| + t.Errorf("failed validation left the drop-in behind (stat err %v)", statErr) |
| + } |
| +} |
diff --git a/deploy/README.md b/deploy/README.md| index f2b965e..420e8ba 100644 |
| --- a/deploy/README.md |
| +++ b/deploy/README.md |
| @@ -91,9 +91,13 @@ ssh root@<box> '/opt/lchat/lchat add-instance chat.example.com' |
| # create a named account per operator (groups lchat-operators + systemd-journal), |
| # install deploy/sudoers-operators -> /etc/sudoers.d/lchat-operators (0440) |
| # and deploy/lchat-env-install + deploy/lchat-svc -> /usr/local/sbin/ (0755), |
| -# add each operator's WireGuard peer, close public 22, and set |
| -# PermitRootLogin no (drop-in /etc/ssh/sshd_config.d/10-lchat-noroot.conf — |
| -# the 10- prefix wins sshd's first-match over the 50-hardening.conf default) |
| +# add each operator's WireGuard peer, close public 22, and run |
| +# `lchat bootstrap --ssh-noroot` — writes the PermitRootLogin-no drop-in |
| +# (/etc/ssh/sshd_config.d/10-lchat-noroot.conf; the 10- prefix wins sshd's |
| +# first-match over the 50-hardening.conf default), validates, reloads sshd. |
| +# It refuses until lchat-operators has members, and every later full |
| +# bootstrap converges the same drop-in — a box that missed the step picks |
| +# it up on the next provision pass, root shell not required. |
| ``` |
| `lchat bootstrap` embeds the unit files (source `deploy/systemd/`) and writes |
diff --git a/docs/instance-router.md b/docs/instance-router.md| index f260997..580dabc 100644 |
| --- a/docs/instance-router.md |
| +++ b/docs/instance-router.md |
| @@ -675,12 +675,17 @@ else binds a unix socket and needs no capabilities (`CapabilityBoundingSet=`). |
| `deploy/sudoers-operators` at `/etc/sudoers.d/lchat-operators` (mode `0440`) |
| and the env-placement helper `deploy/lchat-env-install` at |
| `/usr/local/sbin/lchat-env-install` (mode `0755`), add the WireGuard peers — |
| - and then disable root logins with a drop-in |
| - `/etc/ssh/sshd_config.d/10-lchat-noroot.conf` (`PermitRootLogin no`). The |
| - `10-` prefix matters: sshd_config is first-match-wins, so the drop-in must |
| - sort *before* the box's `50-hardening.conf` (which sets |
| - `prohibit-password`) to take effect — confirm with `sshd -T | grep |
| - permitrootlogin` before trusting the flip. |
| + and then disable root logins: `lchat bootstrap --ssh-noroot` writes the |
| + drop-in `/etc/ssh/sshd_config.d/10-lchat-noroot.conf` (`PermitRootLogin |
| + no`), validates with `sshd -t`, and reloads sshd. The `10-` prefix matters: |
| + sshd_config is first-match-wins, so the drop-in must sort *before* the |
| + box's `50-hardening.conf` (which sets `prohibit-password`) to take effect — |
| + confirm with `sshd -T | grep permitrootlogin` before trusting the flip. |
| + The verb refuses until the `lchat-operators` group has members (a fresh box |
| + can't shut root out before anyone else can get in), and every full |
| + `bootstrap` run converges the same drop-in once operators exist — so a box |
| + that missed the step picks it up on the next provision pass, via the |
| + allowlisted `sudo -n /opt/lchat/lchat`, with no root shell needed. |
| Thereafter operators are named users over the tunnel; root capability exists |
| only through the allowlist, and anything new that needs root gets a line |
| added **via a reviewed PR + redeploy**, never a root shell or a hand-edit on |
diff --git a/docs/planet-scale-guide/operations.html b/docs/planet-scale-guide/operations.html| index 4794269..01cffb3 100644 |
| --- a/docs/planet-scale-guide/operations.html |
| +++ b/docs/planet-scale-guide/operations.html |
| @@ -124,7 +124,7 @@ sqlite3 /tmp/chat-a-snapshot.db "PRAGMA integrity_check" |
| <ul> |
| <li><strong>Firewall:</strong> allow inbound 80, 443, and the WireGuard port (51820, UDP) only; default-deny the rest. SSH gets no public allow — it is permitted only <em>on the WireGuard interface</em> (<code>ufw allow in on wg0 to any port 22 proto tcp</code>). Everything internal is unix sockets, so nothing else needs a port. (Add the allows <em>before</em> enabling the firewall, and verify the tunnel from outside before removing any public SSH rule, so you do not lock out your own session.) <span class="status built">running today</span></li> |
| <li><strong>SSH rides WireGuard:</strong> each operator is a peer on the box’s <code>wg0</code> interface and connects to the box’s tunnel address; the tunnel runs persistently on the operator’s machine with an <code>ssh</code> configuration alias, so every existing command works unchanged. WireGuard does not answer unauthenticated packets, so the SSH port is invisible to scanners and brute-force noise drops to zero. Key-only remains in force (<code>PasswordAuthentication no</code>). The full recipe, including the break-glass path through the hosting provider’s rescue console, is in <code>docs/instance-router.md</code> under Hardening. <span class="status built">running today</span></li> |
| - <li><strong>Root SSH is bootstrap-only:</strong> the first root session bootstraps the box and creates named operator accounts (group <code>lchat-operators</code>), installs the committed sudo allowlist (<code>deploy/sudoers-operators</code> → <code>/etc/sudoers.d/lchat-operators</code>) and the <code>lchat-env-install</code> helper for placing secrets, then sets <code>PermitRootLogin no</code>. From then on operators are named users whose root capability is exactly the allowlist — anything new that needs root gets an allowlist line through a reviewed pull request, never a root shell. This is accident-proofing and attribution with a reviewable surface, not a hard security boundary (the allowlist can deploy binaries); the hard boundary is the WireGuard key. <span class="status built">running today</span></li> |
| + <li><strong>Root SSH is bootstrap-only:</strong> the first root session bootstraps the box and creates named operator accounts (group <code>lchat-operators</code>), installs the committed sudo allowlist (<code>deploy/sudoers-operators</code> → <code>/etc/sudoers.d/lchat-operators</code>) and the <code>lchat-env-install</code> helper for placing secrets, then sets <code>PermitRootLogin no</code> with <code>lchat bootstrap --ssh-noroot</code> (refused until operator accounts exist, so a fresh box cannot shut root out before anyone else can get in; every full <code>bootstrap</code> run converges the same drop-in once they do, so a box that missed the step picks it up on the next provision pass — no root shell needed). From then on operators are named users whose root capability is exactly the allowlist — anything new that needs root gets an allowlist line through a reviewed pull request, never a root shell. This is accident-proofing and attribution with a reviewable surface, not a hard security boundary (the allowlist can deploy binaries); the hard boundary is the WireGuard key. <span class="status built">running today</span></li> |
| <li><strong>Automatic security updates</strong> for the operating system (<code>unattended-upgrades</code>) — the application updates itself (<a href="deploys.html#auto-updates">part 04</a>), but the kernel and OpenSSL do not.</li> |
| <li><strong>Pin the box’s own hostnames to loopback</strong> in <code>/etc/hosts</code>, so on-box clients (bot sidecars, calls to the push relay) reach the local router instead of the public internet — and disable any cloud-image behaviour that regenerates that file on boot. The subtle failure this prevents is described in <a href="processes.html#bot-sidecars">part 03</a>.</li> |
| <li><strong>The unit files already sandbox every service</strong> (<a href="processes.html#security-model">part 03</a>) — nothing to do, but worth knowing when something cannot write where you expected: <code>ProtectSystem=strict</code> means the whole filesystem is read-only to services except <code>/data</code> and <code>/run/lchat</code>.</li> |