eleven / messenger Public

server: bootstrap --ssh-noroot verifies the EFFECTIVE sshd config (#1312)

The verb wrote the 10-lchat-noroot.conf drop-in, passed sshd -t, reloaded,
and reported "root SSH disabled" — while kepler's pre-existing
00-hardening.conf (sorting before 10-; sshd is first-match-wins) silently
kept permitrootlogin at prohibit-password. Root still got in; only the
hardening dashboard's stamp told the truth.

applyNoRootSSH now confirms sshd -T reports permitrootlogin no after the
write/reload — and on the idempotent path too, so an already-present but
losing drop-in can never read as success. On mismatch it errors naming the
first-match cause and leaves the inert drop-in in place for the re-run
after the operator renames the offender.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell 3 months ago 2b8811a553aa10bb41662d3d3d65ddc1aafc8c5e parent 05fc08e

5 files changed, +93 −34

bootstrap.go +55 −23
bootstrap_test.go +26 −4
deploy/README.md +3 −2
docs/instance-router.md +8 −4
docs/planet-scale-guide/operations.html +1 −1

diff --git a/bootstrap.go b/bootstrap.go

index e03fb19..56c08da 100644

--- a/bootstrap.go

+++ b/bootstrap.go

@@ -485,35 +485,67 @@ var sshdCheck = func() error {

return nil

}

-// applyNoRootSSH converges the drop-in: write, validate the merged config,

-// reload sshd — restoring the prior state if validation fails, so a bad write

-// can never leave config sshd would refuse on its next real restart. An

-// already-correct file is a quiet no-op with no reload. Reload only re-reads

-// config: existing sessions (including the one running this) are untouched,

-// and named-account logins are unaffected — only new root logins are refused.

-func applyNoRootSSH(confPath string) (changed bool, err error) {

- prev, prevErr := os.ReadFile(confPath)

- if prevErr == nil && string(prev) == noRootConf {

- return false, nil

+// sshdRootLogin reports sshd's EFFECTIVE permitrootlogin (`sshd -T`) — the

+// merged, first-match-wins truth, not the drop-in's content; a var so tests

+// can stand in for the daemon binary.

+var sshdRootLogin = func() (string, error) {

+ sshd, err := exec.LookPath("sshd")

+ if err != nil {

+ sshd = "/usr/sbin/sshd"

}

- if err := os.WriteFile(confPath, []byte(noRootConf), 0o644); err != nil {

- return false, err

+ out, err := exec.Command(sshd, "-T").Output()

+ if err != nil {

+ return "", fmt.Errorf("sshd -T: %v", err)

}

- if err := sshdCheck(); err != nil {

- if prevErr == nil {

- os.WriteFile(confPath, prev, 0o644)

- } else {

- os.Remove(confPath)

+ for _, line := range strings.Split(string(out), "\n") {

+ if f := strings.Fields(line); len(f) == 2 && f[0] == "permitrootlogin" {

+ return f[1], nil

}

- return false, err

}

- // Ubuntu names the unit ssh; other distros sshd.

- if err := systemctl("reload", "ssh"); err != nil {

- if err2 := systemctl("reload", "sshd"); err2 != nil {

- return false, fmt.Errorf("%v; %v", err, err2)

+ return "", fmt.Errorf("sshd -T reported no permitrootlogin")

+// applyNoRootSSH converges the drop-in: write, validate the merged config,

+// reload sshd, then confirm sshd's EFFECTIVE permitrootlogin is "no". The

+// confirm runs even when the file was already correct, and it is the step

+// that matters: a correct drop-in can still lose sshd's first-match to an

+// earlier-sorting sshd_config.d file — kepler's 00-hardening.conf (2026-07-23)

+// made this verb report success while root login stayed enabled. A failed

+// `sshd -t` restores the prior state so a bad write can never leave config

+// sshd would refuse on its next real restart; a failed effective check leaves

+// the (inert, harmless) drop-in in place and errors with the cause. Reload

+// only re-reads config: existing sessions and named-account logins are

+// untouched — only new root logins are refused.

+func applyNoRootSSH(confPath string) (changed bool, err error) {

+ prev, prevErr := os.ReadFile(confPath)

+ if prevErr != nil || string(prev) != noRootConf {

+ if err := os.WriteFile(confPath, []byte(noRootConf), 0o644); err != nil {

+ return false, err

}

+ if err := sshdCheck(); err != nil {

+ if prevErr == nil {

+ os.WriteFile(confPath, prev, 0o644)

+ } else {

+ os.Remove(confPath)

+ }

+ return false, err

+ }

+ // Ubuntu names the unit ssh; other distros sshd.

+ if err := systemctl("reload", "ssh"); err != nil {

+ if err2 := systemctl("reload", "sshd"); err2 != nil {

+ return false, fmt.Errorf("%v; %v", err, err2)

+ }

+ changed = true

+ }

+ got, gerr := sshdRootLogin()

+ if gerr != nil {

+ return changed, fmt.Errorf("verify effective config: %v", gerr)

+ }

+ if got != "no" {

+ return changed, fmt.Errorf("drop-in in place but sshd's effective permitrootlogin is %q — sshd_config is first-match-wins, so a file sorting before %s (ls /etc/ssh/sshd_config.d) or a directive above the Include line is overriding it; rename the offender to sort after it and re-run", got, filepath.Base(confPath))

}

- return true, nil

+ return changed, nil

}

// runBootstrap installs the systemd units + runtime dir and starts the router, so

diff --git a/bootstrap_test.go b/bootstrap_test.go

index 5878513..1cba573 100644

--- a/bootstrap_test.go

+++ b/bootstrap_test.go

@@ -427,12 +427,14 @@ func TestNoRootDropIn(t *testing.T) {

}

// TestApplyNoRootSSH covers the converge flow with the exec seams stubbed:

-// fresh write reloads sshd, an already-correct file is a silent no-op, and a

+// fresh write reloads sshd, an already-correct file is a silent no-op, a

// failed validation restores exactly the prior state (file absent or prior

-// content) so sshd's next real restart can never hit config it would refuse.

+// content) so sshd's next real restart can never hit config it would refuse,

+// and a drop-in that fails the EFFECTIVE-config check errors on both the

+// fresh and idempotent paths (the kepler 00-hardening.conf shape).

func TestApplyNoRootSSH(t *testing.T) {

- oldCheck, oldSysctl := sshdCheck, systemctl

- defer func() { sshdCheck, systemctl = oldCheck, oldSysctl }()

+ oldCheck, oldSysctl, oldEff := sshdCheck, systemctl, sshdRootLogin

+ defer func() { sshdCheck, systemctl, sshdRootLogin = oldCheck, oldSysctl, oldEff }()

var reloads []string

systemctl = func(args ...string) error {

@@ -440,6 +442,7 @@ func TestApplyNoRootSSH(t *testing.T) {

return nil

}

sshdCheck = func() error { return nil }

+ sshdRootLogin = func() (string, error) { return "no", nil }

p := filepath.Join(t.TempDir(), "10-lchat-noroot.conf")

changed, err := applyNoRootSSH(p)

@@ -483,4 +486,23 @@ func TestApplyNoRootSSH(t *testing.T) {

if _, statErr := os.Stat(p2); !os.IsNotExist(statErr) {

t.Errorf("failed validation left the drop-in behind (stat err %v)", statErr)

}

+ // An earlier-sorting file overriding the drop-in (effective config stays

+ // prohibit-password): the fresh write must error — success here is the

+ // kepler bug — and the drop-in stays in place (inert, not harmful) for the

+ // re-run after the operator fixes the ordering.

+ sshdCheck = func() error { return nil }

+ sshdRootLogin = func() (string, error) { return "prohibit-password", nil }

+ p3 := filepath.Join(t.TempDir(), "10-lchat-noroot.conf")

+ if _, err = applyNoRootSSH(p3); err == nil || !strings.Contains(err.Error(), "first-match-wins") {

+ t.Fatalf("ineffective drop-in: err = %v, want first-match-wins explanation", err)

+ }

+ if b, _ := os.ReadFile(p3); string(b) != noRootConf {

+ t.Errorf("ineffective drop-in should stay in place, got %q", b)

+ }

+ // Same on the idempotent path: an already-correct file that isn't winning

+ // must not report success.

+ if _, err = applyNoRootSSH(p3); err == nil {

+ t.Fatal("idempotent apply over an ineffective drop-in should error")

+ }

}

diff --git a/deploy/README.md b/deploy/README.md

index 420e8ba..c00eb15 100644

--- a/deploy/README.md

+++ b/deploy/README.md

@@ -93,8 +93,9 @@ ssh root@<box> '/opt/lchat/lchat add-instance chat.example.com'

# and deploy/lchat-env-install + deploy/lchat-svc -> /usr/local/sbin/ (0755),

# add each operator's WireGuard peer, close public 22, and run

# `lchat bootstrap --ssh-noroot` — writes the PermitRootLogin-no drop-in

-# (/etc/ssh/sshd_config.d/10-lchat-noroot.conf; the 10- prefix wins sshd's

-# first-match over the 50-hardening.conf default), validates, reloads sshd.

+# (/etc/ssh/sshd_config.d/10-lchat-noroot.conf), validates, reloads sshd,

+# and confirms the EFFECTIVE config reports no (sshd is first-match-wins;

+# a pre-existing drop-in sorting before 10- silently wins otherwise).

# It refuses until lchat-operators has members, and every later full

# bootstrap converges the same drop-in — a box that missed the step picks

# it up on the next provision pass, root shell not required.

diff --git a/docs/instance-router.md b/docs/instance-router.md

index 580dabc..254987d 100644

--- a/docs/instance-router.md

+++ b/docs/instance-router.md

@@ -677,10 +677,14 @@ else binds a unix socket and needs no capabilities (`CapabilityBoundingSet=`).

`/usr/local/sbin/lchat-env-install` (mode `0755`), add the WireGuard peers —

and then disable root logins: `lchat bootstrap --ssh-noroot` writes the

drop-in `/etc/ssh/sshd_config.d/10-lchat-noroot.conf` (`PermitRootLogin

- no`), validates with `sshd -t`, and reloads sshd. The `10-` prefix matters:

- sshd_config is first-match-wins, so the drop-in must sort *before* the

- box's `50-hardening.conf` (which sets `prohibit-password`) to take effect —

- confirm with `sshd -T | grep permitrootlogin` before trusting the flip.

+ no`), validates with `sshd -t`, reloads sshd, and then **confirms the

+ effective config** (`sshd -T`) actually reports `no` — failing loudly when

+ it doesn't. That last check exists because sshd_config is first-match-wins

+ and a box hardened before this verb existed can carry a drop-in that sorts

+ *before* `10-` (kepler's `00-hardening.conf`, found 2026-07-23, silently

+ kept `prohibit-password`); the fix is renaming the offender to sort after

+ `10-` (keeping its other directives, e.g. `PasswordAuthentication no`,

+ effective) and re-running the verb.

The verb refuses until the `lchat-operators` group has members (a fresh box

can't shut root out before anyone else can get in), and every full

`bootstrap` run converges the same drop-in once operators exist — so a box

diff --git a/docs/planet-scale-guide/operations.html b/docs/planet-scale-guide/operations.html

index 01cffb3..16325cf 100644

--- a/docs/planet-scale-guide/operations.html

+++ b/docs/planet-scale-guide/operations.html

@@ -124,7 +124,7 @@ sqlite3 /tmp/chat-a-snapshot.db "PRAGMA integrity_check"

<ul>

<li>Firewall: allow inbound 80, 443, and the WireGuard port (51820, UDP) only; default-deny the rest. SSH gets no public allow — it is permitted only on the WireGuard interface (<code>ufw allow in on wg0 to any port 22 proto tcp</code>). Everything internal is unix sockets, so nothing else needs a port. (Add the allows before enabling the firewall, and verify the tunnel from outside before removing any public SSH rule, so you do not lock out your own session.) running today</li>

<li>SSH rides WireGuard: each operator is a peer on the box’s <code>wg0</code> interface and connects to the box’s tunnel address; the tunnel runs persistently on the operator’s machine with an <code>ssh</code> configuration alias, so every existing command works unchanged. WireGuard does not answer unauthenticated packets, so the SSH port is invisible to scanners and brute-force noise drops to zero. Key-only remains in force (<code>PasswordAuthentication no</code>). The full recipe, including the break-glass path through the hosting provider’s rescue console, is in <code>docs/instance-router.md</code> under Hardening. running today</li>

- <li>Root SSH is bootstrap-only: the first root session bootstraps the box and creates named operator accounts (group <code>lchat-operators</code>), installs the committed sudo allowlist (<code>deploy/sudoers-operators</code> → <code>/etc/sudoers.d/lchat-operators</code>) and the <code>lchat-env-install</code> helper for placing secrets, then sets <code>PermitRootLogin no</code> with <code>lchat bootstrap --ssh-noroot</code> (refused until operator accounts exist, so a fresh box cannot shut root out before anyone else can get in; every full <code>bootstrap</code> run converges the same drop-in once they do, so a box that missed the step picks it up on the next provision pass — no root shell needed). From then on operators are named users whose root capability is exactly the allowlist — anything new that needs root gets an allowlist line through a reviewed pull request, never a root shell. This is accident-proofing and attribution with a reviewable surface, not a hard security boundary (the allowlist can deploy binaries); the hard boundary is the WireGuard key. running today</li>

+ <li>Root SSH is bootstrap-only: the first root session bootstraps the box and creates named operator accounts (group <code>lchat-operators</code>), installs the committed sudo allowlist (<code>deploy/sudoers-operators</code> → <code>/etc/sudoers.d/lchat-operators</code>) and the <code>lchat-env-install</code> helper for placing secrets, then sets <code>PermitRootLogin no</code> with <code>lchat bootstrap --ssh-noroot</code> (refused until operator accounts exist, so a fresh box cannot shut root out before anyone else can get in; every full <code>bootstrap</code> run converges the same drop-in once they do, so a box that missed the step picks it up on the next provision pass — no root shell needed). The verb confirms the change actually took effect (<code>sshd -T</code>) and fails loudly when an earlier-sorting <code>sshd_config.d</code> file overrides the drop-in, rather than reporting a lockdown that is not real. From then on operators are named users whose root capability is exactly the allowlist — anything new that needs root gets an allowlist line through a reviewed pull request, never a root shell. This is accident-proofing and attribution with a reviewable surface, not a hard security boundary (the allowlist can deploy binaries); the hard boundary is the WireGuard key. running today</li>

<li>Automatic security updates for the operating system (<code>unattended-upgrades</code>) — the application updates itself (<a href="deploys.html#auto-updates">part 04</a>), but the kernel and OpenSSL do not.</li>

<li>Pin the box’s own hostnames to loopback in <code>/etc/hosts</code>, so on-box clients (bot sidecars, calls to the push relay) reach the local router instead of the public internet — and disable any cloud-image behaviour that regenerates that file on boot. The subtle failure this prevents is described in <a href="processes.html#bot-sidecars">part 03</a>.</li>

<li>The unit files already sandbox every service (<a href="processes.html#security-model">part 03</a>) — nothing to do, but worth knowing when something cannot write where you expected: <code>ProtectSystem=strict</code> means the whole filesystem is read-only to services except <code>/data</code> and <code>/run/lchat</code>.</li>