eleven / ocho Public

server: ops provision operator — add an operator account with no root shell (#1392)

Operator-account creation was only ever the hand-run stage of a box's one
root session, so a box hardened before an operator joined had no path to
add them: kepler carries James's WireGuard peer (live handshake) but no
james account. New `lchat bootstrap --add-operator <name>` converges the
account + groups (lchat-operators, systemd-journal) + authorized_keys
(replaced from stdin, so re-running is also key rotation), reachable via
the existing `/opt/lchat/lchat *` sudoers line — no allowlist change.
`ops provision operator <name> <box> (--github <handle> | --keys-file
<path>)` is the front door; keys are validated before any account command
runs, since a clobbered authorized_keys with root SSH off is a
console/rescue trip. Docs: instance-router § Hardening + the fleet guide.
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 75eee62574f120a87e59b74b1e2c780bbcf23493 parent c647429
6 files changed, +282 −8
  • bootstrap.go +133 −0
  • bootstrap_test.go +93 −0
  • deploy/ops +3 −2
  • deploy/ops-provision.sh +43 −4
  • docs/instance-router.md +9 −1
  • docs/planet-scale-guide/operations.html +1 −1
diff --git a/bootstrap.go b/bootstrap.go
index 56c08da..6bca373 100644
--- a/bootstrap.go
+++ b/bootstrap.go
@@ -4,10 +4,13 @@ import (
"embed"
"flag"
"fmt"
+ "io"
"os"
"os/exec"
"os/user"
"path/filepath"
+ "regexp"
+ "strconv"
"strings"
"time"
)
@@ -548,6 +551,125 @@ func applyNoRootSSH(confPath string) (changed bool, err error) {
return changed, nil
}
+// --- operator accounts -------------------------------------------------------
+
+// The account half of the hardening recipe's operator-setup stage
+// (docs/instance-router.md § Hardening), encoded because a box hardened before
+// an operator joined it has no root shell left to add them with — kepler
+// carried James's WireGuard peer but no account (2026-07-29). The allowlisted
+// `sudo -n /opt/lchat/lchat bootstrap --add-operator <name>` over the tunnel
+// is the remaining root-capable path; the keys ride stdin so the committed
+// sudoers line (`/opt/lchat/lchat *`) already covers it.
+
+// Conservative unix account names only — this lands in useradd's argv.
+var operatorNameRe = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)
+
+// countAuthorizedKeys confirms the piped bytes LOOK like an authorized_keys
+// file and counts the keys. Installing an empty or wrong file as someone's
+// login credentials must fail loudly here — with root SSH off, a clobbered
+// key file is a console/rescue trip.
+func countAuthorizedKeys(b []byte) (int, error) {
+ n := 0
+ for _, line := range strings.Split(string(b), "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+ f := strings.Fields(line)
+ if len(f) < 2 || !(strings.HasPrefix(f[0], "ssh-") || strings.HasPrefix(f[0], "ecdsa-") || strings.HasPrefix(f[0], "sk-")) {
+ return 0, fmt.Errorf("stdin line %q is not a public key", line)
+ }
+ n++
+ }
+ if n == 0 {
+ return 0, fmt.Errorf("no keys on stdin — pipe the operator's authorized_keys content in")
+ }
+ return n, nil
+}
+
+// runAccountCmd is the exec seam for the user/group commands — a var so tests
+// record instead of running (the chownToService pattern).
+var runAccountCmd = func(name string, args ...string) error {
+ if out, err := exec.Command(name, args...).CombinedOutput(); err != nil {
+ return fmt.Errorf("%s %s: %v: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+ }
+ return nil
+}
+
+// lookupAccount is the user-database seam: home dir + numeric ids, or
+// exists=false. A var for the same reason.
+var lookupAccount = func(name string) (home string, uid, gid int, exists bool) {
+ u, err := user.Lookup(name)
+ if err != nil {
+ return "", 0, 0, false
+ }
+ uid, _ = strconv.Atoi(u.Uid)
+ gid, _ = strconv.Atoi(u.Gid)
+ return u.HomeDir, uid, gid, true
+}
+
+// addOperatorAccount converges one named operator: the account, its groups
+// (lchat-operators carries the sudo allowlist, systemd-journal the log
+// reads), and ~/.ssh/authorized_keys REPLACED by the piped keys —
+// replacement, not append, so re-running with a fresh key set is also the
+// rotation path. Idempotent like the rest of bootstrap.
+func addOperatorAccount(name string, keys io.Reader) error {
+ if !operatorNameRe.MatchString(name) {
+ return fmt.Errorf("invalid operator name %q (want lowercase, [a-z][a-z0-9-]{0,31})", name)
+ }
+ b, err := io.ReadAll(io.LimitReader(keys, 1<<20))
+ if err != nil {
+ return err
+ }
+ nkeys, err := countAuthorizedKeys(b)
+ if err != nil {
+ return err
+ }
+ // -f converges on a box whose operator-setup stage already made the group.
+ if err := runAccountCmd("groupadd", "-f", "lchat-operators"); err != nil {
+ return err
+ }
+ _, _, _, existed := lookupAccount(name)
+ if !existed {
+ if err := runAccountCmd("useradd", "-m", "-s", "/bin/bash", name); err != nil {
+ return err
+ }
+ }
+ if err := runAccountCmd("usermod", "-aG", "lchat-operators,systemd-journal", name); err != nil {
+ return err
+ }
+ home, uid, gid, ok := lookupAccount(name)
+ if !ok {
+ return fmt.Errorf("account %s missing after create", name)
+ }
+ sshDir := filepath.Join(home, ".ssh")
+ if err := os.MkdirAll(sshDir, 0o700); err != nil {
+ return err
+ }
+ if err := os.Chmod(sshDir, 0o700); err != nil {
+ return err
+ }
+ akPath := filepath.Join(sshDir, "authorized_keys")
+ if err := os.WriteFile(akPath, b, 0o600); err != nil {
+ return err
+ }
+ for _, p := range []string{sshDir, akPath} {
+ if err := os.Chown(p, uid, gid); err != nil {
+ return err
+ }
+ }
+ verb := "converged"
+ if !existed {
+ verb = "created"
+ }
+ noun := "keys"
+ if nkeys == 1 {
+ noun = "key"
+ }
+ fmt.Printf("bootstrap: operator %s %s — groups lchat-operators+systemd-journal, %d %s installed\n", name, verb, nkeys, noun)
+ return nil
+}
+
// runBootstrap installs the systemd units + runtime dir and starts the router, so
// provisioning a fresh box is one command after the binary is copied to
// /opt/lchat/lchat. Idempotent — safe to re-run. See docs/instance-router.md.
@@ -560,8 +682,19 @@ func runBootstrap(args []string) {
linksApex := fs.String("links-apex", "", "make this box the short-link apex for this host (starts the redirector and routes the host at it); without it, a box that isn't already routed as the apex doesn't run the redirector at all")
noUpdateTimer := fs.Bool("no-update-timer", false, "ensure the hourly auto-updater is DISABLED instead of enabled (for boxes that must never track a release channel, e.g. the dev mesh)")
sshNoRoot := fs.Bool("ssh-noroot", false, "ONLY converge the root-SSH lockdown (PermitRootLogin no drop-in + sshd reload) — refused until the lchat-operators group has members")
+ addOperator := fs.String("add-operator", "", "ONLY create/converge the named operator account (groups lchat-operators+systemd-journal; ~/.ssh/authorized_keys REPLACED from stdin) — the allowlist-reachable form of the operator-setup stage")
_ = fs.Parse(args)
+ // --add-operator is operator-setup for a box that already shut root out:
+ // an existing operator adds the next one over the tunnel, keys on stdin
+ // (`ops provision operator` is the front door).
+ if *addOperator != "" {
+ if err := addOperatorAccount(*addOperator, os.Stdin); err != nil {
+ die("bootstrap: --add-operator: %v", err)
+ }
+ return
+ }
+
// --ssh-noroot is the scoped, allowlist-reachable form of the hardening
// recipe's last step: on a box where root SSH is already unreachable (keys
// pulled) but sshd still says prohibit-password, an operator flips the
diff --git a/bootstrap_test.go b/bootstrap_test.go
index 1cba573..fa06ea7 100644
--- a/bootstrap_test.go
+++ b/bootstrap_test.go
@@ -506,3 +506,96 @@ func TestApplyNoRootSSH(t *testing.T) {
t.Fatal("idempotent apply over an ineffective drop-in should error")
}
}
+
+func TestCountAuthorizedKeys(t *testing.T) {
+ cases := []struct {
+ name string
+ in string
+ n int
+ ok bool
+ }{
+ {"one ed25519", "ssh-ed25519 AAAAC3Nza james@mac\n", 1, true},
+ {"two with comments and blanks", "# ci key\nssh-ed25519 AAAA a\n\nsk-ssh-ed25519@openssh.com BBBB b\n", 2, true},
+ {"ecdsa", "ecdsa-sha2-nistp256 AAAA\n", 1, true},
+ {"empty", "", 0, false},
+ {"whitespace only", "\n \n", 0, false},
+ {"junk line", "ssh-ed25519 AAAA ok\nnot a key\n", 0, false},
+ {"missing key body", "ssh-ed25519\n", 0, false},
+ }
+ for _, c := range cases {
+ n, err := countAuthorizedKeys([]byte(c.in))
+ if (err == nil) != c.ok || n != c.n {
+ t.Errorf("%s: n=%d err=%v, want n=%d ok=%v", c.name, n, err, c.n, c.ok)
+ }
+ }
+}
+
+// TestAddOperatorAccount drives the whole verb through the exec/user seams:
+// the create path must run useradd, the converge path must not, and a bad
+// keys pipe must fail BEFORE any account command runs — with root SSH off, a
+// clobbered authorized_keys is a console/rescue trip.
+func TestAddOperatorAccount(t *testing.T) {
+ oldRun, oldLookup := runAccountCmd, lookupAccount
+ defer func() { runAccountCmd, lookupAccount = oldRun, oldLookup }()
+
+ home := t.TempDir()
+ exists := false
+ var calls []string
+ runAccountCmd = func(name string, args ...string) error {
+ calls = append(calls, name+" "+strings.Join(args, " "))
+ if name == "useradd" {
+ exists = true
+ }
+ return nil
+ }
+ lookupAccount = func(string) (string, int, int, bool) {
+ return home, os.Getuid(), os.Getgid(), exists
+ }
+
+ keys := "ssh-ed25519 AAAA james@mac\nssh-ed25519 BBBB james@laptop\n"
+ if err := addOperatorAccount("james", strings.NewReader(keys)); err != nil {
+ t.Fatalf("create: %v", err)
+ }
+ want := []string{"groupadd -f lchat-operators", "useradd -m -s /bin/bash james",
+ "usermod -aG lchat-operators,systemd-journal james"}
+ if strings.Join(calls, ";") != strings.Join(want, ";") {
+ t.Errorf("create calls = %v, want %v", calls, want)
+ }
+ ak := filepath.Join(home, ".ssh", "authorized_keys")
+ if b, err := os.ReadFile(ak); err != nil || string(b) != keys {
+ t.Errorf("authorized_keys = %q, %v", b, err)
+ }
+ for p, want := range map[string]os.FileMode{filepath.Join(home, ".ssh"): 0o700, ak: 0o600} {
+ if fi, err := os.Stat(p); err != nil || fi.Mode().Perm() != want {
+ t.Errorf("%s mode = %v, want %v", p, fi.Mode().Perm(), want)
+ }
+ }
+
+ // Converge an existing account with a rotated key set: no useradd, and
+ // the file is REPLACED, not appended.
+ calls = nil
+ rotated := "ssh-ed25519 CCCC james@new\n"
+ if err := addOperatorAccount("james", strings.NewReader(rotated)); err != nil {
+ t.Fatalf("converge: %v", err)
+ }
+ for _, c := range calls {
+ if strings.HasPrefix(c, "useradd") {
+ t.Errorf("converge ran useradd: %v", calls)
+ }
+ }
+ if b, _ := os.ReadFile(ak); string(b) != rotated {
+ t.Errorf("rotated authorized_keys = %q, want replacement", b)
+ }
+
+ // Bad pipes and bad names fail before any account command.
+ calls = nil
+ if err := addOperatorAccount("james", strings.NewReader("")); err == nil {
+ t.Error("empty stdin should error")
+ }
+ if err := addOperatorAccount("James Adam", strings.NewReader(rotated)); err == nil {
+ t.Error("invalid account name should error")
+ }
+ if len(calls) != 0 {
+ t.Errorf("failed runs still executed commands: %v", calls)
+ }
+}
diff --git a/deploy/ops b/deploy/ops
index d7c5013..af9848b 100755
--- a/deploy/ops
+++ b/deploy/ops
@@ -19,8 +19,9 @@ ops — the operator command surface
./deploy/ops doctor [section] can I deploy right now? (read-only preflight)
./deploy/ops validate [section] run every local check (go|web|shell|infra)
- ./deploy/ops provision <units|apex|status> [args]
- refresh box systemd units / set the short-link apex
+ ./deploy/ops provision <units|apex|status|operator> [args]
+ refresh box systemd units / set the short-link apex /
+ create an operator account on a box
./deploy/ops publish "notes" build main -> release -> point edge at it
./deploy/ops promote <ver> <chan> move beta/stable to a published version
./deploy/ops ship [box...] make every box's instances catch up NOW
diff --git a/deploy/ops-provision.sh b/deploy/ops-provision.sh
index ff2544a..83568e9 100755
--- a/deploy/ops-provision.sh
+++ b/deploy/ops-provision.sh
@@ -5,6 +5,8 @@
# ./deploy/ops provision units [box...] # refresh unit files (lchat bootstrap)
# ./deploy/ops provision apex <host> [box] # ...and make that box the apex for <host>
# ./deploy/ops provision status [box...] # read-only: who runs what
+# ./deploy/ops provision operator <name> <box> (--github <handle> | --keys-file <path>)
+# # create/converge an operator account
#
# Why this verb exists: **no new systemd unit ever reaches an existing box on
# its own.** `deploy` and `ship` swap binaries and restart units that already
@@ -45,11 +47,13 @@ APEX_BOX="${LCHAT_APEX_BOX:-elevenbox}"
usage() {
echo "usage: ops provision units [box...] | ops provision apex <host> [box] |" >&2
- echo " ops provision status [box...]" >&2
+ echo " ops provision status [box...] |" >&2
+ echo " ops provision operator <name> <box> (--github <handle> | --keys-file <path>)" >&2
echo >&2
- echo " units re-run 'lchat bootstrap' so newly-added systemd units land (restarts the router)" >&2
- echo " apex the same, declaring that box the short-link apex for <host>" >&2
- echo " status read-only: unit + route state, no changes" >&2
+ echo " units re-run 'lchat bootstrap' so newly-added systemd units land (restarts the router)" >&2
+ echo " apex the same, declaring that box the short-link apex for <host>" >&2
+ echo " status read-only: unit + route state, no changes" >&2
+ echo " operator create/converge a named operator account on <box> (bootstrap --add-operator)" >&2
exit 2
}
@@ -146,6 +150,41 @@ status)
for box in $boxes; do report "$box"; done
;;
+# The later-joiner half of operator setup (docs/instance-router.md §
+# Hardening): the first operators were created in the box's one root
+# session, and a box hardened before an operator joined has no root shell
+# left — kepler had James's WireGuard peer but no account. This drives the
+# allowlist-reachable `bootstrap --add-operator`, keys piped on stdin.
+# The account is only HALF of access: the operator's WireGuard peer must
+# already be on the box's wg0 (see RUNBOOK § Box access), or nothing routes.
+operator)
+ name="${1:-}"; box="${2:-}"; src="${3:-}"; arg="${4:-}"
+ [ -n "$name" ] && [ -n "$box" ] && [ -n "$arg" ] || usage
+ case "$name" in
+ *[!a-z0-9-]*|-*) echo "ops provision: operator name must be lowercase [a-z0-9-]" >&2; exit 1 ;;
+ esac
+ case "$src" in
+ --github) keys="$(curl -fsS "https://github.com/$arg.keys")" ;;
+ --keys-file) keys="$(cat "$arg")" ;;
+ *) usage ;;
+ esac
+ [ -n "$keys" ] || { echo "ops provision: no keys from $src $arg" >&2; exit 1; }
+ echo "ops provision: installing for '$name' on $box:"
+ printf '%s\n' "$keys" | awk '{print " " $1 " ..." substr($2, length($2)-7) " " $3}'
+ # Public keys fetched from GitHub are an INTAKE, not an identity proof —
+ # same caveat as the vault recipients: confirm over the messenger that the
+ # keys are currently theirs.
+ [ "$src" = "--github" ] && echo " (from github.com/$arg.keys — confirm with them over the messenger that these are current)"
+ printf '%s\n' "$keys" | on "$box" "sudo /opt/lchat/lchat bootstrap --add-operator $name"
+ echo "ops provision: verifying:"
+ on "$box" "id $name"
+ echo
+ echo "note: their WireGuard peer must already be on $box's wg0 for SSH to route"
+ echo " (sudo wg show wg0 — see RUNBOOK § Box access to add one)."
+ echo "note: this opened box SSH sessions — if this session hasn't written its"
+ echo " access-ledger why-entry yet: ./deploy/ops ledger \"...\""
+ ;;
+
-h|--help|help|"") usage ;;
*) echo "ops provision: unknown subcommand '$cmd'" >&2; usage ;;
esac
diff --git a/docs/instance-router.md b/docs/instance-router.md
index 254987d..2db2844 100644
--- a/docs/instance-router.md
+++ b/docs/instance-router.md
@@ -693,7 +693,15 @@ else binds a unix socket and needs no capabilities (`CapabilityBoundingSet=`).
Thereafter operators are named users over the tunnel; root capability exists
only through the allowlist, and anything new that needs root gets a line
added **via a reviewed PR + redeploy**, never a root shell or a hand-edit on
- the box. Two small helpers back the allowlist, because a couple of routine
+ the box.
+ **Operators who join after the lockdown** are added the same no-root-shell
+ way: an existing operator runs `ops provision operator <name> <box>
+ (--github <handle> | --keys-file <path>)`, which drives the allowlisted
+ `bootstrap --add-operator <name>` with the keys piped on stdin — account +
+ groups + `authorized_keys` (replaced, so re-running is also key rotation).
+ The account is only half of access: the joiner's WireGuard peer must be on
+ the box's `wg0` too, or nothing routes. (Encoded after kepler was found
+ carrying an operator's peer but no account, 2026-07-29.) Two small helpers back the allowlist, because a couple of routine
privileged ops don't fit a fixed command line — install both alongside the
sudoers file (`/usr/local/sbin/…`, mode `0755`):
- **`deploy/lchat-env-install`** places a secret:
diff --git a/docs/planet-scale-guide/operations.html b/docs/planet-scale-guide/operations.html
index 16325cf..cc47717 100644
--- a/docs/planet-scale-guide/operations.html
+++ b/docs/planet-scale-guide/operations.html
@@ -124,7 +124,7 @@ sqlite3 /tmp/chat-a-snapshot.db "PRAGMA integrity_check"
<ul>
<li><strong>Firewall:</strong> allow inbound 80, 443, and the WireGuard port (51820, UDP) only; default-deny the rest. SSH gets no public allow — it is permitted only <em>on the WireGuard interface</em> (<code>ufw allow in on wg0 to any port 22 proto tcp</code>). Everything internal is unix sockets, so nothing else needs a port. (Add the allows <em>before</em> enabling the firewall, and verify the tunnel from outside before removing any public SSH rule, so you do not lock out your own session.) <span class="status built">running today</span></li>
<li><strong>SSH rides WireGuard:</strong> each operator is a peer on the box&rsquo;s <code>wg0</code> interface and connects to the box&rsquo;s tunnel address; the tunnel runs persistently on the operator&rsquo;s machine with an <code>ssh</code> configuration alias, so every existing command works unchanged. WireGuard does not answer unauthenticated packets, so the SSH port is invisible to scanners and brute-force noise drops to zero. Key-only remains in force (<code>PasswordAuthentication no</code>). The full recipe, including the break-glass path through the hosting provider&rsquo;s rescue console, is in <code>docs/instance-router.md</code> under Hardening. <span class="status built">running today</span></li>
- <li><strong>Root SSH is bootstrap-only:</strong> the first root session bootstraps the box and creates named operator accounts (group <code>lchat-operators</code>), installs the committed sudo allowlist (<code>deploy/sudoers-operators</code> &rarr; <code>/etc/sudoers.d/lchat-operators</code>) and the <code>lchat-env-install</code> helper for placing secrets, then sets <code>PermitRootLogin no</code> with <code>lchat bootstrap --ssh-noroot</code> (refused until operator accounts exist, so a fresh box cannot shut root out before anyone else can get in; every full <code>bootstrap</code> run converges the same drop-in once they do, so a box that missed the step picks it up on the next provision pass — no root shell needed). The verb confirms the change actually took effect (<code>sshd -T</code>) and fails loudly when an earlier-sorting <code>sshd_config.d</code> file overrides the drop-in, rather than reporting a lockdown that is not real. From then on operators are named users whose root capability is exactly the allowlist — anything new that needs root gets an allowlist line through a reviewed pull request, never a root shell. This is accident-proofing and attribution with a reviewable surface, not a hard security boundary (the allowlist can deploy binaries); the hard boundary is the WireGuard key. <span class="status built">running today</span></li>
+ <li><strong>Root SSH is bootstrap-only:</strong> the first root session bootstraps the box and creates named operator accounts (group <code>lchat-operators</code>), installs the committed sudo allowlist (<code>deploy/sudoers-operators</code> &rarr; <code>/etc/sudoers.d/lchat-operators</code>) and the <code>lchat-env-install</code> helper for placing secrets, then sets <code>PermitRootLogin no</code> with <code>lchat bootstrap --ssh-noroot</code> (refused until operator accounts exist, so a fresh box cannot shut root out before anyone else can get in; every full <code>bootstrap</code> run converges the same drop-in once they do, so a box that missed the step picks it up on the next provision pass — no root shell needed). The verb confirms the change actually took effect (<code>sshd -T</code>) and fails loudly when an earlier-sorting <code>sshd_config.d</code> file overrides the drop-in, rather than reporting a lockdown that is not real. From then on operators are named users whose root capability is exactly the allowlist — anything new that needs root gets an allowlist line through a reviewed pull request, never a root shell. This is accident-proofing and attribution with a reviewable surface, not a hard security boundary (the allowlist can deploy binaries); the hard boundary is the WireGuard key. Operators who join <em>after</em> the lockdown are added the same no-root-shell way: <code>ops provision operator &lt;name&gt; &lt;box&gt;</code> drives the allowlisted <code>bootstrap --add-operator</code> over the tunnel, piping the joiner&rsquo;s public keys on stdin (account, groups and <code>authorized_keys</code> in one idempotent verb; re-running it with fresh keys is also the rotation path). The account is half of access — the joiner&rsquo;s WireGuard peer must be on the box&rsquo;s <code>wg0</code> too. <span class="status built">running today</span></li>
<li><strong>Automatic security updates</strong> for the operating system (<code>unattended-upgrades</code>) — the application updates itself (<a href="deploys.html#auto-updates">part 04</a>), but the kernel and OpenSSL do not.</li>
<li><strong>Pin the box&rsquo;s own hostnames to loopback</strong> in <code>/etc/hosts</code>, so on-box clients (bot sidecars, calls to the push relay) reach the local router instead of the public internet — and disable any cloud-image behaviour that regenerates that file on boot. The subtle failure this prevents is described in <a href="processes.html#bot-sidecars">part 03</a>.</li>
<li><strong>The unit files already sandbox every service</strong> (<a href="processes.html#security-model">part 03</a>) — nothing to do, but worth knowing when something cannot write where you expected: <code>ProtectSystem=strict</code> means the whole filesystem is read-only to services except <code>/data</code> and <code>/run/lchat</code>.</li>