amadan

carlos / platform Public

Merge remote-tracking branch 'origin/main' into worktree-carlos-deploy

Paul Campbell 79b484892cb802677329ec5d6e75265dc34a5619 parent bd43b87parent 32dec0b
8 files changed, +912 −54
  • cmd/console/main.go +34 −13
  • docs/superpowers/specs/2026-08-08-instance-socket-consumers-design.md +161 −0
  • internal/activator/activator_test.go +197 −0
  • internal/activator/backend_exec.go +28 −0
  • internal/console/server.go +84 −1
  • internal/console/server_test.go +165 −0
  • internal/host/bootstrap.go +144 −33
  • internal/host/host_test.go +99 −7
diff --git a/cmd/console/main.go b/cmd/console/main.go
index c91aabf00..467268d8d 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -33,10 +33,13 @@
// and optionally CARLOS_RELEASE_KEY (enables promote/rollback from the
// console; unset = read-only), CARLOS_SECRETS_PUBKEY (SERVED to CLI
// clients so `carlos secrets set` can seal with a login alone; never used
-// here) and PLATFORM_APPS_DOMAIN (the same variable the edge and agent
-// read; enables the CLI's member-facing `carlos instances enable`, whose
-// manifest domain is pinned to <sqid>.<this domain>). grant/revoke/owners
-// need only CONSOLE_DB.
+// here), PLATFORM_APPS_DOMAIN (the same variable the edge and agent read;
+// enables the CLI's member-facing `carlos instances enable`, whose manifest
+// domain is pinned to <sqid>.<this domain>), and CONSOLE_REDIRECT_HOSTS
+// (comma-separated retired hostnames this console 301/308s to CONSOLE_ORIGIN
+// — opt-in per host, so a host the API must keep answering on, like
+// platform.carlosframework.com for CLI tokens, is simply left off).
+// grant/revoke/owners need only CONSOLE_DB.
package main
import (
@@ -223,15 +226,16 @@ func serve() {
}
st := openConsoleDB()
srv, err := console.NewServer(console.Config{
- Origin: origin,
- Keymail: envOr("CONSOLE_KEYMAIL", "https://keymail.dev"),
- Registry: envOr("CARLOS_REGISTRY", "/data/registry.db"),
- Releases: openStore(),
- ReleasePub: pub,
- Key: key,
- EdgeIPs: edgeIPs(),
- SecretsPub: secretsPub(),
- AppsDomain: appsDomain(),
+ Origin: origin,
+ Keymail: envOr("CONSOLE_KEYMAIL", "https://keymail.dev"),
+ Registry: envOr("CARLOS_REGISTRY", "/data/registry.db"),
+ Releases: openStore(),
+ ReleasePub: pub,
+ Key: key,
+ EdgeIPs: edgeIPs(),
+ SecretsPub: secretsPub(),
+ AppsDomain: appsDomain(),
+ RedirectHosts: redirectHosts(),
}, st)
if err != nil {
log.Fatal(err)
@@ -342,6 +346,23 @@ func edgeIPs() []string {
return out
}
+// redirectHosts parses CONSOLE_REDIRECT_HOSTS (comma-separated) — retired
+// hostnames this console answers with a permanent redirect to CONSOLE_ORIGIN
+// (server.go's ServeHTTP), rather than serving the app in place. Opt-in per
+// host on purpose: a host that must keep serving the API — CLI bearer
+// tokens are host-agnostic, and a POST redirected through an unexpected hop
+// is how clients break — is simply left off this list, unlike EdgeIPs above
+// there's no single deployment-wide default to fall back to.
+func redirectHosts() []string {
+ var out []string
+ for _, h := range strings.Split(os.Getenv("CONSOLE_REDIRECT_HOSTS"), ",") {
+ if h = strings.TrimSpace(h); h != "" {
+ out = append(out, h)
+ }
+ }
+ return out
+}
+
func envOr(k, def string) string {
if v := os.Getenv(k); v != "" {
return v
diff --git a/docs/superpowers/specs/2026-08-08-instance-socket-consumers-design.md b/docs/superpowers/specs/2026-08-08-instance-socket-consumers-design.md
new file mode 100644
index 000000000..08616e3a9
--- /dev/null
+++ b/docs/superpowers/specs/2026-08-08-instance-socket-consumers-design.md
@@ -0,0 +1,161 @@
+# Instance-socket consumers (issue #103) — design
+
+Date: 2026-08-08. From issue #103: on the tito box, titogo's router (running
+as user `titogo`) proxies root-domain surfaces onward to platform-provisioned
+instances' unix sockets (`/run/carlos/<host>.sock`) and got
+`connect: permission denied` the moment a reconciler-provisioned instance
+woke. Two layers blocked the cross-user dial: `/run/carlos` is
+`0750 carlos:carlos` with no story for enrolling a consumer, and the socket's
+mode is whatever the app's umask/chmod habits produce (titogo bound 0600).
+The interim (`usermod -aG carlos titogo` + hand-chmod per wake) works but
+doesn't survive a rebind — and the failure is a buyer-facing 502 on an
+otherwise healthy instance. Autonomous-session design: the issue offered two
+directions and left the choice to the platform; both halves below were chosen
+inline, with the trade-offs recorded.
+
+## Decision 1: a dedicated `carlos-sockets` group, never group `carlos`
+
+The tito interim enrolled the consumer into group `carlos`. That works for
+traversal, but group `carlos` is also the group on `/data` (`0750
+carlos:carlos`) — membership grants group-read of every instance database and
+every `<host>.env`, including each instance's `CARLOS_ADMIN_TOKEN`. A grant
+whose meaning is "may dial instance sockets" must not silently carry "may
+read every instance's secrets".
+
+So the blessed consumers group is a dedicated system group,
+**`carlos-sockets`**:
+
+- `carlos bootstrap` creates it (`groupadd --system`, idempotent, live root
+ only — a staging root prints the command, same as `useradd`).
+- `/run/carlos` becomes **`2750 carlos carlos-sockets`** — group changed to
+ the consumers group (members get `r-x`: traversal to the sockets), setgid
+ bit added so **every socket bound inside inherits the group**. That
+ inheritance is what makes the mode contract (decision 2) sufficient: the
+ agent never needs to chgrp anything, and no app needs to know the group
+ exists. Both the tmpfiles.d entry and bootstrap's own dir handling change
+ together (they are the same truth in two places).
+- `/run/carlos/apps` stays **`0750 carlos carlos`**. Unit tenants get their
+ listener fd passed by systemd (`SocketUser=carlos`, 0600) and never need
+ path access — and consumer enrollment deliberately does not reach them.
+ A `carlos-sockets` member can traverse `/run/carlos` but not `apps/`.
+
+**Enrollment** is bootstrap's job, per the issue's ask: a repeatable
+`-socket-consumer <user>` flag runs `usermod -aG carlos-sockets <user>` on a
+live root (printed under a staging root). Idempotent — `usermod -aG` on an
+existing member is a no-op, so re-running bootstrap with the same flags
+converges. Consumer users are validated (`user.Lookup`) before any mutation:
+a typo leaves the box untouched rather than aborting bootstrap halfway.
+
+**Boot-ordering safety** (review finding, empirically verified):
+systemd-tmpfiles *drops* a line whose group doesn't resolve, and
+`carlos-edge.service` has `ReadWritePaths=/run/carlos` with no `-` prefix —
+so a tmpfiles line naming a group that doesn't exist yet turns the next
+reboot into a permanent edge-start failure. Two mechanisms close this:
+on a live root the privileged plan (including `groupadd`) runs *before* the
+tmpfiles conf is written, and bootstrap also ships
+`/etc/sysusers.d/carlos.conf` (`g carlos-sockets -`) —
+`systemd-sysusers.service` is ordered `Before=systemd-tmpfiles-setup.service`,
+so the group exists at boot even when the conf arrived via a staged-tree
+unit refresh that never ran a live bootstrap.
+
+**Re-run convergence** (review finding): every chown in the privileged plan
+converges *toward* the target state — `/run/carlos` is recursively
+`carlos:carlos-sockets` (so a re-run, the documented enrollment path, also
+converges live instance sockets' group instead of resetting it and
+stranding already-enrolled consumers), with `apps/` put back to
+`carlos:carlos` right after.
+
+**Scope of the grant, stated plainly:** membership in `carlos-sockets` is
+box-wide — a member may dial *every* exec-backed instance socket on the box.
+That is the intended semantic (the consumer is the deployment's own trusted
+host app, e.g. titogo's router), not a per-instance ACL; a deployment that
+needs per-instance isolation between consumers needs a different mechanism
+and should say so in an issue.
+
+## Decision 2: the agent normalizes the socket mode — the app contract is a
+guarantee, not a chore
+
+The issue's two options were "write 0660 into the app contract" or "the
+agent normalizes the mode after health". Docs-only loses to enforcement:
+the failure case that motivated this issue *was* an app's own chmod habit,
+and the exec'd child inherits the agent's umask either way — a contract no
+mechanism enforces is rediscovered per deployment, which is exactly the
+state #103 asks us out of.
+
+So: **`ExecBackend.Start` chmods the instance socket to `0660` immediately
+after the health wait succeeds.** Properties that make this the right spot:
+
+- It runs only on a socket the activator's own child bound — enforced, not
+ assumed: the health wait tolerates a squatter (it only dials), so `Start`
+ skips normalization when the child has already exited (the lost-bind-race
+ shape — that socket is the #17 external-socket advisory's territory), and
+ an `Lstat` confirms the path is a socket before the chmod, so a planted
+ symlink can never redirect the mode change elsewhere.
+- The agent and the child run as the same user, so the chmod is on a socket
+ we own; failure is essentially impossible, and is logged loudly rather
+ than failing the wake (a box with no consumers must not lose a healthy
+ wake to a normalization it doesn't need).
+- After health means after the app has bound (and possibly chmodded) the
+ socket — the normalization wins against any app habit. The window between
+ bind and normalization is the health-probe interval (milliseconds); a
+ consumer's dial in that window fails once and its retry succeeds.
+
+With decision 1's setgid dir, the normalized socket is
+`0660 carlos:carlos-sockets`: group members dial, everyone else is refused.
+On a box where bootstrap has never been re-run (no setgid, group absent),
+the socket's group is `carlos` and the chmod grants nothing new — the
+normalization is inert until the box converges, never a widening.
+
+The seed path binds no sockets; sidecars bind no sockets; the agent's own
+`.control` purge socket is bound with default umask (no group write) and
+stays effectively agent-only — none of them normalize.
+
+**The contract, stated for app authors** (alongside the seed contract): bind
+your instance socket at the path the platform passes via `--socket`, with
+whatever mode your runtime produces — the platform normalizes it to `0660`
+after your first health response. Do not tighten the mode after startup; a
+post-health chmod strands the deployment's sibling consumers.
+
+## What changes where
+
+- `internal/host/bootstrap.go` — `socketsGroup` const; `tmpfilesConf` line
+ for `/run/carlos` → `2750 carlos carlos-sockets`; live-run: create group,
+ `chgrp` + setgid `/run/carlos` (after the recursive `chown` to the service
+ user, which would otherwise reset the group); `-socket-consumer`
+ repeatable flag → `usermod -aG`; staging prints all of it.
+- `internal/activator/backend_exec.go` — the post-health `os.Chmod(rt.Socket,
+ 0o660)` in `Start`, log-on-error.
+- Tests: bootstrap staging asserts the new tmpfiles line and printed plan;
+ an ExecBackend test spawns a real child (test-binary re-exec) that binds
+ its socket, chmods it 0600 (the titogo habit), and serves — asserting the
+ socket is 0660 after `Start` returns.
+
+No registry, wire, or storage shape changes. No console surface.
+
+## Deployment notes
+
+Per box, after adopting a binary with this change. **Step 1 starts a
+consumer outage window** on a box using the interim grant: the moment
+`/run/carlos`'s group flips to `carlos-sockets`, a running consumer whose
+access came from group `carlos` (supplementary groups are fixed at process
+start) loses traversal — so run steps 1–3 as one uninterrupted sequence and
+the window is seconds.
+
+1. Re-run `carlos bootstrap -socket-consumer <user> ...` (live root). On
+ **tito**: enroll the titogo router/relay users (the ones titogo#535 put
+ in group `carlos`). On **flagship**: no consumers today; a plain re-run
+ just converges the dir. Bootstrap's recursive chown also converges the
+ group on every live instance socket.
+2. Converge live sockets' *mode* (group is already done):
+ `chmod g+rw /run/carlos/*.sock` — or skip and let each instance's next
+ wake normalize it.
+3. `systemctl restart` the consumer processes (picks up the new group
+ membership; ends the outage window).
+4. Only then revoke the interim grant: `gpasswd -d <user> carlos` per
+ enrolled user, plus a restart of each — the group-`carlos` membership
+ carries /data group-read (every instance DB and admin token) and the new
+ group replaces it.
+
+Follow-ups owed elsewhere: website docs batch (standing rule) — the
+app-author contract line belongs on the public app-contract page when that
+batch runs.
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
index d14d19014..e861d928a 100644
--- a/internal/activator/activator_test.go
+++ b/internal/activator/activator_test.go
@@ -3051,3 +3051,200 @@ func TestSeedCreatedTheDB(t *testing.T) {
})
}
}
+
+// --- Instance-socket mode normalization (issue #103) -------------------------
+
+// TestMain lets this test binary double as a fake instance app: Start execs
+// the live/<host> symlink, and stageServingLive points that symlink at the
+// test binary itself with ACTIVATOR_FAKE_APP=1 in the inherited environment
+// (childEnv strips only CARLOS_* vars, so it flows through to the child).
+// The shell-script recorders elsewhere in this file can't answer the health
+// probe — serving HTTP on a unix socket needs a real program, and the test
+// binary is the one program every `go test` run is guaranteed to have.
+func TestMain(m *testing.M) {
+ if os.Getenv("ACTIVATOR_FAKE_APP") == "1" {
+ fakeAppMain()
+ return
+ }
+ os.Exit(m.Run())
+}
+
+// fakeAppMain is the app half: bind the --socket, tighten it to 0600 — the
+// exact titogo habit that stranded the sibling router in issue #103 — and
+// serve 200s until killed (500s under ACTIVATOR_FAKE_APP_UNHEALTHY, the
+// never-becomes-healthy app).
+func fakeAppMain() {
+ var socket string
+ for i, a := range os.Args {
+ if a == "--socket" && i+1 < len(os.Args) {
+ socket = os.Args[i+1]
+ }
+ }
+ if socket == "" {
+ os.Exit(2)
+ }
+ ln, err := net.Listen("unix", socket)
+ if err != nil {
+ os.Exit(1)
+ }
+ if err := os.Chmod(socket, 0o600); err != nil {
+ os.Exit(1)
+ }
+ status := http.StatusOK
+ if os.Getenv("ACTIVATOR_FAKE_APP_UNHEALTHY") == "1" {
+ status = http.StatusInternalServerError
+ }
+ _ = http.Serve(ln, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(status)
+ }))
+}
+
+// stageServingLive points live/<host> at the test binary itself (see
+// TestMain) so Start spawns a child that actually binds and serves.
+func stageServingLive(t *testing.T, b *ExecBackend, host string) {
+ t.Helper()
+ self, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("ACTIVATOR_FAKE_APP", "1")
+ if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ os.Remove(b.liveLink(host))
+ if err := os.Symlink(self, b.liveLink(host)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestExecBackendStartNormalizesSocketMode pins the issue #103 contract: the
+// mode an instance socket carries after a successful wake is the PLATFORM's
+// (0660 — group members, i.e. enrolled carlos-sockets consumers, may dial),
+// never the app's own umask or chmod habit. The fake app deliberately
+// tightens its socket to 0600 after binding; Start must win.
+func TestExecBackendStartNormalizesSocketMode(t *testing.T) {
+ requireUnixSockets(t)
+ b := testExecBackend(t)
+ host := "mode.example.com"
+ stageServingLive(t, b, host)
+
+ rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ proc, _, err := b.Start(ctx, rt)
+ if err != nil {
+ t.Fatalf("Start: %v", err)
+ }
+ defer proc.Stop(context.Background())
+
+ fi, err := os.Stat(rt.Socket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode().Perm() != 0o660 {
+ t.Fatalf("socket mode after healthy Start = %o, want 660 — an app's own chmod must not strand a sibling consumer", fi.Mode().Perm())
+ }
+}
+
+// TestExecBackendFailedWakeDoesNotNormalizeSocketMode pins the other half of
+// the contract: normalization happens only AFTER a successful health wait —
+// a wake that fails must leave the socket exactly as the app left it. The
+// fake app binds and tightens to 0600 but never answers healthily.
+func TestExecBackendFailedWakeDoesNotNormalizeSocketMode(t *testing.T) {
+ requireUnixSockets(t)
+ b := testExecBackend(t)
+ host := "unhealthy.example.com"
+ stageServingLive(t, b, host)
+ t.Setenv("ACTIVATOR_FAKE_APP_UNHEALTHY", "1")
+
+ old := spawnHealthTimeout
+ spawnHealthTimeout = 500 * time.Millisecond
+ defer func() { spawnHealthTimeout = old }()
+
+ rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t)}
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if proc, _, err := b.Start(ctx, rt); err == nil {
+ defer proc.Stop(context.Background())
+ t.Fatal("Start succeeded for a never-healthy app")
+ }
+ fi, err := os.Stat(rt.Socket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode().Perm() != 0o600 {
+ t.Fatalf("socket mode after failed wake = %o, want the app's own 600 — a failed wake must not widen anything", fi.Mode().Perm())
+ }
+}
+
+// TestExecBackendDoesNotNormalizeSquatterSocket pins the only-our-child
+// guard: a stray supervisor squatting the route's socket (the #17 advisory's
+// subject) can answer the health probe while the spawned child — which lost
+// the bind race and exited — cannot. Start must not widen the squatter's
+// socket. The squatter here holds its 200 until the child has verifiably
+// exited, making the race the guard closes deterministic.
+func TestExecBackendDoesNotNormalizeSquatterSocket(t *testing.T) {
+ requireUnixSockets(t)
+ b := testExecBackend(t)
+ host := "squatted.example.com"
+ marker := filepath.Join(t.TempDir(), "child-ran")
+ // The child: proof-of-life marker, then immediate exit — the shape of a
+ // child that could not bind.
+ script := "#!/bin/sh\ntouch '" + marker + "'\nexit 0\n"
+ vdir := filepath.Join(b.OptDir, "versions", "v1")
+ if err := os.MkdirAll(vdir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ bin := filepath.Join(vdir, "app")
+ if err := os.WriteFile(bin, []byte(script), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(filepath.Join(b.OptDir, "live"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(bin, b.liveLink(host)); err != nil {
+ t.Fatal(err)
+ }
+
+ sock := tmpSock(t)
+ ln, err := net.Listen("unix", sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer ln.Close()
+ if err := os.Chmod(sock, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ // Answer only once the child is provably dead: marker written, plus
+ // a grace for the exit itself to be reaped.
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ if _, err := os.Stat(marker); err == nil {
+ break
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ time.Sleep(300 * time.Millisecond)
+ w.WriteHeader(http.StatusOK)
+ })}
+ go srv.Serve(ln)
+ defer srv.Close()
+
+ rt := registry.Route{Host: host, Kind: "instance", Socket: sock}
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ proc, _, err := b.Start(ctx, rt)
+ if err != nil {
+ t.Fatalf("Start: %v (the squatter answers healthily; the wake itself succeeds)", err)
+ }
+ defer proc.Stop(context.Background())
+
+ fi, err := os.Stat(rt.Socket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode().Perm() != 0o600 {
+ t.Fatalf("squatter socket mode after Start = %o, want its own 600 — a socket our child did not bind is never ours to widen", fi.Mode().Perm())
+ }
+}
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 87d27c613..c8514652e 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -488,6 +488,34 @@ func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, strin
_ = p.Stop(stopCtx)
return nil, "", err
}
+ // The instance-socket mode is the PLATFORM's, not the app's (issue #103):
+ // normalize to 0660 so a deployment's enrolled cross-user consumers (the
+ // carlos-sockets group — group-inherited from the setgid /run/carlos,
+ // see host.tmpfilesConf) can dial, no matter what the child's umask or
+ // its own post-bind chmod produced — titogo tightened its socket to 0600
+ // and stranded the sibling router that proxies to it. After the health
+ // wait, so the app's own bind/chmod has already happened and this wins.
+ //
+ // The two guards make "only a socket THIS backend's child bound" true
+ // rather than merely intended. waitHealthy tolerates a squatter (its own
+ // contract — it only dials), so a child that lost the bind race with a
+ // stray supervisor (EADDRINUSE, exits) can still see the health wait
+ // succeed off the SQUATTER's answers; a dead child means the socket is
+ // not ours to widen (that route is the #17 external-socket advisory's
+ // territory). The Lstat keeps the chmod from ever following a
+ // non-socket — a symlink planted at the route's path must not become a
+ // mode change somewhere else. On a box with no consumers group the
+ // socket's group is just `carlos`, so the wider mode grants nobody
+ // anything. All three branches log and never fail the wake: a box with
+ // no consumers must not lose a healthy wake to a normalization it
+ // doesn't need.
+ if p.Exited() {
+ log.Printf("activator: %s: child exited during the health wait (external process serving its socket?) — not normalizing the socket mode", host)
+ } else if fi, statErr := os.Lstat(rt.Socket); statErr != nil || fi.Mode()&os.ModeSocket == 0 {
+ log.Printf("activator: %s: %s is not a socket we can normalize (stat: %v)", host, rt.Socket, statErr)
+ } else if err := os.Chmod(rt.Socket, 0o660); err != nil {
+ log.Printf("activator: normalize socket mode for %s: %v (enrolled consumers would get permission denied)", host, err)
+ }
return p, spawnVer, nil
}
diff --git a/internal/console/server.go b/internal/console/server.go
index 072f5c222..b86b9b357 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -14,6 +14,7 @@ import (
"fmt"
"html/template"
"log"
+ "net"
"net/http"
"net/url"
"os"
@@ -178,6 +179,13 @@ type Config struct {
// does not exist, and the handler refuses with a pointer at the
// operator's store-mode `carlos instances enable`.
AppsDomain string
+ // RedirectHosts are request hosts this console answers with a permanent
+ // redirect to Origin (CONSOLE_REDIRECT_HOSTS, comma-separated) — retired
+ // hostnames after a canonical-origin move. Opt-in per host: hostnames
+ // that must keep serving the API in place (CLI bearer tokens are
+ // host-agnostic, and POSTs through an unexpected redirect are how
+ // clients break) simply aren't listed.
+ RedirectHosts []string
}
// Server is the console's http.Handler.
@@ -189,6 +197,9 @@ type Server struct {
instances *instanceFetcher
secure bool // Secure cookies iff the origin is https
cli *cliLimits // rate limits on the CLI-identity endpoints (authcli.go)
+ // redirects is cfg.RedirectHosts, normalized (lowercase, no port) into a
+ // lookup set — ServeHTTP checks it before the mux sees the request.
+ redirects map[string]bool
// cliNotified, when non-nil, is signalled after each CLI-token issuance
// notification attempt. Tests only — the send is off the request
// goroutine, and a test that asserted on it without this would be racing.
@@ -224,6 +235,33 @@ type Server struct {
// console's Keymail signing key.
func NewServer(cfg Config, st *Store) (*Server, error) {
cfg.Origin = strings.TrimSuffix(cfg.Origin, "/")
+ // originHost is what a RedirectHosts entry must never equal: redirecting
+ // the console's own origin to itself is a loop, not a retirement.
+ originHost := ""
+ if u, err := url.Parse(cfg.Origin); err == nil {
+ originHost = normalizeHost(u.Host)
+ }
+ // An origin with no host in it is fatal ONCE there are hosts to retire,
+ // never before: it is both the loop guard's only reference — an origin
+ // that parsed to nothing silently permits the very entry the guard exists
+ // to refuse — and the prefix every Location is built on, where "" would
+ // hand back the request's own path (a redirect to the retired host) or a
+ // scheme-relative "//host/…" whose destination the request chooses.
+ // Deployments with nothing to retire keep whatever origin they had.
+ if len(cfg.RedirectHosts) > 0 && originHost == "" {
+ return nil, fmt.Errorf("console: CONSOLE_REDIRECT_HOSTS is set, but CONSOLE_ORIGIN %q has no host to redirect to — it needs a scheme, e.g. https://console.example.com", cfg.Origin)
+ }
+ redirects := make(map[string]bool, len(cfg.RedirectHosts))
+ for _, h := range cfg.RedirectHosts {
+ h = normalizeHost(h)
+ if h == "" {
+ continue
+ }
+ if h == originHost {
+ return nil, fmt.Errorf("console: CONSOLE_REDIRECT_HOSTS lists %q, which is this console's own origin host — that config would redirect-loop", h)
+ }
+ redirects[h] = true
+ }
km := NewKeymail(cfg.Keymail, cfg.Origin)
key, err := st.SigningKey()
if err != nil {
@@ -238,6 +276,7 @@ func NewServer(cfg Config, st *Store) (*Server, error) {
instances: newInstanceFetcher(cfg.AppClient, cfg.AppScheme),
secure: strings.HasPrefix(cfg.Origin, "https://"),
cli: newCLILimits(),
+ redirects: redirects,
// Two concurrent ships at a time: roomy for one operator's CI plus a
// human running `carlos ship` by hand, tight enough that a fleet of
// slow-loris uploads against a stolen or shared token can't pile up
@@ -333,7 +372,51 @@ func NewServer(cfg Config, st *Store) (*Server, error) {
return s, nil
}
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.mux.ServeHTTP(w, r) }
+// ServeHTTP checks the host-based redirect BEFORE the mux on purpose: it's
+// purely host-based, touches no session/account state, so it sits in front
+// of the console's security gates without weakening them (the gates
+// ordering rule concerns account-existence leaks; a host-level redirect
+// reveals nothing).
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if s.redirects[normalizeHost(r.Host)] {
+ code := http.StatusMovedPermanently
+ if r.Method != http.MethodGet && r.Method != http.MethodHead {
+ code = http.StatusPermanentRedirect // 308 keeps the method
+ }
+ http.Redirect(w, r, s.cfg.Origin+redirectTarget(r.URL), code)
+ return
+ }
+ s.mux.ServeHTTP(w, r)
+}
+
+// redirectTarget is the path and query to hang off Origin: the request's own
+// RequestURI, but only where that is the rooted path a browser sends. A
+// request target may legally be rootless — `GET mailto:x@evil.com HTTP/1.1`
+// parses to an opaque URL whose RequestURI is "x@evil.com" — and pasting that
+// onto the origin spells https://console.example.com@evil.com, an open
+// redirect wearing the console's own hostname in the userinfo. The console
+// serves nothing at a rootless URL, so anything unrooted goes to the front
+// door instead.
+func redirectTarget(u *url.URL) string {
+ if uri := u.RequestURI(); strings.HasPrefix(uri, "/") {
+ return uri
+ }
+ return "/"
+}
+
+// normalizeHost reduces a request Host header (or a configured origin/
+// redirect host) to the bare, lowercase hostname the RedirectHosts
+// comparison keys on. net.SplitHostPort errors when there's no port to
+// strip — a bare hostname, or an unbracketed IPv6 literal — and then the
+// only thing left to remove is the brackets it would have taken off with the
+// port, so that "[::1]" and "[::1]:8080" key the same.
+func normalizeHost(host string) string {
+ host = strings.ToLower(strings.TrimSpace(host))
+ if h, _, err := net.SplitHostPort(host); err == nil {
+ return h
+ }
+ return strings.TrimSuffix(strings.TrimPrefix(host, "["), "]")
+}
// ---- pages ------------------------------------------------------------------
diff --git a/internal/console/server_test.go b/internal/console/server_test.go
index 7e2edace1..b069892d4 100644
--- a/internal/console/server_test.go
+++ b/internal/console/server_test.go
@@ -1992,3 +1992,168 @@ func TestFleetMetricsRendersFiringAlertsWithUnits(t *testing.T) {
t.Errorf("lag alert value rendered as a percentage — lag is seconds:\n%s", page)
}
}
+
+// newRedirectFixture builds the minimal Server the CONSOLE_REDIRECT_HOSTS
+// tests need — same trimmed shape as newMetricsFixture (metricsview_test.go),
+// with none of consoleFixture's registry/keymail/HTTP-listener scaffolding,
+// but a caller-chosen Origin and RedirectHosts since that's what's under
+// test here.
+func newRedirectFixture(t *testing.T, origin string, redirectHosts []string) *Server {
+ t.Helper()
+ st, err := Open(filepath.Join(t.TempDir(), "console.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { st.Close() })
+ rel := release.NewDirStore(filepath.Join(t.TempDir(), "bucket"))
+ srv, err := NewServer(Config{Releases: rel, Origin: origin, RedirectHosts: redirectHosts}, st)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return srv
+}
+
+// serveRedirect drives srv.ServeHTTP directly with a chosen Host header — no
+// httptest.Server/listener needed, since the redirect check reads r.Host
+// before the mux (or anything network-shaped) is involved.
+func serveRedirect(t *testing.T, srv *Server, method, host, target string) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(method, target, nil)
+ req.Host = host
+ rec := httptest.NewRecorder()
+ srv.ServeHTTP(rec, req)
+ return rec
+}
+
+func TestRedirectHostsPermanentRedirect(t *testing.T) {
+ srv := newRedirectFixture(t, "https://console.example.test", []string{"dashboard.example.test"})
+
+ rec := serveRedirect(t, srv, http.MethodGet, "dashboard.example.test", "/a/foo?x=1")
+ if rec.Code != http.StatusMovedPermanently {
+ t.Fatalf("GET on a redirect host = %d, want 301", rec.Code)
+ }
+ if loc := rec.Header().Get("Location"); loc != "https://console.example.test/a/foo?x=1" {
+ t.Fatalf("Location = %q, want the canonical origin with the request untouched", loc)
+ }
+}
+
+// TestRedirectHostsPreservesMethodWith308 pins the 308 choice for anything
+// other than GET/HEAD: a 301/302 lets browsers replay a POST as a GET and
+// drop its body, which is exactly the client breakage RedirectHosts must not
+// cause on a host still fielding real requests.
+func TestRedirectHostsPreservesMethodWith308(t *testing.T) {
+ srv := newRedirectFixture(t, "https://console.example.test", []string{"dashboard.example.test"})
+
+ rec := serveRedirect(t, srv, http.MethodPost, "dashboard.example.test", "/a/foo?x=1")
+ if rec.Code != http.StatusPermanentRedirect {
+ t.Fatalf("POST on a redirect host = %d, want 308 (keeps the method and body on replay)", rec.Code)
+ }
+ if loc := rec.Header().Get("Location"); loc != "https://console.example.test/a/foo?x=1" {
+ t.Fatalf("Location = %q, want the canonical origin", loc)
+ }
+}
+
+func TestRedirectHostsMatchesHostWithPort(t *testing.T) {
+ srv := newRedirectFixture(t, "https://console.example.test", []string{"dashboard.example.test"})
+
+ rec := serveRedirect(t, srv, http.MethodGet, "dashboard.example.test:443", "/a/foo?x=1")
+ if rec.Code != http.StatusMovedPermanently {
+ t.Fatalf("Host header carrying a port = %d, want 301 — the port must not defeat the match", rec.Code)
+ }
+ if loc := rec.Header().Get("Location"); loc != "https://console.example.test/a/foo?x=1" {
+ t.Fatalf("Location = %q, want the canonical origin", loc)
+ }
+}
+
+// TestRedirectHostsLeavesOtherHostsAlone pins the opt-in contract: a host
+// that was never listed (here, the console's own origin host) gets the
+// route it would always get, not a redirect — this is what keeps
+// platform.carlosframework.com serving CLI bearer tokens untouched.
+func TestRedirectHostsLeavesOtherHostsAlone(t *testing.T) {
+ srv := newRedirectFixture(t, "https://console.example.test", []string{"dashboard.example.test"})
+
+ rec := serveRedirect(t, srv, http.MethodGet, "console.example.test", "/healthz")
+ if rec.Code == http.StatusMovedPermanently || rec.Code == http.StatusPermanentRedirect {
+ t.Fatalf("an unlisted host was redirected: %d -> %q", rec.Code, rec.Header().Get("Location"))
+ }
+ if rec.Code != 200 || !strings.Contains(rec.Body.String(), "ok") {
+ t.Fatalf("unlisted host = %d %q, want /healthz served normally", rec.Code, rec.Body.String())
+ }
+}
+
+// TestNewServerRefusesRedirectingItsOwnOrigin pins the loop guard: a
+// RedirectHosts entry equal to Origin's own host would 301/308 a request
+// straight back to the host that just answered it.
+func TestNewServerRefusesRedirectingItsOwnOrigin(t *testing.T) {
+ st, err := Open(filepath.Join(t.TempDir(), "console.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { st.Close() })
+ rel := release.NewDirStore(filepath.Join(t.TempDir(), "bucket"))
+ _, err = NewServer(Config{
+ Releases: rel,
+ Origin: "https://console.example.test",
+ RedirectHosts: []string{"console.example.test"},
+ }, st)
+ if err == nil {
+ t.Fatal("NewServer accepted a RedirectHosts entry equal to its own origin host — that config would redirect-loop")
+ }
+}
+
+// TestNewServerRefusesRedirectHostsWithoutAnOriginHost pins the other half of
+// the loop guard: an origin it cannot read a host out of (here the classic
+// operator slip, a CONSOLE_ORIGIN with no scheme) leaves the guard with
+// nothing to compare against AND leaves every Location without a host of its
+// own. Refuse at start rather than serve half a redirect.
+func TestNewServerRefusesRedirectHostsWithoutAnOriginHost(t *testing.T) {
+ for _, origin := range []string{"", "console.example.test"} {
+ st, err := Open(filepath.Join(t.TempDir(), "console.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { st.Close() })
+ rel := release.NewDirStore(filepath.Join(t.TempDir(), "bucket"))
+ if _, err := NewServer(Config{
+ Releases: rel,
+ Origin: origin,
+ RedirectHosts: []string{"dashboard.example.test"},
+ }, st); err == nil {
+ t.Errorf("NewServer accepted RedirectHosts with Origin %q — there is no host to redirect to", origin)
+ }
+ }
+}
+
+// TestRedirectHostsRootlessTargetStaysOnTheOrigin pins the Location against a
+// request target that isn't a path. Go parses `GET mailto:x@evil.com` into an
+// opaque URL, and origin+RequestURI would then read
+// https://console.example.test@evil.com — userinfo, not host — handing an
+// outsider a redirect off the deployment with the console's name on the
+// front of it.
+func TestRedirectHostsRootlessTargetStaysOnTheOrigin(t *testing.T) {
+ srv := newRedirectFixture(t, "https://console.example.test", []string{"dashboard.example.test"})
+
+ for _, target := range []string{"mailto:x@evil.com", "https:evil.com"} {
+ rec := serveRedirect(t, srv, http.MethodGet, "dashboard.example.test", target)
+ loc := rec.Header().Get("Location")
+ if loc != "https://console.example.test/" {
+ t.Errorf("target %q redirected to %q, want the origin's front door", target, loc)
+ }
+ }
+}
+
+func TestNormalizeHost(t *testing.T) {
+ for _, tc := range []struct{ in, want string }{
+ {"dashboard.example.test", "dashboard.example.test"},
+ {"DASHBOARD.Example.Test:443", "dashboard.example.test"},
+ {" dashboard.example.test ", "dashboard.example.test"},
+ // An IPv6 literal keys the same with or without its port, which is
+ // only true if the portless form loses its brackets too.
+ {"[::1]", "::1"},
+ {"[::1]:8080", "::1"},
+ } {
+ if got := normalizeHost(tc.in); got != tc.want {
+ t.Errorf("normalizeHost(%q) = %q, want %q", tc.in, got, tc.want)
+ }
+ }
+}
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go
index 039a92c22..1baf02394 100644
--- a/internal/host/bootstrap.go
+++ b/internal/host/bootstrap.go
@@ -34,12 +34,27 @@ var unitsFS embed.FS
// CAP_NET_BIND_SERVICE to still bind :80/:443.
const serviceUser = "carlos"
+// socketsGroup is the blessed cross-user consumers group for exec-backed
+// instance sockets (issue #103): a host app's own process (titogo's router
+// was the finding) enrolled here may dial every instance socket on the box.
+// A DEDICATED group, never serviceUser's own: group `carlos` is also the
+// group on /data (0750), so enrolling a consumer there would grant
+// group-read of every instance database and env file — every
+// CARLOS_ADMIN_TOKEN on the box — when the grant's whole meaning is "may
+// dial sockets".
+const socketsGroup = "carlos-sockets"
+
// tmpfilesConf keeps /run/carlos across reboots (/run is tmpfs). Owned by the
-// service user so the non-root units can bind their sockets there. The apps
-// subdir holds the carlos-app@ tenant sockets: root systemd binds them
-// (SocketUser=carlos, 0600), tenants never need path access (the fd is
-// passed in), and other=0 on both dirs keeps every non-carlos uid out.
-const tmpfilesConf = "d /run/carlos 0750 " + serviceUser + " " + serviceUser + " -\n" +
+// service user so the non-root units can bind their sockets there; group is
+// the consumers group with SETGID so every socket an exec child binds inside
+// inherits it — that inheritance is what lets the activator's post-health
+// 0660 normalization (backend_exec.go) grant exactly the enrolled consumers
+// and nobody else, with no app aware the group exists. The apps subdir holds
+// the carlos-app@ tenant sockets: root systemd binds them (SocketUser=carlos,
+// 0600), tenants never need path access (the fd is passed in) — it stays
+// carlos:carlos, so consumer enrollment deliberately does not reach unit
+// tenants. other=0 on both dirs keeps every unenrolled uid out.
+const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup + " -\n" +
"d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n"
// enabledUnits are what a live bootstrap turns on: the always-on front door,
@@ -91,23 +106,35 @@ func systemctlPlan(units []string) [][]string {
return plan
}
-// RunBootstrap is the `carlos bootstrap` verb. Idempotent — safe to re-run.
+// RunBootstrap is the `carlos bootstrap` verb. Idempotent — safe to re-run
+// (which is also how a box enrolls a new socket consumer after the fact).
func RunBootstrap(args []string) {
fs := flag.NewFlagSet("bootstrap", flag.ExitOnError)
root := fs.String("root", "/", "install root; a staging dir writes files only (no user creation, no systemctl)")
offcloud := fs.Bool("offcloud", false, "off-cloud box: require a staged 0600 credentials file instead of an EC2 instance profile")
+ var consumers repeatableFlag
+ fs.Var(&consumers, "socket-consumer", "host-app user to enroll in the "+socketsGroup+" group (repeatable); members may dial every instance socket on the box")
_ = fs.Parse(args)
if *offcloud {
- if err := bootstrapOffcloud(*root); err != nil {
+ if err := bootstrapOffcloud(*root, consumers); err != nil {
log.Fatalf("bootstrap: %v", err)
}
return
}
- if err := bootstrap(*root); err != nil {
+ if err := bootstrap(*root, consumers); err != nil {
log.Fatalf("bootstrap: %v", err)
}
}
+// repeatableFlag collects every occurrence of a string flag.
+type repeatableFlag []string
+
+func (f *repeatableFlag) String() string { return strings.Join(*f, ",") }
+func (f *repeatableFlag) Set(v string) error {
+ *f = append(*f, v)
+ return nil
+}
+
// bootstrapOffcloud is bootstrap for a box with no EC2 instance profile
// (issue #53 point 4): identical structural work, plus the credential
// preconditions — a staged 0600 credentials file scoped (at IAM) to the
@@ -117,7 +144,7 @@ func RunBootstrap(args []string) {
//
// The credential check runs BEFORE any system mutation: a failed precondition
// leaves the box untouched, same as a failed bootstrap() would.
-func bootstrapOffcloud(root string) error {
+func bootstrapOffcloud(root string, consumers []string) error {
cred := filepath.Join(root, "etc", "carlos", "credentials")
fi, err := os.Stat(cred)
if err != nil {
@@ -126,7 +153,7 @@ func bootstrapOffcloud(root string) error {
if fi.Mode().Perm() != 0o600 {
return fmt.Errorf("offcloud: %s must be mode 0600, is %o", cred, fi.Mode().Perm())
}
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, consumers); err != nil {
return err
}
// Ownership is a live-box-only step, same as ensureServiceUser/chownToService
@@ -146,8 +173,19 @@ func bootstrapOffcloud(root string) error {
// sequence (daemon-reload, enable, and arming the timers; see systemctlPlan).
// A non-"/" root is the staging/test path: files only, with the privileged
// steps printed instead of run.
-func bootstrap(root string) error {
+func bootstrap(root string, consumers []string) error {
live := filepath.Clean(root) == "/"
+ if live {
+ // Validate consumer users BEFORE any mutation (the offcloud
+ // precondition's discipline): a typo in -socket-consumer must leave
+ // the box untouched, not abort bootstrap halfway with units written
+ // but timers never armed.
+ for _, u := range consumers {
+ if _, err := user.Lookup(u); err != nil {
+ return fmt.Errorf("-socket-consumer %s: no such user (create the host app's user first): %w", u, err)
+ }
+ }
+ }
unitDir := filepath.Join(root, "etc", "systemd", "system")
dataDir := filepath.Join(root, "data")
runDir := filepath.Join(root, "run", "carlos")
@@ -159,7 +197,10 @@ func bootstrap(root string) error {
dirs := []dir{
{dataDir, 0o750},
{filepath.Join(dataDir, "acme"), 0o750},
- {runDir, 0o750},
+ // Setgid: sockets bound inside inherit the dir's group — the
+ // consumers group, once a live run (or the tmpfiles line) sets it.
+ // See socketsGroup/tmpfilesConf.
+ {runDir, 0o750 | os.ModeSetgid},
// The carlos-app@ tenant sockets (see tmpfilesConf for the perms
// rationale).
{filepath.Join(runDir, "apps"), 0o750},
@@ -175,6 +216,7 @@ func bootstrap(root string) error {
{filepath.Join(root, "etc", "carlos", "apps"), 0o755},
{unitDir, 0o755},
{filepath.Join(root, "etc", "tmpfiles.d"), 0o755},
+ {filepath.Join(root, "etc", "sysusers.d"), 0o755},
}
for _, d := range dirs {
if err := os.MkdirAll(d.path, d.mode); err != nil {
@@ -203,6 +245,35 @@ func bootstrap(root string) error {
fmt.Printf(" wrote %s\n", path)
}
+ // On a live root the privileged plan runs BEFORE the tmpfiles conf is
+ // written: the conf's /run/carlos line names socketsGroup, and
+ // systemd-tmpfiles DROPS a line whose group doesn't resolve — so a
+ // bootstrap interrupted between "conf written" and "groupadd ran" would
+ // leave a box that looks healthy until its next reboot, when /run/carlos
+ // simply isn't created and carlos-edge (ReadWritePaths=/run/carlos, no
+ // "-" prefix) fails its namespace setup forever. Group first, then the
+ // conf that names it.
+ if live {
+ for _, args := range privilegedPlan(consumers) {
+ if err := runPrivileged(args); err != nil {
+ return err
+ }
+ }
+ }
+
+ // The sysusers.d snippet closes the same hole for confs that arrive
+ // WITHOUT a live bootstrap run: a staging root is how a live box
+ // refreshes its unit files (pinfra's update-platform-binary.sh), and a
+ // refresh that installs the staged etc/ tree ships the tmpfiles line to
+ // a box that never ran groupadd. systemd-sysusers.service is ordered
+ // Before=systemd-tmpfiles-setup.service, so the group exists by the time
+ // the tmpfiles line is parsed, however the conf got there.
+ sysusersPath := filepath.Join(root, "etc", "sysusers.d", "carlos.conf")
+ if err := os.WriteFile(sysusersPath, []byte("g "+socketsGroup+" -\n"), 0o644); err != nil {
+ return fmt.Errorf("write %s: %v", sysusersPath, err)
+ }
+ fmt.Printf(" wrote %s\n", sysusersPath)
+
tmpfilesPath := filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf")
if err := os.WriteFile(tmpfilesPath, []byte(tmpfilesConf), 0o644); err != nil {
return fmt.Errorf("write %s: %v", tmpfilesPath, err)
@@ -211,29 +282,20 @@ func bootstrap(root string) error {
if !live {
fmt.Printf("bootstrap: staged under %s — on a real box (-root /) it would also:\n", root)
- fmt.Printf(" useradd --system --user-group --no-create-home --shell /usr/sbin/nologin %s\n", serviceUser)
- fmt.Printf(" chown -R %s:%s /data /run/carlos\n", serviceUser, serviceUser)
// Printed, never run: a staging root is how a LIVE box refreshes its
// unit files (pinfra's update-platform-binary.sh stages into
// /tmp/carlos-unitstage), so arming anything from here would reach out
// of the temp root and touch the running system from a command whose
// entire contract is "write files, change nothing".
+ for _, args := range privilegedPlan(consumers) {
+ fmt.Printf(" %s\n", strings.Join(args, " "))
+ }
for _, args := range systemctlPlan(enabledUnits) {
fmt.Printf(" systemctl %s\n", strings.Join(args, " "))
}
return nil
}
- if err := ensureServiceUser(); err != nil {
- return err
- }
- // The units run as serviceUser, so it must own where they read/write
- // (recursive, so it also fixes ownership of any data already present).
- for _, p := range []string{dataDir, runDir} {
- if err := chownToService(p); err != nil {
- return err
- }
- }
for _, args := range systemctlPlan(enabledUnits) {
if err := systemctl(args...); err != nil {
return err
@@ -257,21 +319,70 @@ func unitNames() ([]string, error) {
return names, nil
}
-// ensureServiceUser creates the locked, home-less system user the units run
-// as, idempotently.
-func ensureServiceUser() error {
- if _, err := user.Lookup(serviceUser); err == nil {
- return nil
+// privilegedPlan is the exact non-systemctl privileged sequence a live
+// bootstrap runs — and the same sequence a staging run PRINTS instead of
+// running (systemctlPlan's discipline, extended to the user/group half when
+// issue #103 added the consumers group): the two can never drift into
+// describing different behaviour.
+//
+// The sequence: service user, consumers group, ownership. Every chown
+// converges TOWARD the target state, never through an intermediate that
+// undoes a previous run's work: /run/carlos is recursively
+// serviceUser:socketsGroup — recursive on purpose, so a re-run (the
+// documented way to enroll another consumer) also converges the group on
+// every LIVE instance socket instead of resetting it to serviceUser's group
+// and stranding the already-enrolled consumers until each instance rebinds —
+// and apps/ (unit-tenant sockets, systemd-bound, no path consumers) is then
+// put back to serviceUser:serviceUser. The chmod re-asserts the setgid bit
+// the tmpfiles line promises; consumers enroll last, one usermod each
+// (`-aG` is append — an existing member is a no-op, so re-runs converge).
+// It deliberately does NOT touch group `carlos` memberships: an interim
+// grant made there (the tito workaround this replaces) is the operator's to
+// revoke, per the deployment notes in the #103 spec.
+func privilegedPlan(consumers []string) [][]string {
+ plan := [][]string{
+ {"useradd", "--system", "--user-group", "--no-create-home", "--shell", "/usr/sbin/nologin", serviceUser},
+ {"groupadd", "--system", socketsGroup},
+ {"chown", "-R", serviceUser + ":" + serviceUser, "/data"},
+ {"chown", "-R", serviceUser + ":" + socketsGroup, "/run/carlos"},
+ {"chown", "-R", serviceUser + ":" + serviceUser, "/run/carlos/apps"},
+ {"chmod", "2750", "/run/carlos"},
+ }
+ for _, u := range consumers {
+ plan = append(plan, []string{"usermod", "-aG", socketsGroup, u})
+ }
+ return plan
+}
+
+// runPrivileged executes one privilegedPlan step, adding the existence
+// guards that keep the raw commands idempotent: useradd and groupadd exit
+// non-zero when their subject already exists, and "already provisioned" is
+// bootstrap's normal re-run state, not an error. Everything else in the plan
+// is idempotent as-is.
+func runPrivileged(args []string) error {
+ // The subject is derived from the command line itself (both commands
+ // take it as their final argument), never from a package constant — a
+ // guard keyed to a constant would silently skip any SECOND useradd or
+ // groupadd a later plan revision adds.
+ switch subject := args[len(args)-1]; args[0] {
+ case "useradd":
+ if _, err := user.Lookup(subject); err == nil {
+ return nil
+ }
+ case "groupadd":
+ if _, err := user.LookupGroup(subject); err == nil {
+ return nil
+ }
}
- out, err := exec.Command("useradd", "--system", "--user-group",
- "--no-create-home", "--shell", "/usr/sbin/nologin", serviceUser).CombinedOutput()
+ out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
if err != nil {
- return fmt.Errorf("create user %s: %v: %s", serviceUser, err, strings.TrimSpace(string(out)))
+ return fmt.Errorf("%s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out)))
}
return nil
}
-// chownToService recursively gives a path to the service user.
+// chownToService recursively gives a path to the service user (offcloud's
+// credentials file; the standard dirs go through privilegedPlan).
func chownToService(path string) error {
out, err := exec.Command("chown", "-R", serviceUser+":"+serviceUser, path).CombinedOutput()
if err != nil {
diff --git a/internal/host/host_test.go b/internal/host/host_test.go
index 960f7df3a..e432768fc 100644
--- a/internal/host/host_test.go
+++ b/internal/host/host_test.go
@@ -76,7 +76,7 @@ func TestUnitsEmbedComplete(t *testing.T) {
func TestBootstrapWritesUnitsToStagingDir(t *testing.T) {
root := t.TempDir()
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, nil); err != nil {
t.Fatalf("bootstrap: %v", err)
}
@@ -577,10 +577,10 @@ func TestProvisionTickConfig(t *testing.T) {
func TestBootstrapIdempotent(t *testing.T) {
root := t.TempDir()
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, nil); err != nil {
t.Fatalf("first bootstrap: %v", err)
}
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, nil); err != nil {
t.Fatalf("second bootstrap: %v", err)
}
// Still all present after the re-run.
@@ -675,7 +675,7 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
- if err := bootstrap(t.TempDir()); err != nil {
+ if err := bootstrap(t.TempDir(), nil); err != nil {
t.Fatalf("bootstrap: %v", err)
}
if b, err := os.ReadFile(log); err == nil {
@@ -683,6 +683,98 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
}
}
+// TestBootstrapTmpfilesBlessesConsumersGroup pins the instance-socket
+// consumers contract (issue #103): /run/carlos carries the dedicated
+// carlos-sockets group with the setgid bit — so every socket an exec child
+// binds inside inherits the consumers group — while /run/carlos/apps stays
+// carlos:carlos (unit tenants get their fd passed; enrollment deliberately
+// does not reach them). The group is dedicated, never `carlos`: membership
+// in `carlos` also grants group-read of /data, i.e. every instance's DB and
+// CARLOS_ADMIN_TOKEN.
+func TestBootstrapTmpfilesBlessesConsumersGroup(t *testing.T) {
+ root := t.TempDir()
+ if err := bootstrap(root, nil); err != nil {
+ t.Fatalf("bootstrap: %v", err)
+ }
+ tf, err := os.ReadFile(filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf"))
+ if err != nil {
+ t.Fatalf("tmpfiles conf: %v", err)
+ }
+ if want := "d /run/carlos 2750 carlos carlos-sockets -"; !strings.Contains(string(tf), want) {
+ t.Errorf("tmpfiles conf %q missing %q", tf, want)
+ }
+ if want := "d /run/carlos/apps 0750 carlos carlos -"; !strings.Contains(string(tf), want) {
+ t.Errorf("tmpfiles conf %q missing %q — the tenant socket dir must stay out of the consumers group", tf, want)
+ }
+
+ // Bootstrap's own dir handling must agree with the tmpfiles line: setgid
+ // on /run/carlos (mode is what makes group inheritance real between
+ // boots), and NOT on apps/.
+ fi, err := os.Stat(filepath.Join(root, "run", "carlos"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode()&os.ModeSetgid == 0 {
+ t.Errorf("/run/carlos mode %v: missing setgid — bound sockets would not inherit the consumers group", fi.Mode())
+ }
+ if fi.Mode().Perm() != 0o750 {
+ t.Errorf("/run/carlos perm %o, want 750", fi.Mode().Perm())
+ }
+ afi, err := os.Stat(filepath.Join(root, "run", "carlos", "apps"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if afi.Mode()&os.ModeSetgid != 0 {
+ t.Errorf("/run/carlos/apps mode %v: setgid must not leak onto the tenant socket dir", afi.Mode())
+ }
+
+ // The sysusers.d snippet is what makes the tmpfiles line safe at boot on
+ // a box whose conf arrived via a staged-tree refresh (never a live
+ // groupadd): systemd-sysusers runs Before=systemd-tmpfiles-setup, and a
+ // tmpfiles line whose group doesn't resolve is DROPPED — /run/carlos then
+ // isn't created and carlos-edge (ReadWritePaths, no "-") can't start.
+ su, err := os.ReadFile(filepath.Join(root, "etc", "sysusers.d", "carlos.conf"))
+ if err != nil {
+ t.Fatalf("sysusers conf: %v", err)
+ }
+ if want := "g carlos-sockets -"; !strings.Contains(string(su), want) {
+ t.Errorf("sysusers conf %q missing %q", su, want)
+ }
+}
+
+// TestBootstrapPrivilegedPlanEnrollsConsumers pins the printed-equals-run
+// contract for the group half, the same discipline systemctlPlan established:
+// a staging run prints exactly the commands a live run executes, so the two
+// can never drift. -socket-consumer users each get a usermod enrollment into
+// the dedicated group; no consumers means no usermod lines but the group and
+// dir convergence always run.
+func TestBootstrapPrivilegedPlanEnrollsConsumers(t *testing.T) {
+ got := privilegedPlan([]string{"titogo", "titogo-relay"})
+ // The /run/carlos chown is recursive AND targets the consumers group: a
+ // re-run is the documented enrollment path, and a recursive chown back
+ // to carlos:carlos would reset every live instance socket's group,
+ // stranding already-enrolled consumers until each instance rebinds.
+ // apps/ is put back to carlos:carlos right after (unit tenants stay out
+ // of consumer reach).
+ want := [][]string{
+ {"useradd", "--system", "--user-group", "--no-create-home", "--shell", "/usr/sbin/nologin", "carlos"},
+ {"groupadd", "--system", "carlos-sockets"},
+ {"chown", "-R", "carlos:carlos", "/data"},
+ {"chown", "-R", "carlos:carlos-sockets", "/run/carlos"},
+ {"chown", "-R", "carlos:carlos", "/run/carlos/apps"},
+ {"chmod", "2750", "/run/carlos"},
+ {"usermod", "-aG", "carlos-sockets", "titogo"},
+ {"usermod", "-aG", "carlos-sockets", "titogo-relay"},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("privilegedPlan = %v, want %v", got, want)
+ }
+ // No consumers: same plan minus the usermod tail.
+ if got := privilegedPlan(nil); !reflect.DeepEqual(got, want[:6]) {
+ t.Errorf("privilegedPlan(nil) = %v, want %v", got, want[:6])
+ }
+}
+
// TestBootstrapOffcloudRequiresCredentialsFile pins the off-cloud precondition
// (issue #53 point 4): a box with no EC2 instance profile must have its scoped
// credentials staged — and staged at 0600 — before bootstrap proceeds. This is
@@ -690,18 +782,18 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
// service-user chown below never runs; that only happens on a live root.
func TestBootstrapOffcloudRequiresCredentialsFile(t *testing.T) {
root := t.TempDir()
- if err := bootstrapOffcloud(root); err == nil {
+ if err := bootstrapOffcloud(root, nil); err == nil {
t.Fatal("want error when <root>/etc/carlos/credentials is absent")
}
credDir := filepath.Join(root, "etc", "carlos")
os.MkdirAll(credDir, 0o755)
credFile := filepath.Join(credDir, "credentials")
os.WriteFile(credFile, []byte("[default]\naws_access_key_id=X\naws_secret_access_key=Y\n"), 0o644)
- if err := bootstrapOffcloud(root); err == nil {
+ if err := bootstrapOffcloud(root, nil); err == nil {
t.Fatal("want error on 0644 credentials (must be 0600)")
}
os.Chmod(credFile, 0o600)
- if err := bootstrapOffcloud(root); err != nil {
+ if err := bootstrapOffcloud(root, nil); err != nil {
t.Fatalf("offcloud bootstrap with staged 0600 credentials: %v", err)
}
// The normal bootstrap artifacts must exist too — offcloud is a superset.