amadan

carlos / platform Public

bootstrap: bless the carlos-sockets consumers group (issue #103)

/run/carlos becomes 2750 carlos:carlos-sockets (setgid, so bound sockets
inherit the consumers group); apps/ stays carlos:carlos — enrollment never
reaches unit tenants. New repeatable -socket-consumer flag enrolls a
host-app user via usermod -aG. privilegedPlan extends systemctlPlan's
printed-equals-run discipline to the user/group half.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QytpMG8wVhBmWwz3uohtc7
Paul Campbell 8e608408a097968cb4f4d0a2a9138cb5612e0716 parent d8b5e0d
2 files changed, +176 −37
  • internal/host/bootstrap.go +96 −30
  • internal/host/host_test.go +80 −7
diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go
index 283fb3d10..a4dfebe9d 100644
--- a/internal/host/bootstrap.go
+++ b/internal/host/bootstrap.go
@@ -34,12 +34,27 @@ var unitsFS embed.FS
// CAP_NET_BIND_SERVICE to still bind :80/:443.
const serviceUser = "carlos"
+// socketsGroup is the blessed cross-user consumers group for exec-backed
+// instance sockets (issue #103): a host app's own process (titogo's router
+// was the finding) enrolled here may dial every instance socket on the box.
+// A DEDICATED group, never serviceUser's own: group `carlos` is also the
+// group on /data (0750), so enrolling a consumer there would grant
+// group-read of every instance database and env file — every
+// CARLOS_ADMIN_TOKEN on the box — when the grant's whole meaning is "may
+// dial sockets".
+const socketsGroup = "carlos-sockets"
+
// tmpfilesConf keeps /run/carlos across reboots (/run is tmpfs). Owned by the
-// service user so the non-root units can bind their sockets there. The apps
-// subdir holds the carlos-app@ tenant sockets: root systemd binds them
-// (SocketUser=carlos, 0600), tenants never need path access (the fd is
-// passed in), and other=0 on both dirs keeps every non-carlos uid out.
-const tmpfilesConf = "d /run/carlos 0750 " + serviceUser + " " + serviceUser + " -\n" +
+// service user so the non-root units can bind their sockets there; group is
+// the consumers group with SETGID so every socket an exec child binds inside
+// inherits it — that inheritance is what lets the activator's post-health
+// 0660 normalization (backend_exec.go) grant exactly the enrolled consumers
+// and nobody else, with no app aware the group exists. The apps subdir holds
+// the carlos-app@ tenant sockets: root systemd binds them (SocketUser=carlos,
+// 0600), tenants never need path access (the fd is passed in) — it stays
+// carlos:carlos, so consumer enrollment deliberately does not reach unit
+// tenants. other=0 on both dirs keeps every unenrolled uid out.
+const tmpfilesConf = "d /run/carlos 2750 " + serviceUser + " " + socketsGroup + " -\n" +
"d /run/carlos/apps 0750 " + serviceUser + " " + serviceUser + " -\n"
// enabledUnits are what a live bootstrap turns on: the always-on front door,
@@ -90,23 +105,35 @@ func systemctlPlan(units []string) [][]string {
return plan
}
-// RunBootstrap is the `carlos bootstrap` verb. Idempotent — safe to re-run.
+// RunBootstrap is the `carlos bootstrap` verb. Idempotent — safe to re-run
+// (which is also how a box enrolls a new socket consumer after the fact).
func RunBootstrap(args []string) {
fs := flag.NewFlagSet("bootstrap", flag.ExitOnError)
root := fs.String("root", "/", "install root; a staging dir writes files only (no user creation, no systemctl)")
offcloud := fs.Bool("offcloud", false, "off-cloud box: require a staged 0600 credentials file instead of an EC2 instance profile")
+ var consumers repeatableFlag
+ fs.Var(&consumers, "socket-consumer", "host-app user to enroll in the "+socketsGroup+" group (repeatable); members may dial every instance socket on the box")
_ = fs.Parse(args)
if *offcloud {
- if err := bootstrapOffcloud(*root); err != nil {
+ if err := bootstrapOffcloud(*root, consumers); err != nil {
log.Fatalf("bootstrap: %v", err)
}
return
}
- if err := bootstrap(*root); err != nil {
+ if err := bootstrap(*root, consumers); err != nil {
log.Fatalf("bootstrap: %v", err)
}
}
+// repeatableFlag collects every occurrence of a string flag.
+type repeatableFlag []string
+
+func (f *repeatableFlag) String() string { return strings.Join(*f, ",") }
+func (f *repeatableFlag) Set(v string) error {
+ *f = append(*f, v)
+ return nil
+}
+
// bootstrapOffcloud is bootstrap for a box with no EC2 instance profile
// (issue #53 point 4): identical structural work, plus the credential
// preconditions — a staged 0600 credentials file scoped (at IAM) to the
@@ -116,7 +143,7 @@ func RunBootstrap(args []string) {
//
// The credential check runs BEFORE any system mutation: a failed precondition
// leaves the box untouched, same as a failed bootstrap() would.
-func bootstrapOffcloud(root string) error {
+func bootstrapOffcloud(root string, consumers []string) error {
cred := filepath.Join(root, "etc", "carlos", "credentials")
fi, err := os.Stat(cred)
if err != nil {
@@ -125,7 +152,7 @@ func bootstrapOffcloud(root string) error {
if fi.Mode().Perm() != 0o600 {
return fmt.Errorf("offcloud: %s must be mode 0600, is %o", cred, fi.Mode().Perm())
}
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, consumers); err != nil {
return err
}
// Ownership is a live-box-only step, same as ensureServiceUser/chownToService
@@ -145,7 +172,7 @@ func bootstrapOffcloud(root string) error {
// sequence (daemon-reload, enable, and arming the timers; see systemctlPlan).
// A non-"/" root is the staging/test path: files only, with the privileged
// steps printed instead of run.
-func bootstrap(root string) error {
+func bootstrap(root string, consumers []string) error {
live := filepath.Clean(root) == "/"
unitDir := filepath.Join(root, "etc", "systemd", "system")
dataDir := filepath.Join(root, "data")
@@ -158,7 +185,10 @@ func bootstrap(root string) error {
dirs := []dir{
{dataDir, 0o750},
{filepath.Join(dataDir, "acme"), 0o750},
- {runDir, 0o750},
+ // Setgid: sockets bound inside inherit the dir's group — the
+ // consumers group, once a live run (or the tmpfiles line) sets it.
+ // See socketsGroup/tmpfilesConf.
+ {runDir, 0o750 | os.ModeSetgid},
// The carlos-app@ tenant sockets (see tmpfilesConf for the perms
// rationale).
{filepath.Join(runDir, "apps"), 0o750},
@@ -210,26 +240,22 @@ func bootstrap(root string) error {
if !live {
fmt.Printf("bootstrap: staged under %s — on a real box (-root /) it would also:\n", root)
- fmt.Printf(" useradd --system --user-group --no-create-home --shell /usr/sbin/nologin %s\n", serviceUser)
- fmt.Printf(" chown -R %s:%s /data /run/carlos\n", serviceUser, serviceUser)
// Printed, never run: a staging root is how a LIVE box refreshes its
// unit files (pinfra's update-platform-binary.sh stages into
// /tmp/carlos-unitstage), so arming anything from here would reach out
// of the temp root and touch the running system from a command whose
// entire contract is "write files, change nothing".
+ for _, args := range privilegedPlan(consumers) {
+ fmt.Printf(" %s\n", strings.Join(args, " "))
+ }
for _, args := range systemctlPlan(enabledUnits) {
fmt.Printf(" systemctl %s\n", strings.Join(args, " "))
}
return nil
}
- if err := ensureServiceUser(); err != nil {
- return err
- }
- // The units run as serviceUser, so it must own where they read/write
- // (recursive, so it also fixes ownership of any data already present).
- for _, p := range []string{dataDir, runDir} {
- if err := chownToService(p); err != nil {
+ for _, args := range privilegedPlan(consumers) {
+ if err := runPrivileged(args); err != nil {
return err
}
}
@@ -256,21 +282,61 @@ func unitNames() ([]string, error) {
return names, nil
}
-// ensureServiceUser creates the locked, home-less system user the units run
-// as, idempotently.
-func ensureServiceUser() error {
- if _, err := user.Lookup(serviceUser); err == nil {
- return nil
+// privilegedPlan is the exact non-systemctl privileged sequence a live
+// bootstrap runs — and the same sequence a staging run PRINTS instead of
+// running (systemctlPlan's discipline, extended to the user/group half when
+// issue #103 added the consumers group): the two can never drift into
+// describing different behaviour.
+//
+// The sequence: service user, consumers group, ownership (the recursive
+// chown gives serviceUser everything it reads/writes — and resets
+// /run/carlos's group in passing, which is why the dir-only chown to the
+// consumers group comes AFTER it, never before). The chmod re-asserts the
+// setgid bit the tmpfiles line promises; consumers enroll last, one usermod
+// each (`-aG` is append — an existing member is a no-op, so re-runs
+// converge). It deliberately does NOT touch group `carlos` memberships: an
+// interim grant made there (the tito workaround this replaces) is the
+// operator's to revoke, per the deployment notes in the #103 spec.
+func privilegedPlan(consumers []string) [][]string {
+ plan := [][]string{
+ {"useradd", "--system", "--user-group", "--no-create-home", "--shell", "/usr/sbin/nologin", serviceUser},
+ {"groupadd", "--system", socketsGroup},
+ {"chown", "-R", serviceUser + ":" + serviceUser, "/data"},
+ {"chown", "-R", serviceUser + ":" + serviceUser, "/run/carlos"},
+ {"chown", serviceUser + ":" + socketsGroup, "/run/carlos"},
+ {"chmod", "2750", "/run/carlos"},
+ }
+ for _, u := range consumers {
+ plan = append(plan, []string{"usermod", "-aG", socketsGroup, u})
+ }
+ return plan
+}
+
+// runPrivileged executes one privilegedPlan step, adding the existence
+// guards that keep the raw commands idempotent: useradd and groupadd exit
+// non-zero when their subject already exists, and "already provisioned" is
+// bootstrap's normal re-run state, not an error. Everything else in the plan
+// is idempotent as-is.
+func runPrivileged(args []string) error {
+ switch args[0] {
+ case "useradd":
+ if _, err := user.Lookup(serviceUser); err == nil {
+ return nil
+ }
+ case "groupadd":
+ if _, err := user.LookupGroup(socketsGroup); err == nil {
+ return nil
+ }
}
- out, err := exec.Command("useradd", "--system", "--user-group",
- "--no-create-home", "--shell", "/usr/sbin/nologin", serviceUser).CombinedOutput()
+ out, err := exec.Command(args[0], args[1:]...).CombinedOutput()
if err != nil {
- return fmt.Errorf("create user %s: %v: %s", serviceUser, err, strings.TrimSpace(string(out)))
+ return fmt.Errorf("%s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out)))
}
return nil
}
-// chownToService recursively gives a path to the service user.
+// chownToService recursively gives a path to the service user (offcloud's
+// credentials file; the standard dirs go through privilegedPlan).
func chownToService(path string) error {
out, err := exec.Command("chown", "-R", serviceUser+":"+serviceUser, path).CombinedOutput()
if err != nil {
diff --git a/internal/host/host_test.go b/internal/host/host_test.go
index fcca235b5..e365770cd 100644
--- a/internal/host/host_test.go
+++ b/internal/host/host_test.go
@@ -74,7 +74,7 @@ func TestUnitsEmbedComplete(t *testing.T) {
func TestBootstrapWritesUnitsToStagingDir(t *testing.T) {
root := t.TempDir()
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, nil); err != nil {
t.Fatalf("bootstrap: %v", err)
}
@@ -534,10 +534,10 @@ func TestProvisionTickConfig(t *testing.T) {
func TestBootstrapIdempotent(t *testing.T) {
root := t.TempDir()
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, nil); err != nil {
t.Fatalf("first bootstrap: %v", err)
}
- if err := bootstrap(root); err != nil {
+ if err := bootstrap(root, nil); err != nil {
t.Fatalf("second bootstrap: %v", err)
}
// Still all present after the re-run.
@@ -632,7 +632,7 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
}
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
- if err := bootstrap(t.TempDir()); err != nil {
+ if err := bootstrap(t.TempDir(), nil); err != nil {
t.Fatalf("bootstrap: %v", err)
}
if b, err := os.ReadFile(log); err == nil {
@@ -640,6 +640,79 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
}
}
+// TestBootstrapTmpfilesBlessesConsumersGroup pins the instance-socket
+// consumers contract (issue #103): /run/carlos carries the dedicated
+// carlos-sockets group with the setgid bit — so every socket an exec child
+// binds inside inherits the consumers group — while /run/carlos/apps stays
+// carlos:carlos (unit tenants get their fd passed; enrollment deliberately
+// does not reach them). The group is dedicated, never `carlos`: membership
+// in `carlos` also grants group-read of /data, i.e. every instance's DB and
+// CARLOS_ADMIN_TOKEN.
+func TestBootstrapTmpfilesBlessesConsumersGroup(t *testing.T) {
+ root := t.TempDir()
+ if err := bootstrap(root, nil); err != nil {
+ t.Fatalf("bootstrap: %v", err)
+ }
+ tf, err := os.ReadFile(filepath.Join(root, "etc", "tmpfiles.d", "carlos.conf"))
+ if err != nil {
+ t.Fatalf("tmpfiles conf: %v", err)
+ }
+ if want := "d /run/carlos 2750 carlos carlos-sockets -"; !strings.Contains(string(tf), want) {
+ t.Errorf("tmpfiles conf %q missing %q", tf, want)
+ }
+ if want := "d /run/carlos/apps 0750 carlos carlos -"; !strings.Contains(string(tf), want) {
+ t.Errorf("tmpfiles conf %q missing %q — the tenant socket dir must stay out of the consumers group", tf, want)
+ }
+
+ // Bootstrap's own dir handling must agree with the tmpfiles line: setgid
+ // on /run/carlos (mode is what makes group inheritance real between
+ // boots), and NOT on apps/.
+ fi, err := os.Stat(filepath.Join(root, "run", "carlos"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode()&os.ModeSetgid == 0 {
+ t.Errorf("/run/carlos mode %v: missing setgid — bound sockets would not inherit the consumers group", fi.Mode())
+ }
+ if fi.Mode().Perm() != 0o750 {
+ t.Errorf("/run/carlos perm %o, want 750", fi.Mode().Perm())
+ }
+ afi, err := os.Stat(filepath.Join(root, "run", "carlos", "apps"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if afi.Mode()&os.ModeSetgid != 0 {
+ t.Errorf("/run/carlos/apps mode %v: setgid must not leak onto the tenant socket dir", afi.Mode())
+ }
+}
+
+// TestBootstrapPrivilegedPlanEnrollsConsumers pins the printed-equals-run
+// contract for the group half, the same discipline systemctlPlan established:
+// a staging run prints exactly the commands a live run executes, so the two
+// can never drift. -socket-consumer users each get a usermod enrollment into
+// the dedicated group; no consumers means no usermod lines but the group and
+// dir convergence always run.
+func TestBootstrapPrivilegedPlanEnrollsConsumers(t *testing.T) {
+ got := privilegedPlan([]string{"titogo", "titogo-relay"})
+ want := [][]string{
+ {"useradd", "--system", "--user-group", "--no-create-home", "--shell", "/usr/sbin/nologin", "carlos"},
+ {"groupadd", "--system", "carlos-sockets"},
+ {"chown", "-R", "carlos:carlos", "/data"},
+ {"chown", "-R", "carlos:carlos", "/run/carlos"},
+ {"chown", "carlos:carlos-sockets", "/run/carlos"},
+ {"chmod", "2750", "/run/carlos"},
+ {"usermod", "-aG", "carlos-sockets", "titogo"},
+ {"usermod", "-aG", "carlos-sockets", "titogo-relay"},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("privilegedPlan = %v, want %v", got, want)
+ }
+ // No consumers: same plan minus the usermod tail.
+ if got := privilegedPlan(nil); !reflect.DeepEqual(got, want[:6]) {
+ t.Errorf("privilegedPlan(nil) = %v, want %v", got, want[:6])
+ }
+}
+
// TestBootstrapOffcloudRequiresCredentialsFile pins the off-cloud precondition
// (issue #53 point 4): a box with no EC2 instance profile must have its scoped
// credentials staged — and staged at 0600 — before bootstrap proceeds. This is
@@ -647,18 +720,18 @@ func TestBootstrapStagingRunsNoSystemctl(t *testing.T) {
// service-user chown below never runs; that only happens on a live root.
func TestBootstrapOffcloudRequiresCredentialsFile(t *testing.T) {
root := t.TempDir()
- if err := bootstrapOffcloud(root); err == nil {
+ if err := bootstrapOffcloud(root, nil); err == nil {
t.Fatal("want error when <root>/etc/carlos/credentials is absent")
}
credDir := filepath.Join(root, "etc", "carlos")
os.MkdirAll(credDir, 0o755)
credFile := filepath.Join(credDir, "credentials")
os.WriteFile(credFile, []byte("[default]\naws_access_key_id=X\naws_secret_access_key=Y\n"), 0o644)
- if err := bootstrapOffcloud(root); err == nil {
+ if err := bootstrapOffcloud(root, nil); err == nil {
t.Fatal("want error on 0644 credentials (must be 0600)")
}
os.Chmod(credFile, 0o600)
- if err := bootstrapOffcloud(root); err != nil {
+ if err := bootstrapOffcloud(root, nil); err != nil {
t.Fatalf("offcloud bootstrap with staged 0600 credentials: %v", err)
}
// The normal bootstrap artifacts must exist too — offcloud is a superset.