amadan

carlos / platform Public

Merge branch 'fix-instance-init' into main

Nested instances: a minimal init as PID 1 forwards stop signals and reaps, so apps stop cleanly (#126)
claude 40b6f50c87d7d37ed87dd5c68380b0f0f293d914 parent 9e62802parent 1aa0dbc
11 files changed, +646 −24
  • docs/site/cli.md +6 −3
  • internal/activator/backend_exec.go +2 −2
  • internal/host/units/carlos-instance-seed@.service +3 −0
  • internal/host/units/carlos-instance-sidecar@.service +6 −0
  • internal/host/units/carlos-instance@.service +6 −0
  • internal/host/units_isolation_test.go +9 −2
  • internal/instanceexec/exec.go +5 −4
  • internal/instanceexec/init_linux.go +131 −0
  • internal/instanceexec/init_test.go +475 −0
  • internal/instanceexec/scope_linux.go +2 −12
  • internal/instanceexec/scope_other.go +1 −1

CI passed — run details

diff --git a/docs/site/cli.md b/docs/site/cli.md
index 23fccb7e5..44ec13482 100644
--- a/docs/site/cli.md
+++ b/docs/site/cli.md
@@ -1920,10 +1920,13 @@ process. Not for operators: there is nothing to type.
### carlos instance-exec
The first thing an isolated instance unit runs. It reads the launch file
-the agent wrote into the instance's directory and execs the host's live
+the agent wrote into the instance's directory and starts the host's live
link with exactly the arguments and environment in it, refusing anything
-that is not the platform's own live link. `ExecStart=` of the three
-instance templates.
+that is not the platform's own live link. It then stays on as the
+instance's first process: it passes stop signals on to the app, cleans up
+exited processes, and exits with the app's status, so an app with no
+signal handling still stops cleanly. `ExecStart=` of the three instance
+templates.
### carlos instance-exit
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 60fbaa181..7c2caa3ef 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -1186,8 +1186,8 @@ func (b *ExecBackend) instanceEnv(rt registry.Route) (map[string]string, error)
// childEnv is the child's complete environment: the agent's own, minus
// every CARLOS_* variable, with the instance's file values layered on top.
// One entry per key, last value wins: os/exec deduplicates cmd.Env that
-// way, but the unit path hands this list to syscall.Exec, which passes
-// duplicates through, and a Go child then keeps the FIRST occurrence
+// way, but on the unit path the trampoline hands this list to execve as
+// is, duplicates and all, and a Go child then keeps the FIRST occurrence
// (design §6.2, codex R1-16). Composing it deduplicated here keeps the
// two spawn paths identical.
func childEnv(fileEnv map[string]string) []string {
diff --git a/internal/host/units/carlos-instance-seed@.service b/internal/host/units/carlos-instance-seed@.service
index 706bf5723..42facdaae 100644
--- a/internal/host/units/carlos-instance-seed@.service
+++ b/internal/host/units/carlos-instance-seed@.service
@@ -42,6 +42,9 @@ Slice=carlos-instances-%i.slice
# TimeoutStopSec; the activator's stop budget is 20s.
KillMode=mixed
TimeoutStopSec=20
+# No SuccessExitStatus= here, unlike the run and sidecar templates (#126):
+# the trampoline's init reports a seed a signal ended as 128+n, and a seed
+# a signal ended has not seeded.
# The log stream: stdout and stderr connect to the agent's per-role
# listener (§6.4). PID 1 opens this as root, outside the namespace, on the
# nosymfollow logs mount, at a path no tenant mount contains.
diff --git a/internal/host/units/carlos-instance-sidecar@.service b/internal/host/units/carlos-instance-sidecar@.service
index 7d34f2f89..18c672e28 100644
--- a/internal/host/units/carlos-instance-sidecar@.service
+++ b/internal/host/units/carlos-instance-sidecar@.service
@@ -38,6 +38,12 @@ Restart=no
# TimeoutStopSec; the activator's stop budget is 20s.
KillMode=mixed
TimeoutStopSec=20
+# The main process is the trampoline, which stays as the namespace's init
+# and runs the app as its child (#126): it forwards the SIGTERM and exits
+# 128+n when the app dies by signal n. These are SIGHUP, SIGINT, SIGPIPE
+# and SIGTERM, the deaths systemd counts as clean for a main process, so a
+# stop still leaves the unit inactive rather than failed.
+SuccessExitStatus=129 130 141 143
# The log stream: stdout and stderr connect to the agent's per-role
# listener (§6.4). PID 1 opens this as root, outside the namespace, on the
# nosymfollow logs mount, at a path no tenant mount contains.
diff --git a/internal/host/units/carlos-instance@.service b/internal/host/units/carlos-instance@.service
index 0f6ca71b8..f979ff7d6 100644
--- a/internal/host/units/carlos-instance@.service
+++ b/internal/host/units/carlos-instance@.service
@@ -38,6 +38,12 @@ Restart=no
# TimeoutStopSec; the activator's stop budget is 20s.
KillMode=mixed
TimeoutStopSec=20
+# The main process is the trampoline, which stays as the namespace's init
+# and runs the app as its child (#126): it forwards the SIGTERM and exits
+# 128+n when the app dies by signal n. These are SIGHUP, SIGINT, SIGPIPE
+# and SIGTERM, the deaths systemd counts as clean for a main process, so a
+# stop still leaves the unit inactive rather than failed.
+SuccessExitStatus=129 130 141 143
# The log stream: stdout and stderr connect to the agent's per-role
# listener (§6.4). PID 1 opens this as root, outside the namespace, on the
# nosymfollow logs mount, at a path no tenant mount contains.
diff --git a/internal/host/units_isolation_test.go b/internal/host/units_isolation_test.go
index 527c48f05..9bc3ba317 100644
--- a/internal/host/units_isolation_test.go
+++ b/internal/host/units_isolation_test.go
@@ -267,12 +267,15 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) {
"carlos-instance@.service": {"ExecStart=/opt/carlos/carlos instance-exec run %I", "ExecStopPost=/opt/carlos/carlos instance-exit run %I",
"BindPaths=/data/instances/%I /run/carlos/instances/%I", "StandardOutput=file:/run/carlos/logs/%I.run.sock",
"Slice=carlos-instances-%i.slice", "Restart=no", "KillMode=mixed", "TimeoutStopSec=20", "BindsTo=carlos-edge.service",
- "Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "OOMScoreAdjust=500", "OOMPolicy=kill", "TasksMax=256", "Type=simple"},
+ "Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "OOMScoreAdjust=500", "OOMPolicy=kill", "TasksMax=256", "Type=simple",
+ // The trampoline's init exits 128+n for a death by signal n (#126).
+ "SuccessExitStatus=129 130 141 143"},
"carlos-instance-seed@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=120", "ExecStart=/opt/carlos/carlos instance-exec seed %I",
"ExecStopPost=/opt/carlos/carlos instance-exit seed %I", "StandardOutput=file:/run/carlos/logs/%I.seed.sock",
"KillMode=mixed", "TimeoutStopSec=20"},
"carlos-instance-sidecar@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec sidecar %I",
- "ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock"},
+ "ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock",
+ "SuccessExitStatus=129 130 141 143"},
"carlos-instance-probe.service": {"Type=oneshot", "ExecStart=/opt/carlos/carlos instance-probe",
"BindPaths=/data/instances/_probe /run/carlos/instances/_probe", "StandardOutput=truncate:/run/carlos-control/probe.out"},
} {
@@ -285,6 +288,10 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) {
if strings.Contains(u, "ExecStopPost=+") {
t.Errorf("%s uses ExecStopPost=+: the plus prefix keeps the unit's tmpfs (codex R3-2)", name)
}
+ // A seed a signal ended has not seeded (#126).
+ if name == "carlos-instance-seed@.service" && strings.Contains(u, "\nSuccessExitStatus=") {
+ t.Errorf("%s counts a signal's exit status as success", name)
+ }
if name != "carlos-instance-probe.service" && !strings.Contains(u, "User=carlos") {
t.Errorf("%s does not run as carlos", name)
}
diff --git a/internal/instanceexec/exec.go b/internal/instanceexec/exec.go
index ad72521d7..9b2adfb93 100644
--- a/internal/instanceexec/exec.go
+++ b/internal/instanceexec/exec.go
@@ -51,15 +51,16 @@ func RunExec(args []string) {
if !ok {
log.Fatalf("instance-exec: role %q is not run, seed or sidecar", fs.Arg(0))
}
- // The exec is scoped (#178): see ScopeAbstractUnix.
- if err := execLaunch(dataDir(), role, fs.Arg(1), scopedExec); err != nil {
+ // The tenant is scoped (#178, see ScopeAbstractUnix) and runs under
+ // the trampoline as its init (#126, see init_linux.go).
+ if err := execLaunch(dataDir(), role, fs.Arg(1), scopedInit); err != nil {
log.Fatalf("instance-exec: %v", err)
}
}
// execLaunch reads launch.<role>.json from the instance directory through
-// instancefs (never through a symlink), validates it, and execs. It returns
-// only on refusal; a successful exec never returns.
+// instancefs (never through a symlink), validates it, and launches. It
+// returns only on refusal; a successful launch exits with the tenant.
//
// The validation is defence in depth: the agent wrote the file, but the
// directory is the instance's own, so the instance can rewrite it. What
diff --git a/internal/instanceexec/init_linux.go b/internal/instanceexec/init_linux.go
new file mode 100644
index 000000000..1d90a8ca5
--- /dev/null
+++ b/internal/instanceexec/init_linux.go
@@ -0,0 +1,131 @@
+//go:build linux
+
+package instanceexec
+
+import (
+ "fmt"
+ "os"
+ "os/signal"
+ "runtime"
+ "syscall"
+
+ "golang.org/x/sys/unix"
+)
+
+// The trampoline stays behind as the namespace's init (#126, found on the
+// dev fleet). The templates run with PrivatePIDs=yes, so the unit's main
+// process is PID 1 of its own PID namespace, and the kernel gives a
+// namespace init no default signal actions: a signal it has no handler
+// for is dropped, whoever sends it (only SIGKILL from outside gets
+// through). A tenant exec'd into that slot that never asked for SIGTERM,
+// which is most Go servers, cannot be stopped by it: Go's runtime tries
+// to die by the signal, the kernel drops it, and the runtime exits 2
+// instead; a tenant with no handler at all waits out TimeoutStopSec for
+// the SIGKILL. Every hibernate, restart, move and deploy then left the
+// unit failed. A flat tenant was never PID 1.
+//
+// So the trampoline forks the tenant as its child, which is not PID 1 and
+// keeps its default actions exactly as on the flat layout, and does the
+// two things PID 1 owes the namespace:
+//
+// - Forward. KillMode=mixed sends the stop's SIGTERM to the main
+// process, which is this one; it and the other signals in forwarded
+// go on to the tenant, which acts on them as it would have as the
+// main process. SIGKILL needs no forwarding: after TimeoutStopSec
+// systemd sends it to the whole cgroup, and SIGKILL to a namespace
+// init from outside always kills it.
+// - Reap. A process whose parent dies is reparented to the namespace's
+// init, and stays a zombie until init waits for it. The tenant's own
+// children that outlive their parent land here.
+//
+// When the tenant exits, the init exits with its status: the tenant's
+// exit code, or 128+n for a death by signal n (the shell's convention; a
+// namespace init cannot die by a signal it sends itself). The run and
+// sidecar templates count 129, 130, 141 and 143 as success, the signals
+// systemd itself counts as a clean death of a main process. The kernel
+// then kills whatever is left in the namespace, as it did when the tenant
+// was PID 1.
+//
+// The init is small on purpose and allocates nothing per signal: one
+// buffered channel, one WaitStatus, a kill or a wait4 per signal.
+
+// forwarded are the signals the init passes on to the tenant.
+var forwarded = [...]os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGUSR2}
+
+// scopedInit is the trampoline's launch: scope, start the tenant under
+// the init, and exit with its status. It returns only if the tenant could
+// not be started; a scope that cannot be applied refuses the start (the
+// probe has already failed readiness on such a box, so no nested wake
+// gets here).
+func scopedInit(argv0 string, argv, env []string) error {
+ return runInit(argv0, argv, env, ScopeAbstractUnix)
+}
+
+func runInit(argv0 string, argv, env []string, scope func() error) error {
+ // Before anything can start the tenant: a stop that arrives early is
+ // queued and forwarded, not dropped by PID 1's missing defaults.
+ sigs := make(chan os.Signal, 32)
+ signal.Notify(sigs, append(forwarded[:], syscall.SIGCHLD)...)
+
+ // Landlock restricts a thread, and a fork copies the forking thread's
+ // domain, which execve keeps: scoping this locked thread and forking
+ // from it scopes the tenant. The thread stays locked until exit.
+ runtime.LockOSThread()
+ if err := scope(); err != nil {
+ return fmt.Errorf("not starting without the abstract socket scope: %w", err)
+ }
+ // PID 1 in production already reaps orphans; this makes the init
+ // their reaper wherever it runs.
+ if err := unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); err != nil {
+ return fmt.Errorf("subreaper: %w", err)
+ }
+ // Same argv, env, cwd and stdio as the exec this replaces. Fds beyond
+ // stdio pass as they did there: anything not close-on-exec stays
+ // open across fork and exec. Setsid gives the tenant the session of
+ // its own that systemd gave the main process.
+ tenant, err := syscall.ForkExec(argv0, argv, &syscall.ProcAttr{
+ Env: env,
+ Files: []uintptr{0, 1, 2},
+ Sys: &syscall.SysProcAttr{Setsid: true},
+ })
+ if err != nil {
+ return err
+ }
+ os.Exit(superviseTenant(tenant, sigs))
+ return nil
+}
+
+// superviseTenant forwards signals to the tenant and reaps every child
+// until the tenant itself is reaped, and returns the init's exit code.
+func superviseTenant(tenant int, sigs <-chan os.Signal) int {
+ var ws syscall.WaitStatus
+ for s := range sigs {
+ sig := s.(syscall.Signal)
+ if sig != syscall.SIGCHLD {
+ _ = syscall.Kill(tenant, sig)
+ continue
+ }
+ // One SIGCHLD can stand for several exits: drain them all.
+ for {
+ pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil)
+ if err == syscall.EINTR {
+ continue
+ }
+ if err != nil || pid <= 0 {
+ break
+ }
+ if pid == tenant {
+ return initExitCode(ws)
+ }
+ }
+ }
+ return 1 // unreachable: signal.Notify never closes the channel
+}
+
+// initExitCode maps the tenant's wait status to the init's exit code.
+func initExitCode(ws syscall.WaitStatus) int {
+ if ws.Signaled() {
+ return 128 + int(ws.Signal())
+ }
+ return ws.ExitStatus()
+}
diff --git a/internal/instanceexec/init_test.go b/internal/instanceexec/init_test.go
new file mode 100644
index 000000000..ff2f75d38
--- /dev/null
+++ b/internal/instanceexec/init_test.go
@@ -0,0 +1,475 @@
+//go:build linux
+
+package instanceexec
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "os"
+ "os/exec"
+ "os/signal"
+ "strconv"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ "golang.org/x/sys/unix"
+)
+
+// The init tests run the real runInit in a child process: this test
+// binary, re-executed with ieHelperEnv naming a role. The "init" role
+// runs runInit with the binary itself as the tenant, in the tenant role
+// named by ieTenantEnv; every other role is a tenant.
+const (
+ ieHelperEnv = "CARLOS_INSTANCEEXEC_HELPER"
+ ieTenantEnv = "CARLOS_INSTANCEEXEC_TENANT"
+ ieScopeEnv = "CARLOS_INSTANCEEXEC_SCOPE"
+ ieArgEnv = "CARLOS_INSTANCEEXEC_ARG"
+)
+
+func TestMain(m *testing.M) {
+ if role := os.Getenv(ieHelperEnv); role != "" {
+ os.Exit(runHelper(role))
+ }
+ os.Exit(m.Run())
+}
+
+// withRole is the environment with the helper role replaced, not
+// appended: a Go child keeps the FIRST of duplicate keys (design §6.2).
+func withRole(role string) []string {
+ env := []string{ieHelperEnv + "=" + role}
+ for _, kv := range os.Environ() {
+ if !strings.HasPrefix(kv, ieHelperEnv+"=") {
+ env = append(env, kv)
+ }
+ }
+ return env
+}
+
+func runHelper(role string) int {
+ self, err := os.Executable()
+ if err != nil {
+ fmt.Println("executable:", err)
+ return 70
+ }
+ switch role {
+ case "init":
+ env := withRole(os.Getenv(ieTenantEnv))
+ scope := func() error { return nil }
+ if os.Getenv(ieScopeEnv) == "1" {
+ scope = ScopeAbstractUnix
+ }
+ err := runInit(self, []string{self}, env, scope)
+ fmt.Println("init:", err)
+ return 71
+ case "no-handler":
+ // A tenant that never asks for SIGTERM, like most Go servers.
+ fmt.Printf("ready pid=%d ppid=%d\n", os.Getpid(), os.Getppid())
+ time.Sleep(30 * time.Second)
+ return 0
+ case "echo-signals":
+ // Reports every forwarded signal; leaves on SIGTERM.
+ got := make(chan os.Signal, 8)
+ signal.Notify(got, forwarded[:]...)
+ fmt.Println("ready")
+ for s := range got {
+ fmt.Println("got", s)
+ if s == syscall.SIGTERM {
+ return 0
+ }
+ }
+ return 0
+ case "exit":
+ n, _ := strconv.Atoi(os.Getenv(ieArgEnv))
+ return n
+ case "kill-self":
+ _ = syscall.Kill(os.Getpid(), syscall.SIGKILL)
+ time.Sleep(30 * time.Second)
+ return 0
+ case "orphan":
+ // Leave a grandchild behind: the intermediate starts it and exits
+ // at once, so it is reparented to the nearest reaper.
+ cmd := exec.Command(self)
+ cmd.Env = withRole("intermediate")
+ cmd.Stdout = os.Stdout
+ if err := cmd.Run(); err != nil {
+ fmt.Println("intermediate:", err)
+ return 72
+ }
+ time.Sleep(30 * time.Second)
+ return 0
+ case "intermediate":
+ cmd := exec.Command(self)
+ cmd.Env = withRole("short-lived")
+ if err := cmd.Start(); err != nil {
+ fmt.Println("grandchild:", err)
+ return 73
+ }
+ fmt.Printf("grandchild %d\n", cmd.Process.Pid)
+ return 0
+ case "short-lived":
+ time.Sleep(1500 * time.Millisecond)
+ return 0
+ case "inherit":
+ cwd, _ := os.Getwd()
+ sid, _ := unix.Getsid(0)
+ fmt.Printf("cwd=%s arg=%s sid-is-pid=%t\n", cwd, os.Getenv(ieArgEnv), sid == os.Getpid())
+ f := os.NewFile(3, "fd3")
+ if _, err := f.Write([]byte("fd3 reached\n")); err != nil {
+ fmt.Println("fd3:", err)
+ }
+ return 0
+ case "dial":
+ c, err := net.Dial("unix", os.Getenv(ieArgEnv))
+ switch {
+ case err == nil:
+ c.Close()
+ fmt.Println("dial: connected")
+ case errors.Is(unwrapErrno(err), syscall.EPERM):
+ fmt.Println("dial: EPERM")
+ default:
+ fmt.Println("dial:", err)
+ }
+ return 0
+ }
+ fmt.Println("unknown helper role", role)
+ return 74
+}
+
+// startInit starts the init helper with the given tenant role and extra
+// environment, and returns it with a line reader on its stdout (which is
+// the tenant's stdout too: the init passes fd 1 through).
+func startInit(t *testing.T, tenant string, attr *syscall.SysProcAttr, extra ...string) (*exec.Cmd, *bufio.Reader) {
+ t.Helper()
+ self, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.Command(self)
+ cmd.Env = append(os.Environ(), append([]string{ieHelperEnv + "=init", ieTenantEnv + "=" + tenant}, extra...)...)
+ cmd.Stderr = os.Stderr
+ cmd.SysProcAttr = attr
+ out, err := cmd.StdoutPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := cmd.Start(); err != nil {
+ if attr != nil {
+ t.Skipf("no unprivileged user and PID namespace here: %v", err)
+ }
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ // Through the init first, so the tenant stops with it outside a
+ // PID namespace too; the SIGKILL is for a test that failed early.
+ _ = cmd.Process.Signal(syscall.SIGTERM)
+ time.Sleep(100 * time.Millisecond)
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ })
+ return cmd, bufio.NewReader(out)
+}
+
+// readLine reads one line within d, failing the test on a timeout.
+func readLine(t *testing.T, r *bufio.Reader, d time.Duration) string {
+ t.Helper()
+ got := make(chan string, 1)
+ go func() {
+ line, err := r.ReadString('\n')
+ if err != nil && line == "" {
+ line = "read: " + err.Error()
+ }
+ got <- strings.TrimSpace(line)
+ }()
+ select {
+ case line := <-got:
+ return line
+ case <-time.After(d):
+ t.Fatalf("no line from the helper within %v", d)
+ return ""
+ }
+}
+
+// waitExit waits up to d for cmd to exit and returns its exit code, or -1
+// with the signal if it died by one.
+func waitExit(t *testing.T, cmd *exec.Cmd, d time.Duration) (int, syscall.Signal) {
+ t.Helper()
+ done := make(chan error, 1)
+ go func() { done <- cmd.Wait() }()
+ select {
+ case <-done:
+ case <-time.After(d):
+ t.Fatalf("helper still running %v after the signal", d)
+ }
+ ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
+ if ws.Signaled() {
+ return -1, ws.Signal()
+ }
+ return ws.ExitStatus(), 0
+}
+
+func TestInitExitStatusMapping(t *testing.T) {
+ for _, c := range []struct {
+ ws syscall.WaitStatus
+ want int
+ }{
+ {0, 0},
+ {1 << 8, 1},
+ {2 << 8, 2},
+ {255 << 8, 255},
+ {syscall.WaitStatus(syscall.SIGTERM), 143},
+ {syscall.WaitStatus(syscall.SIGKILL), 137},
+ {syscall.WaitStatus(syscall.SIGINT), 130},
+ {syscall.WaitStatus(syscall.SIGSEGV) | 0x80, 139}, // core dumped
+ } {
+ if got := initExitCode(c.ws); got != c.want {
+ t.Errorf("initExitCode(%#x) = %d, want %d", uint32(c.ws), got, c.want)
+ }
+ }
+}
+
+// End to end through the init, outside any namespace: the tenant's own
+// exit code passes through, and a death by signal is 128+n.
+func TestInitExitsWithTheTenantsStatus(t *testing.T) {
+ cmd, _ := startInit(t, "exit", nil, ieArgEnv+"=7")
+ if code, sig := waitExit(t, cmd, 10*time.Second); code != 7 {
+ t.Errorf("init exit = %d (signal %v), want 7", code, sig)
+ }
+ cmd, _ = startInit(t, "kill-self", nil)
+ if code, sig := waitExit(t, cmd, 10*time.Second); code != 128+int(syscall.SIGKILL) {
+ t.Errorf("init exit = %d (signal %v), want 137", code, sig)
+ }
+}
+
+// Every forwarded signal reaches the tenant.
+func TestInitForwardsEachSignal(t *testing.T) {
+ cmd, out := startInit(t, "echo-signals", nil)
+ if line := readLine(t, out, 10*time.Second); line != "ready" {
+ t.Fatalf("tenant said %q", line)
+ }
+ // SIGTERM last: the tenant leaves on it.
+ for _, sig := range []syscall.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGTERM} {
+ if err := cmd.Process.Signal(sig); err != nil {
+ t.Fatal(err)
+ }
+ if line := readLine(t, out, 5*time.Second); line != "got "+sig.String() {
+ t.Errorf("sent %v, tenant said %q", sig, line)
+ }
+ }
+ if code, sig := waitExit(t, cmd, 5*time.Second); code != 0 {
+ t.Errorf("init exit = %d (signal %v), want the tenant's 0", code, sig)
+ }
+}
+
+// A tenant with no handlers acts on a forwarded signal by its default
+// disposition, as the main process did before PrivatePIDs: it dies by
+// the signal, and the init reports 128+n.
+func TestInitStopsATenantWithNoHandler(t *testing.T) {
+ for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP} {
+ t.Run(sig.String(), func(t *testing.T) {
+ cmd, out := startInit(t, "no-handler", nil)
+ if line := readLine(t, out, 10*time.Second); !strings.HasPrefix(line, "ready ") {
+ t.Fatalf("tenant said %q", line)
+ }
+ if err := cmd.Process.Signal(sig); err != nil {
+ t.Fatal(err)
+ }
+ if code, got := waitExit(t, cmd, 5*time.Second); code != 128+int(sig) {
+ t.Errorf("init exit = %d (signal %v), want %d", code, got, 128+int(sig))
+ }
+ })
+ }
+}
+
+// The tenant gets the init's cwd, the launch env, a session of its own
+// (as the unit's main process had) and any inherited fd beyond stdio.
+func TestInitPassesTheTenantItsEnvironment(t *testing.T) {
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer r.Close()
+ self, _ := os.Executable()
+ cmd := exec.Command(self)
+ cmd.Dir = t.TempDir()
+ cmd.Env = append(os.Environ(), ieHelperEnv+"=init", ieTenantEnv+"=inherit", ieArgEnv+"=launch-value")
+ cmd.ExtraFiles = []*os.File{w}
+ var stdout strings.Builder
+ cmd.Stdout = &stdout
+ cmd.Stderr = os.Stderr
+ if err := cmd.Run(); err != nil {
+ t.Fatalf("init: %v; output %q", err, stdout.String())
+ }
+ w.Close()
+ want := fmt.Sprintf("cwd=%s arg=launch-value sid-is-pid=true", cmd.Dir)
+ if got := strings.TrimSpace(stdout.String()); got != want {
+ t.Errorf("tenant said %q, want %q", got, want)
+ }
+ b, _ := io.ReadAll(r)
+ if string(b) != "fd3 reached\n" {
+ t.Errorf("fd 3 carried %q", b)
+ }
+}
+
+// A process orphaned inside the namespace is the init's to reap. The
+// grandchild must be reparented to the init (it is a subreaper, and PID 1
+// in production) and must not stay a zombie after it exits.
+func TestInitReapsOrphans(t *testing.T) {
+ cmd, out := startInit(t, "orphan", nil)
+ line := readLine(t, out, 10*time.Second)
+ var gpid int
+ if _, err := fmt.Sscanf(line, "grandchild %d", &gpid); err != nil {
+ t.Fatalf("tenant said %q", line)
+ }
+ initPid := cmd.Process.Pid
+ adopted := false
+ deadline := time.Now().Add(10 * time.Second)
+ for time.Now().Before(deadline) {
+ state, ppid, ok := procStat(gpid)
+ if !ok {
+ break // reaped
+ }
+ if ppid == initPid {
+ adopted = true
+ }
+ if state == "Z" && time.Now().After(deadline.Add(-7*time.Second)) {
+ t.Fatalf("grandchild %d is a zombie of %d and nobody reaps it", gpid, ppid)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ if _, _, ok := procStat(gpid); ok {
+ t.Fatalf("grandchild %d still present", gpid)
+ }
+ if !adopted {
+ t.Fatalf("grandchild %d was never the init's (%d) child", gpid, initPid)
+ }
+ // The tenant itself is still running: reaping an orphan is not the
+ // tenant's exit.
+ if err := cmd.Process.Signal(syscall.Signal(0)); err != nil {
+ t.Fatalf("init gone after reaping an orphan: %v", err)
+ }
+}
+
+// procStat returns the state and parent of pid from /proc/<pid>/stat.
+func procStat(pid int) (state string, ppid int, ok bool) {
+ b, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
+ if err != nil {
+ return "", 0, false
+ }
+ s := string(b)
+ i := strings.LastIndexByte(s, ')')
+ if i < 0 {
+ return "", 0, false
+ }
+ f := strings.Fields(s[i+1:])
+ if len(f) < 2 {
+ return "", 0, false
+ }
+ ppid, _ = strconv.Atoi(f[1])
+ return f[0], ppid, true
+}
+
+// The Landlock scope applied by the init holds in the tenant: the scope is
+// on the thread that forks, and a domain is inherited across fork and
+// exec. The unscoped run is the control, proving the dial can succeed.
+func TestInitTenantKeepsTheAbstractSocketScope(t *testing.T) {
+ abi, _, e := unix.Syscall(unix.SYS_LANDLOCK_CREATE_RULESET, 0, 0, unix.LANDLOCK_CREATE_RULESET_VERSION)
+ if e != 0 || abi < landlockScopeABI {
+ t.Skipf("no Landlock scope on this kernel (abi %d, %v)", abi, e)
+ }
+ name := "@carlos-init-test-" + randomName()
+ ln, err := net.Listen("unix", name)
+ if err != nil {
+ t.Skipf("abstract listen: %v", err)
+ }
+ defer ln.Close()
+ go func() {
+ for {
+ c, err := ln.Accept()
+ if err != nil {
+ return
+ }
+ c.Close()
+ }
+ }()
+ for scope, want := range map[string]string{"0": "dial: connected", "1": "dial: EPERM"} {
+ cmd, out := startInit(t, "dial", nil, ieArgEnv+"="+name, ieScopeEnv+"="+scope)
+ if got := readLine(t, out, 10*time.Second); got != want {
+ t.Errorf("scope=%s: tenant said %q, want %q", scope, got, want)
+ }
+ if code, sig := waitExit(t, cmd, 10*time.Second); code != 0 {
+ t.Errorf("scope=%s: init exit = %d (signal %v)", scope, code, sig)
+ }
+ }
+}
+
+// The finding itself (#126, dev fleet): in a PID namespace, as under
+// PrivatePIDs=yes, the first process is init and has no default signal
+// actions. A Go tenant with no SIGTERM handler in that slot does not die
+// by SIGTERM. Through the init it is the init's child and does, promptly. Needs an
+// unprivileged user namespace; skipped where there is none.
+func TestInitStopsATenantWithNoHandlerInAPIDNamespace(t *testing.T) {
+ userPIDNS := func() *syscall.SysProcAttr {
+ return &syscall.SysProcAttr{
+ Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWPID,
+ UidMappings: []syscall.SysProcIDMap{{ContainerID: os.Getuid(), HostID: os.Getuid(), Size: 1}},
+ GidMappings: []syscall.SysProcIDMap{{ContainerID: os.Getgid(), HostID: os.Getgid(), Size: 1}},
+ }
+ }
+
+ t.Run("tenant as PID 1", func(t *testing.T) {
+ self, _ := os.Executable()
+ cmd := exec.Command(self)
+ cmd.Env = append(os.Environ(), ieHelperEnv+"=no-handler")
+ cmd.SysProcAttr = userPIDNS()
+ cmd.Stderr = io.Discard // the runtime's own complaint, if any
+ out, err := cmd.StdoutPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := cmd.Start(); err != nil {
+ t.Skipf("no unprivileged user and PID namespace here: %v", err)
+ }
+ defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }()
+ if line := readLine(t, bufio.NewReader(out), 10*time.Second); !strings.HasPrefix(line, "ready pid=1 ") {
+ t.Fatalf("tenant said %q, want it at PID 1", line)
+ }
+ _ = cmd.Process.Signal(syscall.SIGTERM)
+ done := make(chan struct{})
+ go func() { _ = cmd.Wait(); close(done) }()
+ select {
+ case <-done:
+ ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
+ if ws.Signaled() && ws.Signal() == syscall.SIGTERM {
+ t.Skip("this kernel delivered SIGTERM's default action to a namespace init")
+ }
+ t.Logf("as PID 1 the tenant did not die by SIGTERM: %v (the bug the init fixes)", cmd.ProcessState)
+ case <-time.After(2 * time.Second):
+ t.Logf("as PID 1 the tenant ignored SIGTERM (the bug the init fixes)")
+ }
+ })
+
+ t.Run("through the init", func(t *testing.T) {
+ cmd, out := startInit(t, "no-handler", userPIDNS())
+ // Not PID 2 as such: the init's own threads take PIDs in the
+ // namespace too. What matters is that the tenant is not PID 1.
+ line := readLine(t, out, 10*time.Second)
+ var pid, ppid int
+ if _, err := fmt.Sscanf(line, "ready pid=%d ppid=%d", &pid, &ppid); err != nil || pid == 1 || ppid != 1 {
+ t.Fatalf("tenant said %q, want a child of the init at PID 1", line)
+ }
+ start := time.Now()
+ if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
+ t.Fatal(err)
+ }
+ code, sig := waitExit(t, cmd, 5*time.Second)
+ if code != 128+int(syscall.SIGTERM) {
+ t.Errorf("init exit = %d (signal %v), want 143", code, sig)
+ }
+ t.Logf("stopped in %v", time.Since(start))
+ })
+}
diff --git a/internal/instanceexec/scope_linux.go b/internal/instanceexec/scope_linux.go
index 557a8f303..6ed398668 100644
--- a/internal/instanceexec/scope_linux.go
+++ b/internal/instanceexec/scope_linux.go
@@ -28,7 +28,8 @@ const landlockScopeABI = 6
// other's.
//
// Landlock restricts a thread, not a process, so the caller must hold
-// runtime.LockOSThread and either exec or let the thread die locked.
+// runtime.LockOSThread and then exec, fork (the child inherits the
+// domain), or let the thread die locked.
// Nothing here needs privilege: no_new_privs (which the unit already sets
// with NoNewPrivileges=) is the only precondition.
func ScopeAbstractUnix() error {
@@ -54,17 +55,6 @@ func ScopeAbstractUnix() error {
return nil
}
-// scopedExec is the trampoline's exec: scope, then exec on the same
-// thread. A scope that cannot be applied refuses the exec; the probe has
-// already failed readiness on such a box, so no nested wake gets here.
-func scopedExec(argv0 string, argv, env []string) error {
- runtime.LockOSThread() // never unlocked: the thread execs or the process exits
- if err := ScopeAbstractUnix(); err != nil {
- return fmt.Errorf("not starting without the abstract socket scope: %w", err)
- }
- return syscall.Exec(argv0, argv, env)
-}
-
var errAbstractReached = errors.New("connected to an abstract socket bound outside the scope")
// realAbstractCheck proves the scope on this kernel, end to end: it binds
diff --git a/internal/instanceexec/scope_other.go b/internal/instanceexec/scope_other.go
index 9176634f2..6d68fca9b 100644
--- a/internal/instanceexec/scope_other.go
+++ b/internal/instanceexec/scope_other.go
@@ -5,7 +5,7 @@ package instanceexec
import "errors"
func ScopeAbstractUnix() error { return errors.ErrUnsupported }
-func scopedExec(string, []string, []string) error { return errors.ErrUnsupported }
+func scopedInit(string, []string, []string) error { return errors.ErrUnsupported }
func realAbstractCheck() error { return errors.ErrUnsupported }
func ipcNamespaceInode() (uint64, error) { return 0, errors.ErrUnsupported }
func ioWeightEnforced(string) bool { return false }