amadan
cgroup: kernel test, shim placement under carlos-edge's sandbox lines (#272)
TestEdgeHardeningPlacement copies carlos-edge.service's [Service] sandbox lines (20 applied, 13 excluded with reasons; a new key fails the test until it is sorted) onto a transient delegated user unit and runs TestEdgeHardeningInner there, after an unrestricted control in which clone3 and Go's UseCgroupFD start work. Under the hardening the inner test observes each line it can (16 enforced, 4 untested), shows clone3 and the UseCgroupFD start answer ENOSYS, places a generation through the shim into carlos-exec@hardening.example/g1, OOM-kills a 64M hog in g2 while g1 lives, and passes VerifyPlacement.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 file changed,
+811
−0
internal/cgroup/edge_hardening_kernel_linux_test.go+811 −0
diff --git a/internal/cgroup/edge_hardening_kernel_linux_test.go b/internal/cgroup/edge_hardening_kernel_linux_test.go| new file mode 100644 |
| index 000000000..74fe41255 |
| --- /dev/null |
| +++ b/internal/cgroup/edge_hardening_kernel_linux_test.go |
| @@ -0,0 +1,811 @@ |
| +//go:build linux |
| + |
| +package cgroup |
| + |
| +import ( |
| + "bufio" |
| + "bytes" |
| + "context" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "runtime" |
| + "strconv" |
| + "strings" |
| + "sync" |
| + "syscall" |
| + "testing" |
| + "time" |
| + |
| + "golang.org/x/sys/unix" |
| +) |
| + |
| +// The shim placement under carlos-edge.service's own sandbox lines (zdd |
| +// readiness 3, Task 4; amadan #272). The shipped unit's RestrictNamespaces= |
| +// yes has systemd's seccomp filter answer clone3 with ENOSYS, which broke |
| +// clone3 placement on the dev box. TestEdgeHardeningPlacement copies the |
| +// unit's [Service] sandbox lines onto a transient user unit and runs |
| +// TestEdgeHardeningInner there, after a control run in a unit without them. |
| +// Opt-in: CARLOS_EDGE_HARDENING_KERNEL=1, run outside any unit, unsandboxed, |
| +// with TMPDIR under /tmp (the brief's /tmp/k/zddr3). |
| + |
| +const ( |
| + edgeUnitPath = "../host/units/carlos-edge.service" |
| + edgeInnerEnv = "CARLOS_EDGE_HARDENING_INNER" |
| + edgeHostUTSEnv = "CARLOS_EDGE_HARDENING_HOST_UTS" |
| + edgeHomeEnv = "CARLOS_EDGE_HARDENING_HOME" |
| + edgeMarkerEnv = "CARLOS_EDGE_HARDENING_MARKER" |
| + edgeTasksMaxEnv = "CARLOS_EDGE_HARDENING_TASKS_MAX" |
| + edgeUnitLifetime = "RuntimeMaxSec=300" |
| +) |
| + |
| +// edgeApplied are the [Service] keys passed to the hardened unit as |
| +// -p Key=value, exactly as the unit file has them. |
| +var edgeApplied = []string{ |
| + "NoNewPrivileges", |
| + "ProtectSystem", |
| + "ProtectHome", |
| + "PrivateTmp", |
| + "PrivateDevices", |
| + "ProtectClock", |
| + "ProtectHostname", |
| + "ProtectKernelTunables", |
| + "ProtectKernelModules", |
| + "ProtectKernelLogs", |
| + "ProtectProc", |
| + "RestrictAddressFamilies", |
| + "RestrictNamespaces", |
| + "RestrictRealtime", |
| + "LockPersonality", |
| + "MemoryDenyWriteExecute", |
| + "SystemCallFilter", |
| + "SystemCallErrorNumber", |
| + "SystemCallArchitectures", |
| + "TasksMax", |
| +} |
| + |
| +// edgeExcluded are the [Service] keys the hardened unit does not get, each |
| +// with its reason. A [Service] key in neither table fails the driver, so a |
| +// new line in the unit forces a decision here. |
| +var edgeExcluded = map[string]string{ |
| + "EnvironmentFile": "process control, not sandbox: names the box's /etc/carlos/host.env", |
| + "ExecStart": "process control, not sandbox: the unit runs the test binary instead", |
| + "ExecReload": "process control, not sandbox", |
| + "Restart": "process control, not sandbox", |
| + "RestartSec": "process control, not sandbox", |
| + "User": "a user manager cannot change identity", |
| + "Group": "a user manager cannot change identity", |
| + "AmbientCapabilities": "a user manager cannot grant capabilities", |
| + "CapabilityBoundingSet": "a user manager cannot change capabilities", |
| + "ReadWritePaths": "names box paths (/data, /run/carlos) absent here", |
| + "Slice": "carlos.slice is the box's system slice; a user unit stays in the user manager's tree", |
| + "OOMPolicy": "unit lifecycle on an OOM kill, not sandbox; the test reads memory.events itself", |
| + "Delegate": "the test passes its own Delegate=yes (a user manager delegates cpu memory pids, no io)", |
| +} |
| + |
| +// edgeMustApply are the lines the brief requires applied: each must be in |
| +// the unit file and in edgeApplied. |
| +var edgeMustApply = []string{ |
| + "RestrictNamespaces", "SystemCallFilter", "SystemCallErrorNumber", "SystemCallArchitectures", |
| + "NoNewPrivileges", "MemoryDenyWriteExecute", "LockPersonality", "RestrictRealtime", |
| + "RestrictAddressFamilies", "ProtectProc", "ProtectClock", "ProtectHostname", |
| + "ProtectKernelTunables", "ProtectKernelModules", "ProtectKernelLogs", "PrivateDevices", |
| + "ProtectSystem", "TasksMax", |
| +} |
| + |
| +type edgeLine struct{ key, value string } |
| + |
| +// readEdgeService returns every Key=value line of path's [Service] section, |
| +// in file order. |
| +func readEdgeService(t *testing.T, path string) []edgeLine { |
| + t.Helper() |
| + raw, err := os.ReadFile(path) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var lines []edgeLine |
| + section := "" |
| + for n, line := range strings.Split(string(raw), "\n") { |
| + line = strings.TrimSpace(line) |
| + switch { |
| + case line == "", strings.HasPrefix(line, "#"), strings.HasPrefix(line, ";"): |
| + continue |
| + case strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]"): |
| + section = line |
| + continue |
| + case section != "[Service]": |
| + continue |
| + case strings.HasSuffix(line, `\`): |
| + t.Fatalf("%s:%d: a continued line; the driver reads one line per key", path, n+1) |
| + } |
| + key, value, ok := strings.Cut(line, "=") |
| + if !ok { |
| + t.Fatalf("%s:%d: %q is no Key=value line", path, n+1, line) |
| + } |
| + lines = append(lines, edgeLine{strings.TrimSpace(key), strings.TrimSpace(value)}) |
| + } |
| + if len(lines) == 0 { |
| + t.Fatalf("%s has no [Service] lines", path) |
| + } |
| + return lines |
| +} |
| + |
| +// edgeLog is the driver's view of a unit's output: each line goes to the |
| +// test log as it arrives, and the whole is kept for the verdict. |
| +type edgeLog struct { |
| + t *testing.T |
| + label string |
| + mu sync.Mutex |
| + part []byte |
| + all bytes.Buffer |
| +} |
| + |
| +func (w *edgeLog) Write(p []byte) (int, error) { |
| + w.mu.Lock() |
| + defer w.mu.Unlock() |
| + w.all.Write(p) |
| + w.part = append(w.part, p...) |
| + for { |
| + i := bytes.IndexByte(w.part, '\n') |
| + if i < 0 { |
| + break |
| + } |
| + w.t.Logf("%s| %s", w.label, w.part[:i]) |
| + w.part = w.part[i+1:] |
| + } |
| + return len(p), nil |
| +} |
| + |
| +func (w *edgeLog) flush() { |
| + w.mu.Lock() |
| + defer w.mu.Unlock() |
| + if len(w.part) > 0 { |
| + w.t.Logf("%s| %s", w.label, w.part) |
| + w.part = nil |
| + } |
| +} |
| + |
| +// runEdgeUnit runs the inner test in a transient user unit with props and |
| +// fails unless it passed, without a skip. |
| +func runEdgeUnit(t *testing.T, label, bin string, props, env []string) { |
| + t.Helper() |
| + args := []string{"--user", "--wait", "--pipe", "--collect"} |
| + for _, p := range props { |
| + args = append(args, "-p", p) |
| + } |
| + for _, e := range env { |
| + args = append(args, "-E", e) |
| + } |
| + args = append(args, bin, "-test.run", "^TestEdgeHardeningInner$", "-test.v", "-test.count=1") |
| + t.Logf("%s: systemd-run %s", label, strings.Join(quoteArgs(args), " ")) |
| + out := &edgeLog{t: t, label: label} |
| + cmd := exec.Command("systemd-run", args...) |
| + cmd.Stdout = out |
| + cmd.Stderr = out |
| + err := cmd.Run() |
| + out.flush() |
| + all := out.all.String() |
| + if err != nil { |
| + t.Fatalf("%s: systemd-run: %v", label, err) |
| + } |
| + if !strings.Contains(all, "--- PASS: TestEdgeHardeningInner ") { |
| + t.Fatalf("%s: no --- PASS line for TestEdgeHardeningInner", label) |
| + } |
| + if strings.Contains(all, "--- SKIP") || strings.Contains(all, "--- FAIL") { |
| + t.Fatalf("%s: the inner run skipped or failed", label) |
| + } |
| +} |
| + |
| +func quoteArgs(args []string) []string { |
| + q := make([]string, len(args)) |
| + for i, a := range args { |
| + if strings.ContainsAny(a, " $'\"") { |
| + a = strconv.Quote(a) |
| + } |
| + q[i] = a |
| + } |
| + return q |
| +} |
| + |
| +// TestEdgeHardeningPlacement is the driver: an unrestricted control, where |
| +// clone3 placement works, then the inner test under the edge unit's |
| +// sandbox lines, where clone3 answers ENOSYS and the shim must place. |
| +func TestEdgeHardeningPlacement(t *testing.T) { |
| + if os.Getenv("CARLOS_EDGE_HARDENING_KERNEL") != "1" { |
| + t.Skip("set CARLOS_EDGE_HARDENING_KERNEL=1, outside any unit, unsandboxed (zdd readiness 3, Task 4)") |
| + } |
| + if os.Getenv(edgeInnerEnv) != "" { |
| + t.Fatalf("the driver runs outside the unit, but %s is set", edgeInnerEnv) |
| + } |
| + if _, err := exec.LookPath("systemd-run"); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // 1. The unit's [Service] lines, each in exactly one table. |
| + lines := readEdgeService(t, edgeUnitPath) |
| + applied := map[string]bool{} |
| + for _, k := range edgeApplied { |
| + if _, both := edgeExcluded[k]; both { |
| + t.Fatalf("%s is in both the applied and the excluded table", k) |
| + } |
| + applied[k] = true |
| + } |
| + inUnit := map[string]bool{} |
| + var props []string |
| + tasksMax := "" |
| + for _, l := range lines { |
| + inUnit[l.key] = true |
| + _, excluded := edgeExcluded[l.key] |
| + switch { |
| + case applied[l.key]: |
| + props = append(props, l.key+"="+l.value) |
| + if l.key == "TasksMax" { |
| + tasksMax = l.value |
| + } |
| + case excluded: |
| + default: |
| + t.Fatalf("%s: [Service] %s=%s is in neither the applied nor the excluded table: decide which, and why", edgeUnitPath, l.key, l.value) |
| + } |
| + } |
| + for _, k := range edgeApplied { |
| + if !inUnit[k] { |
| + t.Fatalf("applied table names %s, which %s no longer has", k, edgeUnitPath) |
| + } |
| + } |
| + for k := range edgeExcluded { |
| + if !inUnit[k] { |
| + t.Fatalf("excluded table names %s, which %s no longer has", k, edgeUnitPath) |
| + } |
| + } |
| + for _, k := range edgeMustApply { |
| + if !inUnit[k] || !applied[k] { |
| + t.Fatalf("%s must be in %s and applied (in unit %v, applied %v)", k, edgeUnitPath, inUnit[k], applied[k]) |
| + } |
| + } |
| + t.Logf("applied (%d), passed as -p Key=value:", len(props)) |
| + for _, p := range props { |
| + t.Logf(" %s", p) |
| + } |
| + t.Logf("excluded (%d):", len(edgeExcluded)) |
| + for _, l := range lines { |
| + if r, ok := edgeExcluded[l.key]; ok { |
| + t.Logf(" %-22s %s", l.key, r) |
| + } |
| + } |
| + |
| + // The probes' inputs: what the inner run compares against this side. |
| + bin, err := os.Executable() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + uts, err := os.Readlink("/proc/self/ns/uts") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + home, err := os.UserHomeDir() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := os.ReadDir(home); err != nil { |
| + t.Fatalf("the ProtectHome probe needs a readable home here: %v", err) |
| + } |
| + marker := filepath.Join(t.TempDir(), "visible-outside") |
| + if !strings.HasPrefix(marker, "/tmp/") { |
| + t.Fatalf("the PrivateTmp probe needs TMPDIR under /tmp (the brief's /tmp/k/zddr3), got marker %s", marker) |
| + } |
| + if err := os.WriteFile(marker, []byte("x"), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + env := []string{ |
| + edgeHostUTSEnv + "=" + uts, |
| + edgeHomeEnv + "=" + home, |
| + edgeMarkerEnv + "=" + marker, |
| + edgeTasksMaxEnv + "=" + tasksMax, |
| + } |
| + |
| + // 2. The unrestricted control: clone3 placement works here, or the |
| + // hardened run below proves nothing. |
| + runEdgeUnit(t, "control", bin, |
| + []string{"Delegate=yes", edgeUnitLifetime}, |
| + append([]string{edgeInnerEnv + "=control"}, env...)) |
| + |
| + // 3. The hardened run. WorkingDirectory=/ because a user unit's default |
| + // is the home directory, which ProtectHome=yes hides; the binary is |
| + // bound read-only because PrivateTmp=yes (and ProtectHome=yes, for a |
| + // binary built under home) would hide it. |
| + hardened := append([]string{"Delegate=yes", edgeUnitLifetime, "WorkingDirectory=/", "BindReadOnlyPaths=" + bin}, props...) |
| + runEdgeUnit(t, "hardened", bin, hardened, |
| + append([]string{edgeInnerEnv + "=hardened", "CARLOS_SHIM_KERNEL=1", "CARLOS_GEN_KERNEL=1"}, env...)) |
| +} |
| + |
| +// edgeProbeEnv is what the probes compare against: the driver's side, and |
| +// this unit's own cgroup. |
| +type edgeProbeEnv struct { |
| + hostUTS, home, marker, tasksMax string |
| + unitCgroup string // below /sys/fs/cgroup |
| +} |
| + |
| +// An edgeProbe reports whether the line's effect is observed in this |
| +// process, and what it saw. The control run requires every probe to read |
| +// false: a probe that reads true without the hardening tells nothing. |
| +type edgeProbe func(t *testing.T, e edgeProbeEnv) (bool, string) |
| + |
| +// edgeProbes holds the observable effect of each applied line that has one |
| +// here. |
| +var edgeProbes = map[string]edgeProbe{ |
| + "NoNewPrivileges": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| + v := procStatusField(t, "NoNewPrivs") |
| + return v == "1", "NoNewPrivs: " + v |
| + }, |
| + "ProtectSystem": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + const p = "/usr/.carlos-edge-hardening-probe" |
| + f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) |
| + if err == nil { |
| + _ = f.Close() |
| + _ = os.Remove(p) |
| + return false, "created " + p |
| + } |
| + return errors.Is(err, syscall.EROFS), "create under /usr: " + errnoName(err) |
| + }, |
| + "ProtectHome": func(_ *testing.T, e edgeProbeEnv) (bool, string) { |
| + _, err := os.ReadDir(e.home) |
| + if err == nil { |
| + return false, "read " + e.home |
| + } |
| + return true, "read " + e.home + ": " + errnoName(err) |
| + }, |
| + "PrivateTmp": func(_ *testing.T, e edgeProbeEnv) (bool, string) { |
| + _, err := os.Stat(e.marker) |
| + if err == nil { |
| + return false, "the driver's /tmp marker is visible" |
| + } |
| + return errors.Is(err, os.ErrNotExist), "the driver's /tmp marker: " + errnoName(err) |
| + }, |
| + "PrivateDevices": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| + m, ok := lastMount(t, "/dev") |
| + if !ok { |
| + return false, "no /dev mount" |
| + } |
| + return m.fstype == "tmpfs", "/dev is " + m.fstype |
| + }, |
| + "ProtectHostname": func(t *testing.T, e edgeProbeEnv) (bool, string) { |
| + uts, err := os.Readlink("/proc/self/ns/uts") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return uts != e.hostUTS, fmt.Sprintf("%s here, %s in the driver", uts, e.hostUTS) |
| + }, |
| + "ProtectKernelTunables": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| + m, ok := lastMount(t, "/proc/sys") |
| + if !ok { |
| + return false, "no /proc/sys mount" |
| + } |
| + return m.readOnly(), "/proc/sys mounted " + m.opts |
| + }, |
| + "ProtectKernelModules": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + _, err := os.ReadDir("/usr/lib/modules") |
| + if err == nil { |
| + return false, "read /usr/lib/modules" |
| + } |
| + return true, "read /usr/lib/modules: " + errnoName(err) |
| + }, |
| + "ProtectKernelLogs": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| + m, ok := lastMount(t, "/proc/kmsg") |
| + if !ok { |
| + return false, "no mount over /proc/kmsg" |
| + } |
| + return true, "/proc/kmsg mounted over (" + m.opts + ")" |
| + }, |
| + "ProtectProc": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + b, err := os.ReadFile("/proc/1/cmdline") |
| + if err != nil { |
| + return true, "/proc/1/cmdline: " + errnoName(err) |
| + } |
| + if len(b) == 0 { |
| + return true, "/proc/1/cmdline is empty" |
| + } |
| + return false, fmt.Sprintf("/proc/1/cmdline readable (%q)", strings.SplitN(string(b), "\x00", 2)[0]) |
| + }, |
| + "RestrictAddressFamilies": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) |
| + if err == nil { |
| + _ = unix.Close(fd) |
| + return false, "socket(AF_NETLINK) succeeded" |
| + } |
| + return true, "socket(AF_NETLINK): " + errnoName(err) |
| + }, |
| + "RestrictNamespaces": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + e := rawClone3() |
| + return e == unix.ENOSYS, "raw clone3: " + errnoName(e) |
| + }, |
| + "LockPersonality": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + // personality is per thread: try PER_LINUX32, and if the change |
| + // went through, restore the previous persona (its return value) on |
| + // the same thread. The lock refuses any persona but the current |
| + // one, the 0xffffffff query included, so there is no query first. |
| + runtime.LockOSThread() |
| + defer runtime.UnlockOSThread() |
| + const perLinux32 = 0x0008 |
| + prev, _, e := unix.RawSyscall(unix.SYS_PERSONALITY, perLinux32, 0, 0) |
| + if e == 0 { |
| + _, _, _ = unix.RawSyscall(unix.SYS_PERSONALITY, prev, 0, 0) |
| + return false, "personality(PER_LINUX32) succeeded" |
| + } |
| + return e == unix.EPERM, "personality(PER_LINUX32): " + errnoName(e) |
| + }, |
| + "MemoryDenyWriteExecute": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + b, err := unix.Mmap(-1, 0, os.Getpagesize(), unix.PROT_READ|unix.PROT_WRITE|unix.PROT_EXEC, unix.MAP_PRIVATE|unix.MAP_ANON) |
| + if err == nil { |
| + _ = unix.Munmap(b) |
| + return false, "mmap(W|X) succeeded" |
| + } |
| + return true, "mmap(W|X): " + errnoName(err) |
| + }, |
| + // ptrace is in @debug, not @system-service. PEEKDATA on a process that |
| + // is not our tracee answers ESRCH when the call is allowed. |
| + "SystemCallFilter": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + e := ptracePeekSelf() |
| + return e != unix.ESRCH, "ptrace(PEEKDATA, self): " + errnoName(e) |
| + }, |
| + "SystemCallErrorNumber": func(*testing.T, edgeProbeEnv) (bool, string) { |
| + e := ptracePeekSelf() |
| + return e == unix.EPERM, "a filtered call answers " + errnoName(e) |
| + }, |
| + "TasksMax": func(t *testing.T, e edgeProbeEnv) (bool, string) { |
| + got := strings.TrimSpace(readFixture(t, filepath.Join("/sys/fs/cgroup", e.unitCgroup, "pids.max"))) |
| + return got == e.tasksMax, "unit pids.max " + got |
| + }, |
| +} |
| + |
| +// edgeNoProbe are the applied lines whose effect cannot be told apart here |
| +// from another line's or from what this user already lacks: untested, so |
| +// the dev re-check on the real unit carries them. |
| +var edgeNoProbe = map[string]string{ |
| + "ProtectClock": "the @clock calls are already refused by SystemCallFilter=@system-service, and /dev/rtc is already hidden by PrivateDevices", |
| + "RestrictRealtime": "RLIMIT_RTPRIO is 0 (hard) for this user, so SCHED_FIFO answers EPERM with or without the line", |
| + "SystemCallArchitectures": "a non-native call needs 32-bit code, which a Go test cannot issue", |
| +} |
| + |
| +// edgeMayGoUnobserved are probed lines a user unit is known not to |
| +// enforce on DOCS-T1. Observed, they are enforced; unobserved, untested. |
| +// Every other probed line must be observed in the hardened run. |
| +var edgeMayGoUnobserved = map[string]string{ |
| + "ProtectProc": "the user manager cannot mount a hidepid procfs (the unit's /proc is the host's)", |
| +} |
| + |
| +func procStatusField(t *testing.T, name string) string { |
| + t.Helper() |
| + for _, line := range strings.Split(readFixture(t, "/proc/self/status"), "\n") { |
| + if v, ok := strings.CutPrefix(line, name+":"); ok { |
| + return strings.TrimSpace(v) |
| + } |
| + } |
| + t.Fatalf("/proc/self/status has no %s line", name) |
| + return "" |
| +} |
| + |
| +type mountEntry struct{ point, opts, fstype string } |
| + |
| +func (m mountEntry) readOnly() bool { |
| + for _, o := range strings.Split(m.opts, ",") { |
| + if o == "ro" { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// lastMount returns the topmost mount at point, from /proc/self/mountinfo. |
| +func lastMount(t *testing.T, point string) (mountEntry, bool) { |
| + t.Helper() |
| + var found mountEntry |
| + ok := false |
| + for _, line := range strings.Split(readFixture(t, "/proc/self/mountinfo"), "\n") { |
| + pre, post, sep := strings.Cut(line, " - ") |
| + f := strings.Fields(pre) |
| + if !sep || len(f) < 6 || f[4] != point { |
| + continue |
| + } |
| + ff := strings.Fields(post) |
| + if len(ff) == 0 { |
| + continue |
| + } |
| + found, ok = mountEntry{point: f[4], opts: f[5], fstype: ff[0]}, true |
| + } |
| + return found, ok |
| +} |
| + |
| +func errnoName(err error) string { |
| + var e syscall.Errno |
| + if errors.As(err, &e) { |
| + if n := unix.ErrnoName(e); n != "" { |
| + return n |
| + } |
| + } |
| + if err == nil { |
| + return "ok" |
| + } |
| + return err.Error() |
| +} |
| + |
| +// rawClone3 calls clone3(NULL, 0): an allowed clone3 answers EINVAL (the |
| +// size is below CLONE_ARGS_SIZE_VER0) and forks nothing; a filtered one |
| +// answers what the filter says. |
| +func rawClone3() syscall.Errno { |
| + _, _, e := unix.RawSyscall(unix.SYS_CLONE3, 0, 0, 0) |
| + return e |
| +} |
| + |
| +func ptracePeekSelf() syscall.Errno { |
| + _, _, e := unix.RawSyscall6(unix.SYS_PTRACE, unix.PTRACE_PEEKDATA, uintptr(os.Getpid()), 0, 0, 0, 0) |
| + return e |
| +} |
| + |
| +// plainCgroupStart is Go's own clone3 placement, the one the shim replaced: |
| +// exec.Cmd with UseCgroupFD into dir. |
| +func plainCgroupStart(t *testing.T, dir string) (*exec.Cmd, error) { |
| + t.Helper() |
| + f, err := os.Open(dir) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer f.Close() |
| + cmd := exec.Command("/bin/sleep", "60") |
| + cmd.Env = []string{} |
| + cmd.SysProcAttr = &syscall.SysProcAttr{UseCgroupFD: true, CgroupFD: int(f.Fd())} |
| + return cmd, cmd.Start() |
| +} |
| + |
| +// TestEdgeHardeningInner runs only in the driver's units. With the |
| +// variable set nothing in it skips: a precondition it cannot establish |
| +// fails it. |
| +func TestEdgeHardeningInner(t *testing.T) { |
| + mode := os.Getenv(edgeInnerEnv) |
| + switch mode { |
| + case "": |
| + t.Skip("runs inside TestEdgeHardeningPlacement's units only") |
| + case "control", "hardened": |
| + default: |
| + t.Fatalf("%s=%q, want control or hardened", edgeInnerEnv, mode) |
| + } |
| + pe := edgeProbeEnv{ |
| + hostUTS: os.Getenv(edgeHostUTSEnv), |
| + home: os.Getenv(edgeHomeEnv), |
| + marker: os.Getenv(edgeMarkerEnv), |
| + tasksMax: os.Getenv(edgeTasksMaxEnv), |
| + } |
| + if pe.hostUTS == "" || pe.home == "" || pe.marker == "" || pe.tasksMax == "" { |
| + t.Fatalf("the driver's probe inputs are missing: %+v", pe) |
| + } |
| + rel, ok := ParseSelfCgroup(readFixture(t, "/proc/self/cgroup")) |
| + if !ok { |
| + t.Fatal("no unified cgroup path") |
| + } |
| + if filepath.Base(rel) == "supervise" { |
| + rel = filepath.Dir(rel) |
| + } |
| + pe.unitCgroup = rel |
| + |
| + // Step 0: what the unit enforces, observed from inside. |
| + seccomp := procStatusField(t, "Seccomp") |
| + t.Logf("[%s] Seccomp: %s, NoNewPrivs: %s", mode, seccomp, procStatusField(t, "NoNewPrivs")) |
| + if mode == "hardened" && seccomp != "2" { |
| + t.Fatalf("Seccomp: %s, want 2 (filter mode)", seccomp) |
| + } |
| + if mode == "control" && seccomp != "0" { |
| + t.Fatalf("Seccomp: %s in the control, want 0", seccomp) |
| + } |
| + t.Logf("[%s] %-24s %-9s %s", mode, "property", "status", "observation") |
| + var untested []string |
| + for _, k := range edgeApplied { |
| + probe, probed := edgeProbes[k] |
| + why, unprobed := edgeNoProbe[k] |
| + if probed == unprobed { |
| + t.Fatalf("%s needs exactly one of a probe or a no-probe reason", k) |
| + } |
| + if unprobed { |
| + t.Logf("[%s] %-24s %-9s %s", mode, k, "untested", "no probe: "+why) |
| + untested = append(untested, k) |
| + continue |
| + } |
| + seen, what := probe(t, pe) |
| + status := "enforced" |
| + switch { |
| + case mode == "control" && seen: |
| + t.Fatalf("control: the %s probe reads enforced without the hardening (%s), so it tells nothing", k, what) |
| + case mode == "control": |
| + status = "absent" |
| + case !seen && edgeMayGoUnobserved[k] != "": |
| + status = "untested" |
| + what += "; " + edgeMayGoUnobserved[k] |
| + untested = append(untested, k) |
| + case !seen: |
| + t.Fatalf("hardened: %s is applied but not observed: %s", k, what) |
| + } |
| + t.Logf("[%s] %-24s %-9s %s", mode, k, status, what) |
| + } |
| + if mode == "hardened" { |
| + t.Logf("[hardened] untested here, for the dev re-check on the real unit: %s", strings.Join(untested, ", ")) |
| + } |
| + |
| + // The delegation: this test is the gate, so it fails rather than skips. |
| + t.Setenv("CARLOS_SHIM_KERNEL", "1") |
| + m := shimKernelManager(t) |
| + scratch := filepath.Join(m.Root(), "edge-hardening-scratch") |
| + if err := os.Mkdir(scratch, 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { |
| + if err := os.Remove(scratch); err != nil { |
| + t.Errorf("remove %s: %v", scratch, err) |
| + } |
| + }) |
| + |
| + // Step 1 (and the control's whole point): clone3 and Go's UseCgroupFD |
| + // start work without the hardening and answer ENOSYS under it. |
| + e := rawClone3() |
| + cmd, err := plainCgroupStart(t, scratch) |
| + if mode == "control" { |
| + if e != unix.EINVAL { |
| + t.Fatalf("control: raw clone3 answered %s, want EINVAL (clone3 allowed)", errnoName(e)) |
| + } |
| + if err != nil { |
| + t.Fatalf("control: UseCgroupFD start: %v", err) |
| + } |
| + k := &kernelKid{cmd: cmd, done: make(chan struct{})} |
| + got, cgErr := cgroupOf(cmd.Process.Pid) |
| + _ = cmd.Process.Kill() |
| + <-k.reaped() |
| + if want := strings.TrimPrefix(scratch, "/sys/fs/cgroup/"); cgErr != nil || got != want { |
| + t.Fatalf("control: the UseCgroupFD child ran in %q (%v), want %s", got, cgErr, want) |
| + } |
| + t.Logf("[control] raw clone3: EINVAL; UseCgroupFD start landed in %s", got) |
| + return |
| + } |
| + if e != unix.ENOSYS { |
| + t.Fatalf("hardened: raw clone3 answered %s, want ENOSYS", errnoName(e)) |
| + } |
| + if err == nil { |
| + _ = cmd.Process.Kill() |
| + _ = cmd.Wait() |
| + t.Fatal("hardened: a UseCgroupFD start succeeded under the filter") |
| + } |
| + if !errors.Is(err, syscall.ENOSYS) { |
| + t.Fatalf("hardened: UseCgroupFD start: %v, want ENOSYS", err) |
| + } |
| + t.Logf("[hardened] raw clone3: ENOSYS; UseCgroupFD start: %v", err) |
| + |
| + // Step 2: placement through the shim succeeds. |
| + const host = "hardening.example" |
| + lim := Limit{MemoryMax: 64 << 20} |
| + parent := m.hostDir(host) |
| + base := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host)) |
| + // Registered first, so it runs last: after every child is reaped. |
| + t.Cleanup(func() { |
| + m.RemoveChild(host, GenChild(1)) |
| + m.RemoveChild(host, GenChild(2)) |
| + m.CleanupHost(host) |
| + }) |
| + var tenantIn io.WriteCloser |
| + var tenantOut io.ReadCloser |
| + var pipeErr error |
| + tenantCmd, err := m.StartChildGated(host, GenChild(1), lim, lim, func() *exec.Cmd { |
| + c := exec.Command("/bin/sh", "-c", `echo "$$"; cat "/proc/$$/cgroup"; read -r _`) |
| + c.Env = []string{"PATH=/usr/bin:/bin"} |
| + c.Stderr = os.Stderr |
| + if tenantIn, pipeErr = c.StdinPipe(); pipeErr == nil { |
| + tenantOut, pipeErr = c.StdoutPipe() |
| + } |
| + return c |
| + }, nil) |
| + tenant := reapOnCleanup(t, m, host, GenChild(1), tenantCmd, tenantIn) |
| + if err != nil { |
| + t.Fatalf("hardened: StartChildGated g1: %v", err) |
| + } |
| + if pipeErr != nil { |
| + t.Fatal(pipeErr) |
| + } |
| + type said struct { |
| + pid, cg string |
| + err error |
| + } |
| + saidc := make(chan said, 1) |
| + go func() { |
| + r := bufio.NewReader(tenantOut) |
| + pid, err := r.ReadString('\n') |
| + if err != nil { |
| + saidc <- said{err: err} |
| + return |
| + } |
| + cg, err := r.ReadString('\n') |
| + saidc <- said{pid: strings.TrimSpace(pid), cg: cg, err: err} |
| + }() |
| + var s said |
| + select { |
| + case s = <-saidc: |
| + case <-time.After(10 * time.Second): |
| + t.Fatal("hardened: the tenant printed nothing within 10s") |
| + } |
| + if s.err != nil { |
| + t.Fatalf("hardened: reading the tenant: %v", s.err) |
| + } |
| + wantG1 := filepath.Join(base, GenChild(1)) |
| + gotCg, ok := ParseSelfCgroup(s.cg) |
| + if !ok || gotCg != wantG1 { |
| + t.Fatalf("hardened: the tenant says it runs in %q, want %s", s.cg, wantG1) |
| + } |
| + if s.pid != strconv.Itoa(tenantCmd.Process.Pid) { |
| + t.Fatalf("hardened: the tenant's pid is %s, cmd.Process.Pid is %d", s.pid, tenantCmd.Process.Pid) |
| + } |
| + if comm, err := os.ReadFile(fmt.Sprintf("/proc/%d/comm", tenantCmd.Process.Pid)); err != nil || strings.TrimSpace(string(comm)) != "sh" { |
| + t.Fatalf("hardened: pid %d is %q (%v), want the exec'd tenant, sh", tenantCmd.Process.Pid, comm, err) |
| + } |
| + t.Logf("[hardened] g1 tenant pid %s = cmd.Process.Pid, in /%s", s.pid, gotCg) |
| + |
| + // Step 3: the limit applies. The parent is raised as a swap raises it, |
| + // so g2's own memory.max is the one that bites. |
| + if err := m.SetParentLimit(host, OverlapLimit(lim, lim)); err != nil { |
| + t.Fatal(err) |
| + } |
| + var hogIn io.WriteCloser |
| + var hogPipeErr error |
| + hogCmd, err := m.StartChildGated(host, GenChild(2), lim, lim, func() *exec.Cmd { |
| + c := genKernelCmd("hog") |
| + hogIn, hogPipeErr = c.StdinPipe() |
| + return c |
| + }, nil) |
| + hog := reapOnCleanup(t, m, host, GenChild(2), hogCmd, hogIn) |
| + if err != nil { |
| + t.Fatalf("hardened: StartChildGated g2: %v", err) |
| + } |
| + if hogPipeErr != nil { |
| + t.Fatal(hogPipeErr) |
| + } |
| + g2 := filepath.Join(parent, GenChild(2)) |
| + expectFile(t, filepath.Join(g2, "memory.max"), "67108864") |
| + if got, err := cgroupOf(hogCmd.Process.Pid); err != nil || got != filepath.Join(base, GenChild(2)) { |
| + t.Fatalf("hardened: the hog runs in %s (%v), want %s", got, err, filepath.Join(base, GenChild(2))) |
| + } |
| + if _, err := fmt.Fprintln(hogIn, "go"); err != nil { |
| + t.Fatalf("hog go-ahead: %v", err) |
| + } |
| + select { |
| + case <-hog.reaped(): |
| + case <-time.After(60 * time.Second): |
| + t.Fatal("hardened: the hog was not OOM-killed within 60s") |
| + } |
| + var exitErr *exec.ExitError |
| + if !errors.As(hog.err, &exitErr) { |
| + t.Fatalf("hardened: hog exit: %v", hog.err) |
| + } |
| + if ws, ok := exitErr.Sys().(syscall.WaitStatus); !ok || !ws.Signaled() || ws.Signal() != syscall.SIGKILL { |
| + t.Fatalf("hardened: the hog ended with %v, want SIGKILL from the OOM killer", hog.err) |
| + } |
| + kills := oomKills(t, g2) |
| + if kills < 1 { |
| + t.Fatalf("hardened: g2 memory.events oom_kill = %d, want at least 1", kills) |
| + } |
| + if n := oomKills(t, filepath.Join(parent, GenChild(1))); n != 0 { |
| + t.Fatalf("hardened: g1 saw %d OOM kills", n) |
| + } |
| + select { |
| + case <-tenant.reaped(): |
| + t.Fatalf("hardened: g1's tenant died beside the hog: %v", tenant.err) |
| + default: |
| + } |
| + if err := syscall.Kill(tenantCmd.Process.Pid, 0); err != nil { |
| + t.Fatalf("hardened: g1's tenant is gone: %v", err) |
| + } |
| + t.Logf("[hardened] g2 hog: %v, g2 oom_kill %d; g1 tenant alive", hog.err, kills) |
| + |
| + // Step 4: arming would arm. |
| + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) |
| + defer cancel() |
| + if err := m.VerifyPlacement(ctx); err != nil { |
| + t.Fatalf("hardened: VerifyPlacement: %v", err) |
| + } |
| + t.Logf("[hardened] VerifyPlacement: nil") |
| +} |