| 1 | //go:build linux |
| 2 | |
| 3 | package cgroup |
| 4 | |
| 5 | import ( |
| 6 | "bufio" |
| 7 | "bytes" |
| 8 | "context" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "path/filepath" |
| 15 | "runtime" |
| 16 | "strconv" |
| 17 | "strings" |
| 18 | "sync" |
| 19 | "syscall" |
| 20 | "testing" |
| 21 | "time" |
| 22 | |
| 23 | "golang.org/x/sys/unix" |
| 24 | ) |
| 25 | |
| 26 | // The shim placement under carlos-edge.service's own sandbox lines (zdd |
| 27 | // readiness 3, Task 4; amadan #272). The shipped unit's RestrictNamespaces= |
| 28 | // yes has systemd's seccomp filter answer clone3 with ENOSYS, which broke |
| 29 | // clone3 placement on the dev box. TestEdgeHardeningPlacement copies the |
| 30 | // unit's [Service] sandbox lines onto a transient user unit and runs |
| 31 | // TestEdgeHardeningInner there, after a control run in a unit without them. |
| 32 | // Opt-in: CARLOS_EDGE_HARDENING_KERNEL=1, run outside any unit, unsandboxed, |
| 33 | // with TMPDIR under /tmp (the brief's /tmp/k/zddr3). |
| 34 | |
| 35 | const ( |
| 36 | edgeUnitPath = "../host/units/carlos-edge.service" |
| 37 | edgeInnerEnv = "CARLOS_EDGE_HARDENING_INNER" |
| 38 | edgeHostUTSEnv = "CARLOS_EDGE_HARDENING_HOST_UTS" |
| 39 | edgeHomeEnv = "CARLOS_EDGE_HARDENING_HOME" |
| 40 | edgeMarkerEnv = "CARLOS_EDGE_HARDENING_MARKER" |
| 41 | edgeTasksMaxEnv = "CARLOS_EDGE_HARDENING_TASKS_MAX" |
| 42 | edgeUnitLifetime = "RuntimeMaxSec=300" |
| 43 | ) |
| 44 | |
| 45 | // edgeApplied are the [Service] keys passed to the hardened unit as |
| 46 | // -p Key=value, exactly as the unit file has them. |
| 47 | var edgeApplied = []string{ |
| 48 | "NoNewPrivileges", |
| 49 | "ProtectSystem", |
| 50 | "ProtectHome", |
| 51 | "PrivateTmp", |
| 52 | "PrivateDevices", |
| 53 | "ProtectClock", |
| 54 | "ProtectHostname", |
| 55 | "ProtectKernelTunables", |
| 56 | "ProtectKernelModules", |
| 57 | "ProtectKernelLogs", |
| 58 | "ProtectProc", |
| 59 | "RestrictAddressFamilies", |
| 60 | "RestrictNamespaces", |
| 61 | "RestrictRealtime", |
| 62 | "LockPersonality", |
| 63 | "MemoryDenyWriteExecute", |
| 64 | "SystemCallFilter", |
| 65 | "SystemCallErrorNumber", |
| 66 | "SystemCallArchitectures", |
| 67 | "TasksMax", |
| 68 | } |
| 69 | |
| 70 | // edgeExcluded are the [Service] keys the hardened unit does not get, each |
| 71 | // with its reason. A [Service] key in neither table fails the driver, so a |
| 72 | // new line in the unit forces a decision here. |
| 73 | var edgeExcluded = map[string]string{ |
| 74 | "EnvironmentFile": "process control, not sandbox: names the box's /etc/carlos/host.env", |
| 75 | "ExecStart": "process control, not sandbox: the unit runs the test binary instead", |
| 76 | "ExecReload": "process control, not sandbox", |
| 77 | "Restart": "process control, not sandbox", |
| 78 | "RestartSec": "process control, not sandbox", |
| 79 | "User": "a user manager cannot change identity", |
| 80 | "Group": "a user manager cannot change identity", |
| 81 | "AmbientCapabilities": "a user manager cannot grant capabilities", |
| 82 | "CapabilityBoundingSet": "a user manager cannot change capabilities", |
| 83 | "ReadWritePaths": "names box paths (/data, /run/carlos) absent here", |
| 84 | "Slice": "carlos.slice is the box's system slice; a user unit stays in the user manager's tree", |
| 85 | "OOMPolicy": "unit lifecycle on an OOM kill, not sandbox; the test reads memory.events itself", |
| 86 | "Delegate": "the test passes its own Delegate=yes (a user manager delegates cpu memory pids, no io)", |
| 87 | } |
| 88 | |
| 89 | // edgeMustApply are the lines the brief requires applied: each must be in |
| 90 | // the unit file and in edgeApplied. |
| 91 | var edgeMustApply = []string{ |
| 92 | "RestrictNamespaces", "SystemCallFilter", "SystemCallErrorNumber", "SystemCallArchitectures", |
| 93 | "NoNewPrivileges", "MemoryDenyWriteExecute", "LockPersonality", "RestrictRealtime", |
| 94 | "RestrictAddressFamilies", "ProtectProc", "ProtectClock", "ProtectHostname", |
| 95 | "ProtectKernelTunables", "ProtectKernelModules", "ProtectKernelLogs", "PrivateDevices", |
| 96 | "ProtectSystem", "TasksMax", |
| 97 | } |
| 98 | |
| 99 | type edgeLine struct{ key, value string } |
| 100 | |
| 101 | // readEdgeService returns every Key=value line of path's [Service] section, |
| 102 | // in file order. |
| 103 | func readEdgeService(t *testing.T, path string) []edgeLine { |
| 104 | t.Helper() |
| 105 | raw, err := os.ReadFile(path) |
| 106 | if err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | var lines []edgeLine |
| 110 | section := "" |
| 111 | for n, line := range strings.Split(string(raw), "\n") { |
| 112 | line = strings.TrimSpace(line) |
| 113 | switch { |
| 114 | case line == "", strings.HasPrefix(line, "#"), strings.HasPrefix(line, ";"): |
| 115 | continue |
| 116 | case strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]"): |
| 117 | section = line |
| 118 | continue |
| 119 | case section != "[Service]": |
| 120 | continue |
| 121 | case strings.HasSuffix(line, `\`): |
| 122 | t.Fatalf("%s:%d: a continued line; the driver reads one line per key", path, n+1) |
| 123 | } |
| 124 | key, value, ok := strings.Cut(line, "=") |
| 125 | if !ok { |
| 126 | t.Fatalf("%s:%d: %q is no Key=value line", path, n+1, line) |
| 127 | } |
| 128 | lines = append(lines, edgeLine{strings.TrimSpace(key), strings.TrimSpace(value)}) |
| 129 | } |
| 130 | if len(lines) == 0 { |
| 131 | t.Fatalf("%s has no [Service] lines", path) |
| 132 | } |
| 133 | return lines |
| 134 | } |
| 135 | |
| 136 | // edgeLog is the driver's view of a unit's output: each line goes to the |
| 137 | // test log as it arrives, and the whole is kept for the verdict. |
| 138 | type edgeLog struct { |
| 139 | t *testing.T |
| 140 | label string |
| 141 | mu sync.Mutex |
| 142 | part []byte |
| 143 | all bytes.Buffer |
| 144 | } |
| 145 | |
| 146 | func (w *edgeLog) Write(p []byte) (int, error) { |
| 147 | w.mu.Lock() |
| 148 | defer w.mu.Unlock() |
| 149 | w.all.Write(p) |
| 150 | w.part = append(w.part, p...) |
| 151 | for { |
| 152 | i := bytes.IndexByte(w.part, '\n') |
| 153 | if i < 0 { |
| 154 | break |
| 155 | } |
| 156 | w.t.Logf("%s| %s", w.label, w.part[:i]) |
| 157 | w.part = w.part[i+1:] |
| 158 | } |
| 159 | return len(p), nil |
| 160 | } |
| 161 | |
| 162 | func (w *edgeLog) flush() { |
| 163 | w.mu.Lock() |
| 164 | defer w.mu.Unlock() |
| 165 | if len(w.part) > 0 { |
| 166 | w.t.Logf("%s| %s", w.label, w.part) |
| 167 | w.part = nil |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | // runEdgeUnit runs the inner test in a transient user unit with props and |
| 172 | // fails unless it passed, without a skip. |
| 173 | func runEdgeUnit(t *testing.T, label, bin string, props, env []string) { |
| 174 | t.Helper() |
| 175 | args := []string{"--user", "--wait", "--pipe", "--collect"} |
| 176 | for _, p := range props { |
| 177 | args = append(args, "-p", p) |
| 178 | } |
| 179 | for _, e := range env { |
| 180 | args = append(args, "-E", e) |
| 181 | } |
| 182 | args = append(args, bin, "-test.run", "^TestEdgeHardeningInner$", "-test.v", "-test.count=1") |
| 183 | t.Logf("%s: systemd-run %s", label, strings.Join(quoteArgs(args), " ")) |
| 184 | out := &edgeLog{t: t, label: label} |
| 185 | cmd := exec.Command("systemd-run", args...) |
| 186 | cmd.Stdout = out |
| 187 | cmd.Stderr = out |
| 188 | err := cmd.Run() |
| 189 | out.flush() |
| 190 | all := out.all.String() |
| 191 | if err != nil { |
| 192 | t.Fatalf("%s: systemd-run: %v", label, err) |
| 193 | } |
| 194 | if !strings.Contains(all, "--- PASS: TestEdgeHardeningInner ") { |
| 195 | t.Fatalf("%s: no --- PASS line for TestEdgeHardeningInner", label) |
| 196 | } |
| 197 | if strings.Contains(all, "--- SKIP") || strings.Contains(all, "--- FAIL") { |
| 198 | t.Fatalf("%s: the inner run skipped or failed", label) |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | func quoteArgs(args []string) []string { |
| 203 | q := make([]string, len(args)) |
| 204 | for i, a := range args { |
| 205 | if strings.ContainsAny(a, " $'\"") { |
| 206 | a = strconv.Quote(a) |
| 207 | } |
| 208 | q[i] = a |
| 209 | } |
| 210 | return q |
| 211 | } |
| 212 | |
| 213 | // TestEdgeHardeningPlacement is the driver: an unrestricted control, where |
| 214 | // clone3 placement works, then the inner test under the edge unit's |
| 215 | // sandbox lines, where clone3 answers ENOSYS and the shim must place. |
| 216 | func TestEdgeHardeningPlacement(t *testing.T) { |
| 217 | if os.Getenv("CARLOS_EDGE_HARDENING_KERNEL") != "1" { |
| 218 | t.Skip("set CARLOS_EDGE_HARDENING_KERNEL=1, outside any unit, unsandboxed (zdd readiness 3, Task 4)") |
| 219 | } |
| 220 | if os.Getenv(edgeInnerEnv) != "" { |
| 221 | t.Fatalf("the driver runs outside the unit, but %s is set", edgeInnerEnv) |
| 222 | } |
| 223 | if _, err := exec.LookPath("systemd-run"); err != nil { |
| 224 | t.Fatal(err) |
| 225 | } |
| 226 | |
| 227 | // 1. The unit's [Service] lines, each in exactly one table. |
| 228 | lines := readEdgeService(t, edgeUnitPath) |
| 229 | applied := map[string]bool{} |
| 230 | for _, k := range edgeApplied { |
| 231 | if _, both := edgeExcluded[k]; both { |
| 232 | t.Fatalf("%s is in both the applied and the excluded table", k) |
| 233 | } |
| 234 | applied[k] = true |
| 235 | } |
| 236 | inUnit := map[string]bool{} |
| 237 | var props []string |
| 238 | tasksMax := "" |
| 239 | for _, l := range lines { |
| 240 | inUnit[l.key] = true |
| 241 | _, excluded := edgeExcluded[l.key] |
| 242 | switch { |
| 243 | case applied[l.key]: |
| 244 | props = append(props, l.key+"="+l.value) |
| 245 | if l.key == "TasksMax" { |
| 246 | tasksMax = l.value |
| 247 | } |
| 248 | case excluded: |
| 249 | default: |
| 250 | t.Fatalf("%s: [Service] %s=%s is in neither the applied nor the excluded table: decide which, and why", edgeUnitPath, l.key, l.value) |
| 251 | } |
| 252 | } |
| 253 | for _, k := range edgeApplied { |
| 254 | if !inUnit[k] { |
| 255 | t.Fatalf("applied table names %s, which %s no longer has", k, edgeUnitPath) |
| 256 | } |
| 257 | } |
| 258 | for k := range edgeExcluded { |
| 259 | if !inUnit[k] { |
| 260 | t.Fatalf("excluded table names %s, which %s no longer has", k, edgeUnitPath) |
| 261 | } |
| 262 | } |
| 263 | for _, k := range edgeMustApply { |
| 264 | if !inUnit[k] || !applied[k] { |
| 265 | t.Fatalf("%s must be in %s and applied (in unit %v, applied %v)", k, edgeUnitPath, inUnit[k], applied[k]) |
| 266 | } |
| 267 | } |
| 268 | t.Logf("applied (%d), passed as -p Key=value:", len(props)) |
| 269 | for _, p := range props { |
| 270 | t.Logf(" %s", p) |
| 271 | } |
| 272 | t.Logf("excluded (%d):", len(edgeExcluded)) |
| 273 | for _, l := range lines { |
| 274 | if r, ok := edgeExcluded[l.key]; ok { |
| 275 | t.Logf(" %-22s %s", l.key, r) |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // The probes' inputs: what the inner run compares against this side. |
| 280 | bin, err := os.Executable() |
| 281 | if err != nil { |
| 282 | t.Fatal(err) |
| 283 | } |
| 284 | uts, err := os.Readlink("/proc/self/ns/uts") |
| 285 | if err != nil { |
| 286 | t.Fatal(err) |
| 287 | } |
| 288 | home, err := os.UserHomeDir() |
| 289 | if err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | if _, err := os.ReadDir(home); err != nil { |
| 293 | t.Fatalf("the ProtectHome probe needs a readable home here: %v", err) |
| 294 | } |
| 295 | marker := filepath.Join(t.TempDir(), "visible-outside") |
| 296 | if !strings.HasPrefix(marker, "/tmp/") { |
| 297 | t.Fatalf("the PrivateTmp probe needs TMPDIR under /tmp (the brief's /tmp/k/zddr3), got marker %s", marker) |
| 298 | } |
| 299 | if err := os.WriteFile(marker, []byte("x"), 0o600); err != nil { |
| 300 | t.Fatal(err) |
| 301 | } |
| 302 | env := []string{ |
| 303 | edgeHostUTSEnv + "=" + uts, |
| 304 | edgeHomeEnv + "=" + home, |
| 305 | edgeMarkerEnv + "=" + marker, |
| 306 | edgeTasksMaxEnv + "=" + tasksMax, |
| 307 | } |
| 308 | |
| 309 | // 2. The unrestricted control: clone3 placement works here, or the |
| 310 | // hardened run below proves nothing. |
| 311 | runEdgeUnit(t, "control", bin, |
| 312 | []string{"Delegate=yes", edgeUnitLifetime}, |
| 313 | append([]string{edgeInnerEnv + "=control"}, env...)) |
| 314 | |
| 315 | // 3. The hardened run. WorkingDirectory=/ because a user unit's default |
| 316 | // is the home directory, which ProtectHome=yes hides; the binary is |
| 317 | // bound read-only because PrivateTmp=yes (and ProtectHome=yes, for a |
| 318 | // binary built under home) would hide it. |
| 319 | hardened := append([]string{"Delegate=yes", edgeUnitLifetime, "WorkingDirectory=/", "BindReadOnlyPaths=" + bin}, props...) |
| 320 | runEdgeUnit(t, "hardened", bin, hardened, |
| 321 | append([]string{edgeInnerEnv + "=hardened", "CARLOS_SHIM_KERNEL=1", "CARLOS_GEN_KERNEL=1"}, env...)) |
| 322 | } |
| 323 | |
| 324 | // edgeProbeEnv is what the probes compare against: the driver's side, and |
| 325 | // this unit's own cgroup. |
| 326 | type edgeProbeEnv struct { |
| 327 | hostUTS, home, marker, tasksMax string |
| 328 | unitCgroup string // below /sys/fs/cgroup |
| 329 | } |
| 330 | |
| 331 | // An edgeProbe reports whether the line's effect is observed in this |
| 332 | // process, and what it saw. The control run requires every probe to read |
| 333 | // false: a probe that reads true without the hardening tells nothing. |
| 334 | type edgeProbe func(t *testing.T, e edgeProbeEnv) (bool, string) |
| 335 | |
| 336 | // edgeProbes holds the observable effect of each applied line that has one |
| 337 | // here. |
| 338 | var edgeProbes = map[string]edgeProbe{ |
| 339 | "NoNewPrivileges": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| 340 | v := procStatusField(t, "NoNewPrivs") |
| 341 | return v == "1", "NoNewPrivs: " + v |
| 342 | }, |
| 343 | "ProtectSystem": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 344 | const p = "/usr/.carlos-edge-hardening-probe" |
| 345 | f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) |
| 346 | if err == nil { |
| 347 | _ = f.Close() |
| 348 | _ = os.Remove(p) |
| 349 | return false, "created " + p |
| 350 | } |
| 351 | return errors.Is(err, syscall.EROFS), "create under /usr: " + errnoName(err) |
| 352 | }, |
| 353 | "ProtectHome": func(_ *testing.T, e edgeProbeEnv) (bool, string) { |
| 354 | _, err := os.ReadDir(e.home) |
| 355 | if err == nil { |
| 356 | return false, "read " + e.home |
| 357 | } |
| 358 | return true, "read " + e.home + ": " + errnoName(err) |
| 359 | }, |
| 360 | "PrivateTmp": func(_ *testing.T, e edgeProbeEnv) (bool, string) { |
| 361 | _, err := os.Stat(e.marker) |
| 362 | if err == nil { |
| 363 | return false, "the driver's /tmp marker is visible" |
| 364 | } |
| 365 | return errors.Is(err, os.ErrNotExist), "the driver's /tmp marker: " + errnoName(err) |
| 366 | }, |
| 367 | "PrivateDevices": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| 368 | m, ok := lastMount(t, "/dev") |
| 369 | if !ok { |
| 370 | return false, "no /dev mount" |
| 371 | } |
| 372 | return m.fstype == "tmpfs", "/dev is " + m.fstype |
| 373 | }, |
| 374 | "ProtectHostname": func(t *testing.T, e edgeProbeEnv) (bool, string) { |
| 375 | uts, err := os.Readlink("/proc/self/ns/uts") |
| 376 | if err != nil { |
| 377 | t.Fatal(err) |
| 378 | } |
| 379 | return uts != e.hostUTS, fmt.Sprintf("%s here, %s in the driver", uts, e.hostUTS) |
| 380 | }, |
| 381 | "ProtectKernelTunables": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| 382 | m, ok := lastMount(t, "/proc/sys") |
| 383 | if !ok { |
| 384 | return false, "no /proc/sys mount" |
| 385 | } |
| 386 | return m.readOnly(), "/proc/sys mounted " + m.opts |
| 387 | }, |
| 388 | "ProtectKernelModules": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 389 | _, err := os.ReadDir("/usr/lib/modules") |
| 390 | if err == nil { |
| 391 | return false, "read /usr/lib/modules" |
| 392 | } |
| 393 | return true, "read /usr/lib/modules: " + errnoName(err) |
| 394 | }, |
| 395 | "ProtectKernelLogs": func(t *testing.T, _ edgeProbeEnv) (bool, string) { |
| 396 | m, ok := lastMount(t, "/proc/kmsg") |
| 397 | if !ok { |
| 398 | return false, "no mount over /proc/kmsg" |
| 399 | } |
| 400 | return true, "/proc/kmsg mounted over (" + m.opts + ")" |
| 401 | }, |
| 402 | "ProtectProc": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 403 | b, err := os.ReadFile("/proc/1/cmdline") |
| 404 | if err != nil { |
| 405 | return true, "/proc/1/cmdline: " + errnoName(err) |
| 406 | } |
| 407 | if len(b) == 0 { |
| 408 | return true, "/proc/1/cmdline is empty" |
| 409 | } |
| 410 | return false, fmt.Sprintf("/proc/1/cmdline readable (%q)", strings.SplitN(string(b), "\x00", 2)[0]) |
| 411 | }, |
| 412 | "RestrictAddressFamilies": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 413 | fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) |
| 414 | if err == nil { |
| 415 | _ = unix.Close(fd) |
| 416 | return false, "socket(AF_NETLINK) succeeded" |
| 417 | } |
| 418 | return true, "socket(AF_NETLINK): " + errnoName(err) |
| 419 | }, |
| 420 | "RestrictNamespaces": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 421 | e := rawClone3() |
| 422 | return e == unix.ENOSYS, "raw clone3: " + errnoName(e) |
| 423 | }, |
| 424 | "LockPersonality": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 425 | // personality is per thread: try PER_LINUX32, and if the change |
| 426 | // went through, restore the previous persona (its return value) on |
| 427 | // the same thread. The lock refuses any persona but the current |
| 428 | // one, the 0xffffffff query included, so there is no query first. |
| 429 | runtime.LockOSThread() |
| 430 | defer runtime.UnlockOSThread() |
| 431 | const perLinux32 = 0x0008 |
| 432 | prev, _, e := unix.RawSyscall(unix.SYS_PERSONALITY, perLinux32, 0, 0) |
| 433 | if e == 0 { |
| 434 | _, _, _ = unix.RawSyscall(unix.SYS_PERSONALITY, prev, 0, 0) |
| 435 | return false, "personality(PER_LINUX32) succeeded" |
| 436 | } |
| 437 | return e == unix.EPERM, "personality(PER_LINUX32): " + errnoName(e) |
| 438 | }, |
| 439 | "MemoryDenyWriteExecute": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 440 | b, err := unix.Mmap(-1, 0, os.Getpagesize(), unix.PROT_READ|unix.PROT_WRITE|unix.PROT_EXEC, unix.MAP_PRIVATE|unix.MAP_ANON) |
| 441 | if err == nil { |
| 442 | _ = unix.Munmap(b) |
| 443 | return false, "mmap(W|X) succeeded" |
| 444 | } |
| 445 | return true, "mmap(W|X): " + errnoName(err) |
| 446 | }, |
| 447 | // ptrace is in @debug, not @system-service. PEEKDATA on a process that |
| 448 | // is not our tracee answers ESRCH when the call is allowed. |
| 449 | "SystemCallFilter": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 450 | e := ptracePeekSelf() |
| 451 | return e != unix.ESRCH, "ptrace(PEEKDATA, self): " + errnoName(e) |
| 452 | }, |
| 453 | "SystemCallErrorNumber": func(*testing.T, edgeProbeEnv) (bool, string) { |
| 454 | e := ptracePeekSelf() |
| 455 | return e == unix.EPERM, "a filtered call answers " + errnoName(e) |
| 456 | }, |
| 457 | "TasksMax": func(t *testing.T, e edgeProbeEnv) (bool, string) { |
| 458 | got := strings.TrimSpace(readFixture(t, filepath.Join("/sys/fs/cgroup", e.unitCgroup, "pids.max"))) |
| 459 | return got == e.tasksMax, "unit pids.max " + got |
| 460 | }, |
| 461 | } |
| 462 | |
| 463 | // edgeNoProbe are the applied lines whose effect cannot be told apart here |
| 464 | // from another line's or from what this user already lacks: untested, so |
| 465 | // the dev re-check on the real unit carries them. |
| 466 | var edgeNoProbe = map[string]string{ |
| 467 | "ProtectClock": "the @clock calls are already refused by SystemCallFilter=@system-service, and /dev/rtc is already hidden by PrivateDevices", |
| 468 | "RestrictRealtime": "RLIMIT_RTPRIO is 0 (hard) for this user, so SCHED_FIFO answers EPERM with or without the line", |
| 469 | "SystemCallArchitectures": "a non-native call needs 32-bit code, which a Go test cannot issue", |
| 470 | } |
| 471 | |
| 472 | // edgeMayGoUnobserved are probed lines a user unit is known not to |
| 473 | // enforce on DOCS-T1. Observed, they are enforced; unobserved, untested. |
| 474 | // Every other probed line must be observed in the hardened run. |
| 475 | var edgeMayGoUnobserved = map[string]string{ |
| 476 | "ProtectProc": "the user manager cannot mount a hidepid procfs (the unit's /proc is the host's)", |
| 477 | } |
| 478 | |
| 479 | func procStatusField(t *testing.T, name string) string { |
| 480 | t.Helper() |
| 481 | for _, line := range strings.Split(readFixture(t, "/proc/self/status"), "\n") { |
| 482 | if v, ok := strings.CutPrefix(line, name+":"); ok { |
| 483 | return strings.TrimSpace(v) |
| 484 | } |
| 485 | } |
| 486 | t.Fatalf("/proc/self/status has no %s line", name) |
| 487 | return "" |
| 488 | } |
| 489 | |
| 490 | type mountEntry struct{ point, opts, fstype string } |
| 491 | |
| 492 | func (m mountEntry) readOnly() bool { |
| 493 | for _, o := range strings.Split(m.opts, ",") { |
| 494 | if o == "ro" { |
| 495 | return true |
| 496 | } |
| 497 | } |
| 498 | return false |
| 499 | } |
| 500 | |
| 501 | // lastMount returns the topmost mount at point, from /proc/self/mountinfo. |
| 502 | func lastMount(t *testing.T, point string) (mountEntry, bool) { |
| 503 | t.Helper() |
| 504 | var found mountEntry |
| 505 | ok := false |
| 506 | for _, line := range strings.Split(readFixture(t, "/proc/self/mountinfo"), "\n") { |
| 507 | pre, post, sep := strings.Cut(line, " - ") |
| 508 | f := strings.Fields(pre) |
| 509 | if !sep || len(f) < 6 || f[4] != point { |
| 510 | continue |
| 511 | } |
| 512 | ff := strings.Fields(post) |
| 513 | if len(ff) == 0 { |
| 514 | continue |
| 515 | } |
| 516 | found, ok = mountEntry{point: f[4], opts: f[5], fstype: ff[0]}, true |
| 517 | } |
| 518 | return found, ok |
| 519 | } |
| 520 | |
| 521 | func errnoName(err error) string { |
| 522 | var e syscall.Errno |
| 523 | if errors.As(err, &e) { |
| 524 | if n := unix.ErrnoName(e); n != "" { |
| 525 | return n |
| 526 | } |
| 527 | } |
| 528 | if err == nil { |
| 529 | return "ok" |
| 530 | } |
| 531 | return err.Error() |
| 532 | } |
| 533 | |
| 534 | // rawClone3 calls clone3(NULL, 0): an allowed clone3 answers EINVAL (the |
| 535 | // size is below CLONE_ARGS_SIZE_VER0) and forks nothing; a filtered one |
| 536 | // answers what the filter says. |
| 537 | func rawClone3() syscall.Errno { |
| 538 | _, _, e := unix.RawSyscall(unix.SYS_CLONE3, 0, 0, 0) |
| 539 | return e |
| 540 | } |
| 541 | |
| 542 | func ptracePeekSelf() syscall.Errno { |
| 543 | _, _, e := unix.RawSyscall6(unix.SYS_PTRACE, unix.PTRACE_PEEKDATA, uintptr(os.Getpid()), 0, 0, 0, 0) |
| 544 | return e |
| 545 | } |
| 546 | |
| 547 | // plainCgroupStart is Go's own clone3 placement, the one the shim replaced: |
| 548 | // exec.Cmd with UseCgroupFD into dir. |
| 549 | func plainCgroupStart(t *testing.T, dir string) (*exec.Cmd, error) { |
| 550 | t.Helper() |
| 551 | f, err := os.Open(dir) |
| 552 | if err != nil { |
| 553 | t.Fatal(err) |
| 554 | } |
| 555 | defer f.Close() |
| 556 | cmd := exec.Command("/bin/sleep", "60") |
| 557 | cmd.Env = []string{} |
| 558 | cmd.SysProcAttr = &syscall.SysProcAttr{UseCgroupFD: true, CgroupFD: int(f.Fd())} |
| 559 | return cmd, cmd.Start() |
| 560 | } |
| 561 | |
| 562 | // TestEdgeHardeningInner runs only in the driver's units. With the |
| 563 | // variable set nothing in it skips: a precondition it cannot establish |
| 564 | // fails it. |
| 565 | func TestEdgeHardeningInner(t *testing.T) { |
| 566 | mode := os.Getenv(edgeInnerEnv) |
| 567 | switch mode { |
| 568 | case "": |
| 569 | t.Skip("runs inside TestEdgeHardeningPlacement's units only") |
| 570 | case "control", "hardened": |
| 571 | default: |
| 572 | t.Fatalf("%s=%q, want control or hardened", edgeInnerEnv, mode) |
| 573 | } |
| 574 | pe := edgeProbeEnv{ |
| 575 | hostUTS: os.Getenv(edgeHostUTSEnv), |
| 576 | home: os.Getenv(edgeHomeEnv), |
| 577 | marker: os.Getenv(edgeMarkerEnv), |
| 578 | tasksMax: os.Getenv(edgeTasksMaxEnv), |
| 579 | } |
| 580 | if pe.hostUTS == "" || pe.home == "" || pe.marker == "" || pe.tasksMax == "" { |
| 581 | t.Fatalf("the driver's probe inputs are missing: %+v", pe) |
| 582 | } |
| 583 | rel, ok := ParseSelfCgroup(readFixture(t, "/proc/self/cgroup")) |
| 584 | if !ok { |
| 585 | t.Fatal("no unified cgroup path") |
| 586 | } |
| 587 | if filepath.Base(rel) == "supervise" { |
| 588 | rel = filepath.Dir(rel) |
| 589 | } |
| 590 | pe.unitCgroup = rel |
| 591 | |
| 592 | // Step 0: what the unit enforces, observed from inside. |
| 593 | seccomp := procStatusField(t, "Seccomp") |
| 594 | t.Logf("[%s] Seccomp: %s, NoNewPrivs: %s", mode, seccomp, procStatusField(t, "NoNewPrivs")) |
| 595 | if mode == "hardened" && seccomp != "2" { |
| 596 | t.Fatalf("Seccomp: %s, want 2 (filter mode)", seccomp) |
| 597 | } |
| 598 | if mode == "control" && seccomp != "0" { |
| 599 | t.Fatalf("Seccomp: %s in the control, want 0", seccomp) |
| 600 | } |
| 601 | t.Logf("[%s] %-24s %-9s %s", mode, "property", "status", "observation") |
| 602 | var untested []string |
| 603 | for _, k := range edgeApplied { |
| 604 | probe, probed := edgeProbes[k] |
| 605 | why, unprobed := edgeNoProbe[k] |
| 606 | if probed == unprobed { |
| 607 | t.Fatalf("%s needs exactly one of a probe or a no-probe reason", k) |
| 608 | } |
| 609 | if unprobed { |
| 610 | t.Logf("[%s] %-24s %-9s %s", mode, k, "untested", "no probe: "+why) |
| 611 | untested = append(untested, k) |
| 612 | continue |
| 613 | } |
| 614 | seen, what := probe(t, pe) |
| 615 | status := "enforced" |
| 616 | switch { |
| 617 | case mode == "control" && seen: |
| 618 | t.Fatalf("control: the %s probe reads enforced without the hardening (%s), so it tells nothing", k, what) |
| 619 | case mode == "control": |
| 620 | status = "absent" |
| 621 | case !seen && edgeMayGoUnobserved[k] != "": |
| 622 | status = "untested" |
| 623 | what += "; " + edgeMayGoUnobserved[k] |
| 624 | untested = append(untested, k) |
| 625 | case !seen: |
| 626 | t.Fatalf("hardened: %s is applied but not observed: %s", k, what) |
| 627 | } |
| 628 | t.Logf("[%s] %-24s %-9s %s", mode, k, status, what) |
| 629 | } |
| 630 | if mode == "hardened" { |
| 631 | t.Logf("[hardened] untested here, for the dev re-check on the real unit: %s", strings.Join(untested, ", ")) |
| 632 | } |
| 633 | |
| 634 | // The delegation: this test is the gate, so it fails rather than skips. |
| 635 | t.Setenv("CARLOS_SHIM_KERNEL", "1") |
| 636 | m := shimKernelManager(t) |
| 637 | scratch := filepath.Join(m.Root(), "edge-hardening-scratch") |
| 638 | if err := os.Mkdir(scratch, 0o755); err != nil { |
| 639 | t.Fatal(err) |
| 640 | } |
| 641 | t.Cleanup(func() { |
| 642 | if err := os.Remove(scratch); err != nil { |
| 643 | t.Errorf("remove %s: %v", scratch, err) |
| 644 | } |
| 645 | }) |
| 646 | |
| 647 | // Step 1 (and the control's whole point): clone3 and Go's UseCgroupFD |
| 648 | // start work without the hardening and answer ENOSYS under it. |
| 649 | e := rawClone3() |
| 650 | cmd, err := plainCgroupStart(t, scratch) |
| 651 | if mode == "control" { |
| 652 | if e != unix.EINVAL { |
| 653 | t.Fatalf("control: raw clone3 answered %s, want EINVAL (clone3 allowed)", errnoName(e)) |
| 654 | } |
| 655 | if err != nil { |
| 656 | t.Fatalf("control: UseCgroupFD start: %v", err) |
| 657 | } |
| 658 | k := &kernelKid{cmd: cmd, done: make(chan struct{})} |
| 659 | got, cgErr := cgroupOf(cmd.Process.Pid) |
| 660 | _ = cmd.Process.Kill() |
| 661 | <-k.reaped() |
| 662 | if want := strings.TrimPrefix(scratch, "/sys/fs/cgroup/"); cgErr != nil || got != want { |
| 663 | t.Fatalf("control: the UseCgroupFD child ran in %q (%v), want %s", got, cgErr, want) |
| 664 | } |
| 665 | t.Logf("[control] raw clone3: EINVAL; UseCgroupFD start landed in %s", got) |
| 666 | return |
| 667 | } |
| 668 | if e != unix.ENOSYS { |
| 669 | t.Fatalf("hardened: raw clone3 answered %s, want ENOSYS", errnoName(e)) |
| 670 | } |
| 671 | if err == nil { |
| 672 | _ = cmd.Process.Kill() |
| 673 | _ = cmd.Wait() |
| 674 | t.Fatal("hardened: a UseCgroupFD start succeeded under the filter") |
| 675 | } |
| 676 | if !errors.Is(err, syscall.ENOSYS) { |
| 677 | t.Fatalf("hardened: UseCgroupFD start: %v, want ENOSYS", err) |
| 678 | } |
| 679 | t.Logf("[hardened] raw clone3: ENOSYS; UseCgroupFD start: %v", err) |
| 680 | |
| 681 | // Step 2: placement through the shim succeeds. |
| 682 | const host = "hardening.example" |
| 683 | lim := Limit{MemoryMax: 64 << 20} |
| 684 | parent := m.hostDir(host) |
| 685 | base := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host)) |
| 686 | // Registered first, so it runs last: after every child is reaped. |
| 687 | t.Cleanup(func() { |
| 688 | m.RemoveChild(host, GenChild(1)) |
| 689 | m.RemoveChild(host, GenChild(2)) |
| 690 | m.CleanupHost(host) |
| 691 | }) |
| 692 | var tenantIn io.WriteCloser |
| 693 | var tenantOut io.ReadCloser |
| 694 | var pipeErr error |
| 695 | tenantCmd, err := m.StartChildGated(host, GenChild(1), lim, lim, func() *exec.Cmd { |
| 696 | c := exec.Command("/bin/sh", "-c", `echo "$$"; cat "/proc/$$/cgroup"; read -r _`) |
| 697 | c.Env = []string{"PATH=/usr/bin:/bin"} |
| 698 | c.Stderr = os.Stderr |
| 699 | if tenantIn, pipeErr = c.StdinPipe(); pipeErr == nil { |
| 700 | tenantOut, pipeErr = c.StdoutPipe() |
| 701 | } |
| 702 | return c |
| 703 | }, nil) |
| 704 | tenant := reapOnCleanup(t, m, host, GenChild(1), tenantCmd, tenantIn) |
| 705 | if err != nil { |
| 706 | t.Fatalf("hardened: StartChildGated g1: %v", err) |
| 707 | } |
| 708 | if pipeErr != nil { |
| 709 | t.Fatal(pipeErr) |
| 710 | } |
| 711 | type said struct { |
| 712 | pid, cg string |
| 713 | err error |
| 714 | } |
| 715 | saidc := make(chan said, 1) |
| 716 | go func() { |
| 717 | r := bufio.NewReader(tenantOut) |
| 718 | pid, err := r.ReadString('\n') |
| 719 | if err != nil { |
| 720 | saidc <- said{err: err} |
| 721 | return |
| 722 | } |
| 723 | cg, err := r.ReadString('\n') |
| 724 | saidc <- said{pid: strings.TrimSpace(pid), cg: cg, err: err} |
| 725 | }() |
| 726 | var s said |
| 727 | select { |
| 728 | case s = <-saidc: |
| 729 | case <-time.After(10 * time.Second): |
| 730 | t.Fatal("hardened: the tenant printed nothing within 10s") |
| 731 | } |
| 732 | if s.err != nil { |
| 733 | t.Fatalf("hardened: reading the tenant: %v", s.err) |
| 734 | } |
| 735 | wantG1 := filepath.Join(base, GenChild(1)) |
| 736 | gotCg, ok := ParseSelfCgroup(s.cg) |
| 737 | if !ok || gotCg != wantG1 { |
| 738 | t.Fatalf("hardened: the tenant says it runs in %q, want %s", s.cg, wantG1) |
| 739 | } |
| 740 | if s.pid != strconv.Itoa(tenantCmd.Process.Pid) { |
| 741 | t.Fatalf("hardened: the tenant's pid is %s, cmd.Process.Pid is %d", s.pid, tenantCmd.Process.Pid) |
| 742 | } |
| 743 | if comm, err := os.ReadFile(fmt.Sprintf("/proc/%d/comm", tenantCmd.Process.Pid)); err != nil || strings.TrimSpace(string(comm)) != "sh" { |
| 744 | t.Fatalf("hardened: pid %d is %q (%v), want the exec'd tenant, sh", tenantCmd.Process.Pid, comm, err) |
| 745 | } |
| 746 | t.Logf("[hardened] g1 tenant pid %s = cmd.Process.Pid, in /%s", s.pid, gotCg) |
| 747 | |
| 748 | // Step 3: the limit applies. The parent is raised as a swap raises it, |
| 749 | // so g2's own memory.max is the one that bites. |
| 750 | if err := m.SetParentLimit(host, OverlapLimit(lim, lim)); err != nil { |
| 751 | t.Fatal(err) |
| 752 | } |
| 753 | var hogIn io.WriteCloser |
| 754 | var hogPipeErr error |
| 755 | hogCmd, err := m.StartChildGated(host, GenChild(2), lim, lim, func() *exec.Cmd { |
| 756 | c := genKernelCmd("hog") |
| 757 | hogIn, hogPipeErr = c.StdinPipe() |
| 758 | return c |
| 759 | }, nil) |
| 760 | hog := reapOnCleanup(t, m, host, GenChild(2), hogCmd, hogIn) |
| 761 | if err != nil { |
| 762 | t.Fatalf("hardened: StartChildGated g2: %v", err) |
| 763 | } |
| 764 | if hogPipeErr != nil { |
| 765 | t.Fatal(hogPipeErr) |
| 766 | } |
| 767 | g2 := filepath.Join(parent, GenChild(2)) |
| 768 | expectFile(t, filepath.Join(g2, "memory.max"), "67108864") |
| 769 | if got, err := cgroupOf(hogCmd.Process.Pid); err != nil || got != filepath.Join(base, GenChild(2)) { |
| 770 | t.Fatalf("hardened: the hog runs in %s (%v), want %s", got, err, filepath.Join(base, GenChild(2))) |
| 771 | } |
| 772 | if _, err := fmt.Fprintln(hogIn, "go"); err != nil { |
| 773 | t.Fatalf("hog go-ahead: %v", err) |
| 774 | } |
| 775 | select { |
| 776 | case <-hog.reaped(): |
| 777 | case <-time.After(60 * time.Second): |
| 778 | t.Fatal("hardened: the hog was not OOM-killed within 60s") |
| 779 | } |
| 780 | var exitErr *exec.ExitError |
| 781 | if !errors.As(hog.err, &exitErr) { |
| 782 | t.Fatalf("hardened: hog exit: %v", hog.err) |
| 783 | } |
| 784 | if ws, ok := exitErr.Sys().(syscall.WaitStatus); !ok || !ws.Signaled() || ws.Signal() != syscall.SIGKILL { |
| 785 | t.Fatalf("hardened: the hog ended with %v, want SIGKILL from the OOM killer", hog.err) |
| 786 | } |
| 787 | kills := oomKills(t, g2) |
| 788 | if kills < 1 { |
| 789 | t.Fatalf("hardened: g2 memory.events oom_kill = %d, want at least 1", kills) |
| 790 | } |
| 791 | if n := oomKills(t, filepath.Join(parent, GenChild(1))); n != 0 { |
| 792 | t.Fatalf("hardened: g1 saw %d OOM kills", n) |
| 793 | } |
| 794 | select { |
| 795 | case <-tenant.reaped(): |
| 796 | t.Fatalf("hardened: g1's tenant died beside the hog: %v", tenant.err) |
| 797 | default: |
| 798 | } |
| 799 | if err := syscall.Kill(tenantCmd.Process.Pid, 0); err != nil { |
| 800 | t.Fatalf("hardened: g1's tenant is gone: %v", err) |
| 801 | } |
| 802 | t.Logf("[hardened] g2 hog: %v, g2 oom_kill %d; g1 tenant alive", hog.err, kills) |
| 803 | |
| 804 | // Step 4: arming would arm. |
| 805 | ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) |
| 806 | defer cancel() |
| 807 | if err := m.VerifyPlacement(ctx); err != nil { |
| 808 | t.Fatalf("hardened: VerifyPlacement: %v", err) |
| 809 | } |
| 810 | t.Logf("[hardened] VerifyPlacement: nil") |
| 811 | } |
| 812 | |
amadan