| 1 | //go:build linux |
| 2 | |
| 3 | package instanceexec |
| 4 | |
| 5 | import ( |
| 6 | "errors" |
| 7 | "net" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "strings" |
| 11 | "syscall" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/instancectl" |
| 16 | ) |
| 17 | |
| 18 | // goodEnv is a fixture that passes every check: the trees an isolated |
| 19 | // instance sees, pid 1, a read-only cgroupfs, an unopenable /etc/carlos, a |
| 20 | // nosymfollow mount, EPERM on both metadata addresses, a live loopback and |
| 21 | // a TCP connect that never succeeds. |
| 22 | func goodEnv(t *testing.T) probeEnv { |
| 23 | t.Helper() |
| 24 | root := shortProbeSocketDir(t) |
| 25 | mk := func(p string) string { |
| 26 | if err := os.MkdirAll(filepath.Join(root, p), 0o700); err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | return filepath.Join(root, p) |
| 30 | } |
| 31 | data := mk("data") |
| 32 | mk("data/instances/_probe") |
| 33 | run := mk("run/carlos") |
| 34 | mk("run/carlos/instances/_probe") |
| 35 | varCache := mk("var/cache") |
| 36 | varLib := mk("var/lib") |
| 37 | etc := filepath.Join(root, "etc", "carlos") // deliberately absent: opening it fails |
| 38 | shm := mk("dev/shm") |
| 39 | proc := mk("proc") |
| 40 | mk("proc/1") |
| 41 | mk("proc/self") |
| 42 | opt := mk("opt/carlos") |
| 43 | mk("opt/carlos/live") |
| 44 | for _, f := range []string{"opt/carlos/app", "opt/carlos/carlos"} { |
| 45 | if err := os.WriteFile(filepath.Join(root, f), nil, 0o755); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | } |
| 49 | if err := os.Symlink("../app", filepath.Join(opt, "live", "_probe")); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | resolv := filepath.Join(root, "resolv.conf") |
| 53 | if err := os.WriteFile(resolv, []byte("# fixture\nnameserver 10.0.0.2\nsearch eu-west-1.compute.internal\n"), 0o644); err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | mqueue := mk("dev/mqueue") |
| 57 | cg := mk("cgroup/self") |
| 58 | for name, v := range map[string]string{"cpu.max": "50000 100000\n", "cpu.weight": "50\n", "io.weight": "default 50\n"} { |
| 59 | if err := os.WriteFile(filepath.Join(cg, name), []byte(v), 0o644); err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | } |
| 63 | return probeEnv{ |
| 64 | dataRoot: data, runRoot: run, varCache: varCache, varLib: varLib, etcCarlos: etc, devShm: shm, procRoot: proc, |
| 65 | controlDir: filepath.Join(root, "run", "carlos-control"), // deliberately absent: ReadDir and dial fail |
| 66 | controlSock: filepath.Join(root, "run", "carlos-control", "control.sock"), |
| 67 | cgroupProcs: filepath.Join(root, "sys", "cgroup.procs"), |
| 68 | pid: func() int { return 1 }, |
| 69 | writable: func(string) error { return syscall.EROFS }, |
| 70 | // The fence: everything is refused but the box's resolver. |
| 71 | sendUDP: func(network, addr string) error { |
| 72 | if addr == "10.0.0.2:53" { |
| 73 | return nil |
| 74 | } |
| 75 | return syscall.EPERM |
| 76 | }, |
| 77 | connectTCP: func(string, time.Duration) error { return syscall.ETIMEDOUT }, |
| 78 | mountNoSym: func(string) (bool, error) { return true, nil }, |
| 79 | mountOption: func(string, string) (bool, error) { return true, nil }, |
| 80 | symlinkCheck: func(string) error { return nil }, |
| 81 | |
| 82 | optRoot: opt, resolvConf: resolv, devMqueue: mqueue, |
| 83 | ipcInode: func() (uint64, error) { return 4026532412, nil }, |
| 84 | abstractScoped: func() error { return nil }, |
| 85 | selfCgroup: func() (string, error) { return cg, nil }, |
| 86 | ioEnforced: func(string) bool { return true }, |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | func verdict(r instancectl.ProbeReport, check string) string { |
| 91 | for _, l := range r.Lines { |
| 92 | if l.Check == check { |
| 93 | return l.Verdict |
| 94 | } |
| 95 | } |
| 96 | return "<missing>" |
| 97 | } |
| 98 | |
| 99 | func TestProbePassesOnAnIsolatedNamespace(t *testing.T) { |
| 100 | r := runChecks(goodEnv(t)) |
| 101 | if !r.Ready { |
| 102 | t.Fatalf("not ready: %+v", r.Lines) |
| 103 | } |
| 104 | for _, c := range []string{"data-tree", "run-tree", "var-empty", "pid1", "cgroup-ro", "cgroup-ro-mount", "control-unreachable", "etc-carlos", "shm-private", "symlink-eloop", "nosymfollow", "imds-v4", "imds-v6", "imds-tcp", |
| 105 | "opt-own-build", "dns", "resolver-ports", "loopback-v4", "loopback-v6", "private-v4", "private-v6", "ipc-private", "abstract-unix", "cpu-limits", "io-weight"} { |
| 106 | if v := verdict(r, c); v != "ok" { |
| 107 | t.Errorf("%s = %s", c, v) |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | func TestProbeFailsWhenASiblingIsVisible(t *testing.T) { |
| 113 | e := goodEnv(t) |
| 114 | if err := os.MkdirAll(filepath.Join(e.dataRoot, "instances", "other.example.com"), 0o700); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | r := runChecks(e) |
| 118 | if r.Ready || verdict(r, "data-tree") != "fail" { |
| 119 | t.Errorf("sibling visible yet ready=%v data-tree=%s", r.Ready, verdict(r, "data-tree")) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | func TestProbeFailsWhenCgroupfsIsWritable(t *testing.T) { |
| 124 | e := goodEnv(t) |
| 125 | e.writable = func(string) error { return nil } |
| 126 | r := runChecks(e) |
| 127 | if r.Ready || verdict(r, "cgroup-ro") != "fail" { |
| 128 | t.Errorf("writable cgroupfs yet ready=%v", r.Ready) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | // TestCgroupROVerdictDistinguishesTheThreeAnswers pins that DAC (checked |
| 133 | // first by the kernel) and the mount flag are told apart, not merged into |
| 134 | // one pass/fail: EACCES means the file is not delegated to carlos, ENOENT |
| 135 | // means the edge unit is not loaded, and neither is the mount flag's own |
| 136 | // answer (final review, plan 2, item 1). |
| 137 | func TestCgroupROVerdictDistinguishesTheThreeAnswers(t *testing.T) { |
| 138 | cases := []struct { |
| 139 | err error |
| 140 | detail string |
| 141 | ok bool |
| 142 | }{ |
| 143 | {syscall.EROFS, "", true}, |
| 144 | {syscall.EACCES, "not delegated", false}, |
| 145 | {syscall.ENOENT, "edge not loaded", false}, |
| 146 | {nil, "writable", false}, |
| 147 | } |
| 148 | for _, c := range cases { |
| 149 | detail, ok := cgroupROVerdict(c.err) |
| 150 | if detail != c.detail || ok != c.ok { |
| 151 | t.Errorf("cgroupROVerdict(%v) = %q, %v, want %q, %v", c.err, detail, ok, c.detail, c.ok) |
| 152 | } |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | func TestProbeFailsWhenTheCgroupMountLacksRO(t *testing.T) { |
| 157 | e := goodEnv(t) |
| 158 | e.mountOption = func(string, string) (bool, error) { return false, nil } |
| 159 | r := runChecks(e) |
| 160 | if r.Ready || verdict(r, "cgroup-ro-mount") != "fail" { |
| 161 | t.Errorf("cgroup mount without ro yet ready=%v", r.Ready) |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | func TestProbeFailsWhenTheControlSocketDirectoryIsVisible(t *testing.T) { |
| 166 | e := goodEnv(t) |
| 167 | if err := os.MkdirAll(e.controlDir, 0o750); err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | r := runChecks(e) |
| 171 | if r.Ready || verdict(r, "control-unreachable") != "fail" { |
| 172 | t.Errorf("visible control socket directory yet ready=%v", r.Ready) |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | // TestProbePassesOnAnInaccessibleControlNode pins the shape |
| 177 | // InaccessiblePaths= really leaves: a mode 000 directory that still |
| 178 | // answers Lstat. The first probe on a box failed here because it checked |
| 179 | // Lstat; what a tenant cannot do is list it or dial through it (#126). |
| 180 | func TestProbePassesOnAnInaccessibleControlNode(t *testing.T) { |
| 181 | if os.Geteuid() == 0 { |
| 182 | t.Skip("root reads a mode 000 directory") |
| 183 | } |
| 184 | e := goodEnv(t) |
| 185 | if err := os.MkdirAll(e.controlDir, 0o700); err != nil { |
| 186 | t.Fatal(err) |
| 187 | } |
| 188 | if err := os.Chmod(e.controlDir, 0); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | t.Cleanup(func() { os.Chmod(e.controlDir, 0o700) }) |
| 192 | if _, err := os.Lstat(e.controlDir); err != nil { |
| 193 | t.Fatalf("fixture: Lstat = %v, want the node to exist", err) |
| 194 | } |
| 195 | r := runChecks(e) |
| 196 | if v := verdict(r, "control-unreachable"); v != "ok" { |
| 197 | t.Errorf("control-unreachable = %s on a mode 000 node: %+v", v, r.Lines) |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // TestProbeFailsWhenTheControlSocketDials: a directory that refuses a |
| 202 | // listing but still lets a connect through is not unreachable. |
| 203 | func TestProbeFailsWhenTheControlSocketDials(t *testing.T) { |
| 204 | if os.Geteuid() == 0 { |
| 205 | t.Skip("root reads a mode 0300 directory") |
| 206 | } |
| 207 | e := goodEnv(t) |
| 208 | if err := os.MkdirAll(e.controlDir, 0o700); err != nil { |
| 209 | t.Fatal(err) |
| 210 | } |
| 211 | l, err := net.Listen("unix", e.controlSock) |
| 212 | if err != nil { |
| 213 | t.Fatalf("listen at %s (a socket test wants a short TMPDIR): %v", e.controlSock, err) |
| 214 | } |
| 215 | defer l.Close() |
| 216 | go func() { |
| 217 | for { |
| 218 | c, err := l.Accept() |
| 219 | if err != nil { |
| 220 | return |
| 221 | } |
| 222 | c.Close() |
| 223 | } |
| 224 | }() |
| 225 | if err := os.Chmod(e.controlDir, 0o300); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | t.Cleanup(func() { os.Chmod(e.controlDir, 0o700) }) |
| 229 | r := runChecks(e) |
| 230 | if r.Ready || verdict(r, "control-unreachable") != "fail" { |
| 231 | t.Errorf("dialable control socket yet ready=%v control-unreachable=%s", r.Ready, verdict(r, "control-unreachable")) |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // TestProbeFailsOnAStaleControlSocket: a socket file with nothing |
| 236 | // listening refuses the connection, which proves the path was reached. |
| 237 | func TestProbeFailsOnAStaleControlSocket(t *testing.T) { |
| 238 | if os.Geteuid() == 0 { |
| 239 | t.Skip("root reads a mode 0300 directory") |
| 240 | } |
| 241 | e := goodEnv(t) |
| 242 | if err := os.MkdirAll(e.controlDir, 0o700); err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | l, err := net.Listen("unix", e.controlSock) |
| 246 | if err != nil { |
| 247 | t.Fatalf("listen at %s (a socket test wants a short TMPDIR): %v", e.controlSock, err) |
| 248 | } |
| 249 | l.(*net.UnixListener).SetUnlinkOnClose(false) |
| 250 | l.Close() |
| 251 | if err := os.Chmod(e.controlDir, 0o300); err != nil { |
| 252 | t.Fatal(err) |
| 253 | } |
| 254 | t.Cleanup(func() { os.Chmod(e.controlDir, 0o700) }) |
| 255 | r := runChecks(e) |
| 256 | if r.Ready || verdict(r, "control-unreachable") != "fail" { |
| 257 | t.Errorf("stale control socket yet ready=%v control-unreachable=%s", r.Ready, verdict(r, "control-unreachable")) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // TestProbeFailsWhenTheHostShmIsVisible: PrivateDevices= alone binds the |
| 262 | // host's /dev/shm, where tmpfiles plants the fixture (#126, R15). |
| 263 | func TestProbeFailsWhenTheHostShmIsVisible(t *testing.T) { |
| 264 | e := goodEnv(t) |
| 265 | if err := os.WriteFile(filepath.Join(e.devShm, "carlos-probe"), []byte("probe"), 0o644); err != nil { |
| 266 | t.Fatal(err) |
| 267 | } |
| 268 | r := runChecks(e) |
| 269 | if r.Ready || verdict(r, "shm-private") != "fail" { |
| 270 | t.Errorf("host /dev/shm visible yet ready=%v shm-private=%s", r.Ready, verdict(r, "shm-private")) |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | func TestProbeFailsWhenEtcCarlosIsReadable(t *testing.T) { |
| 275 | e := goodEnv(t) |
| 276 | if err := os.MkdirAll(e.etcCarlos, 0o755); err != nil { |
| 277 | t.Fatal(err) |
| 278 | } |
| 279 | if err := os.WriteFile(filepath.Join(e.etcCarlos, "probe-secret"), []byte("probe"), 0o644); err != nil { |
| 280 | t.Fatal(err) |
| 281 | } |
| 282 | r := runChecks(e) |
| 283 | if r.Ready || verdict(r, "etc-carlos") != "fail" { |
| 284 | t.Errorf("readable /etc/carlos yet ready=%v", r.Ready) |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | func TestProbeIMDSVerdicts(t *testing.T) { |
| 289 | // v4 must be EPERM; anything else is a failure, including a timeout. |
| 290 | e := goodEnv(t) |
| 291 | e.sendUDP = func(_, addr string) error { |
| 292 | if addr == "169.254.169.254:80" { |
| 293 | return syscall.ETIMEDOUT |
| 294 | } |
| 295 | return syscall.EPERM |
| 296 | } |
| 297 | if r := runChecks(e); r.Ready || verdict(r, "imds-v4") != "fail" { |
| 298 | t.Errorf("v4 timeout: ready=%v v4=%s", r.Ready, verdict(r, "imds-v4")) |
| 299 | } |
| 300 | // v6 ENETUNREACH is unverified and does not block readiness. |
| 301 | e = goodEnv(t) |
| 302 | e.sendUDP = func(_, addr string) error { |
| 303 | if addr == "[fd00:ec2::254]:80" { |
| 304 | return syscall.ENETUNREACH |
| 305 | } |
| 306 | if addr == "10.0.0.2:53" { |
| 307 | return nil |
| 308 | } |
| 309 | return syscall.EPERM |
| 310 | } |
| 311 | if r := runChecks(e); !r.Ready || verdict(r, "imds-v6") != "unverified" { |
| 312 | t.Errorf("v6 no route: ready=%v v6=%s", r.Ready, verdict(r, "imds-v6")) |
| 313 | } |
| 314 | // A successful send to either address is a failure. |
| 315 | e = goodEnv(t) |
| 316 | e.sendUDP = func(string, string) error { return nil } |
| 317 | if r := runChecks(e); r.Ready || verdict(r, "imds-v4") != "fail" { |
| 318 | t.Errorf("v4 success: ready=%v", r.Ready) |
| 319 | } |
| 320 | // A TCP connect that succeeds is a failure. |
| 321 | e = goodEnv(t) |
| 322 | e.connectTCP = func(string, time.Duration) error { return nil } |
| 323 | if r := runChecks(e); r.Ready || verdict(r, "imds-tcp") != "fail" { |
| 324 | t.Errorf("tcp success: ready=%v", r.Ready) |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | func TestProbeSymlinkCheckUsesARealSymlink(t *testing.T) { |
| 329 | // The fixture's data tree is a plain temp dir with no nosymfollow, so |
| 330 | // the real symlink check FAILS here: that proves the check is real. |
| 331 | e := goodEnv(t) |
| 332 | if !errors.Is(realSymlinkCheck(filepath.Join(e.dataRoot, "instances", "_probe")), errNotELOOP) { |
| 333 | t.Error("a followable symlink was not reported") |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // TestRealSymlinkCheckReportsAPlantedName pins the review fix: a name the |
| 338 | // check would have used is refused (EEXIST), reported as a failure naming |
| 339 | // the planted entry, and left in place, rather than removed. A same-uid |
| 340 | // tenant sharing this directory could otherwise have its own file deleted |
| 341 | // by a probe run (final review, plan 2, item 5). |
| 342 | func TestRealSymlinkCheckReportsAPlantedName(t *testing.T) { |
| 343 | dir := t.TempDir() |
| 344 | orig := randomName |
| 345 | t.Cleanup(func() { randomName = orig }) |
| 346 | randomName = func() string { return "planted" } |
| 347 | if err := os.MkdirAll(filepath.Join(dir, "planted"), 0o700); err != nil { |
| 348 | t.Fatal(err) |
| 349 | } |
| 350 | err := realSymlinkCheck(dir) |
| 351 | if err == nil { |
| 352 | t.Fatal("a colliding name was not reported") |
| 353 | } |
| 354 | if !strings.Contains(err.Error(), filepath.Join(dir, "planted")) { |
| 355 | t.Errorf("error does not name the planted entry: %v", err) |
| 356 | } |
| 357 | if fi, statErr := os.Stat(filepath.Join(dir, "planted")); statErr != nil || !fi.IsDir() { |
| 358 | t.Errorf("the planted entry was removed: %v, %v", fi, statErr) |
| 359 | } |
| 360 | } |
| 361 | |
| 362 | func shortProbeSocketDir(t *testing.T) string { |
| 363 | t.Helper() |
| 364 | dir, err := os.MkdirTemp(os.TempDir(), "probe-") |
| 365 | if err != nil { |
| 366 | t.Fatal(err) |
| 367 | } |
| 368 | t.Cleanup(func() { |
| 369 | if err := os.RemoveAll(dir); err != nil { |
| 370 | t.Error(err) |
| 371 | } |
| 372 | }) |
| 373 | return dir |
| 374 | } |
| 375 | |
amadan