| 1 | package instanceexec |
| 2 | |
| 3 | import ( |
| 4 | "crypto/rand" |
| 5 | "encoding/hex" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "flag" |
| 9 | "fmt" |
| 10 | "net" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "sort" |
| 14 | "strconv" |
| 15 | "syscall" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/cliflag" |
| 20 | "github.com/carlosframework/platform/internal/controlsock" |
| 21 | "github.com/carlosframework/platform/internal/instancectl" |
| 22 | "github.com/carlosframework/platform/internal/instancefs" |
| 23 | ) |
| 24 | |
| 25 | // probeEnv is everything the checks touch, so the verdict logic is testable |
| 26 | // against a fixture tree and fake syscalls while production runs against |
| 27 | // the real namespace. |
| 28 | type probeEnv struct { |
| 29 | dataRoot string // /data as the probe sees it |
| 30 | runRoot string // /run/carlos as the probe sees it |
| 31 | varCache string // /var/cache |
| 32 | varLib string // /var/lib |
| 33 | etcCarlos string // /etc/carlos |
| 34 | devShm string // /dev/shm, which must be this unit's own |
| 35 | procRoot string // /proc |
| 36 | controlDir string // /run/carlos-control, which must be unusable from in here |
| 37 | controlSock string // the control socket inside it, which must not dial |
| 38 | cgroupProcs string // the edge's delegated cgroup.procs, to test writability on |
| 39 | pid func() int |
| 40 | writable func(path string) error |
| 41 | sendUDP func(network, addr string) error |
| 42 | connectTCP func(addr string, timeout time.Duration) error |
| 43 | mountNoSym func(mountpoint string) (bool, error) |
| 44 | // mountOption asks whether a mountpoint carries a given per-mount |
| 45 | // option; wired to instancefs.MountOptionOf in production. It backs |
| 46 | // the cgroup-ro-mount check, which asserts /sys/fs/cgroup itself |
| 47 | // carries "ro" rather than trusting cgroup-ro's access(W_OK) alone. |
| 48 | mountOption func(mountpoint, opt string) (bool, error) |
| 49 | // symlinkCheck plants a symlink under dir and reports whether it was |
| 50 | // refused. Production wires this to realSymlinkCheck; the test fixture |
| 51 | // stubs it because the fixture's tree is a plain temp dir with no |
| 52 | // nosymfollow mount, so the real check would fail there for a reason |
| 53 | // unrelated to the property under test in the other cases. |
| 54 | symlinkCheck func(dir string) error |
| 55 | |
| 56 | // The hardening of #214 and #178. |
| 57 | optRoot string // /opt/carlos, which must hold only this unit's own build |
| 58 | resolvConf string // /etc/resolv.conf, whose resolvers must be reachable |
| 59 | devMqueue string // /dev/mqueue, which must not be the host's |
| 60 | // ipcInode is this process's IPC namespace inode; the box's own is |
| 61 | // initIPCInode. |
| 62 | ipcInode func() (uint64, error) |
| 63 | // abstractScoped proves the trampoline's Landlock scope on this |
| 64 | // kernel: nil when a scoped connect to an outside abstract name was |
| 65 | // refused with EPERM. |
| 66 | abstractScoped func() error |
| 67 | // selfCgroup is this unit's own cgroup directory, where the probe |
| 68 | // unit's CPUQuota=, CPUWeight= and IOWeight= must have landed. |
| 69 | selfCgroup func() (string, error) |
| 70 | // ioEnforced says whether io.weight has a controller behind it. |
| 71 | ioEnforced func(selfCgroup string) bool |
| 72 | } |
| 73 | |
| 74 | // cgroupMountpoint is the fixed system path the cgroup-ro-mount check |
| 75 | // asks instancefs.MountOptionOf about. It is not a probeEnv field: every |
| 76 | // test that cares stubs the mountOption function itself, so no fixture |
| 77 | // needs a real mount at this path. |
| 78 | const cgroupMountpoint = "/sys/fs/cgroup" |
| 79 | |
| 80 | var errNotELOOP = errors.New("symlink was followed") |
| 81 | |
| 82 | // RunProbe is `carlos instance-probe`: it runs inside the probe unit's |
| 83 | // namespace (the same text as the tenant templates) and prints a |
| 84 | // ProbeReport on stdout, which the unit's StandardOutput=truncate: lands in |
| 85 | // /run/carlos-control/probe.out for the control service to read (design |
| 86 | // §8.5). It exits 0 whatever the verdict: the verdict is the output, and a |
| 87 | // failed unit would hide it. |
| 88 | func RunProbe(args []string) { |
| 89 | fs := flag.NewFlagSet("instance-probe", flag.ExitOnError) |
| 90 | fs.Usage = cliflag.Usage(fs, "usage: carlos instance-probe") |
| 91 | _ = fs.Parse(args) |
| 92 | r := runChecks(productionProbeEnv()) |
| 93 | enc := json.NewEncoder(os.Stdout) |
| 94 | enc.SetIndent("", " ") |
| 95 | _ = enc.Encode(r) |
| 96 | } |
| 97 | |
| 98 | func productionProbeEnv() probeEnv { |
| 99 | return probeEnv{ |
| 100 | dataRoot: "/data", runRoot: "/run/carlos", varCache: "/var/cache", varLib: "/var/lib", |
| 101 | etcCarlos: "/etc/carlos", devShm: "/dev/shm", procRoot: "/proc", controlDir: controlsock.Dir, controlSock: controlsock.Path, |
| 102 | // The edge's own delegated cgroup.procs, not the probe's: Delegate= |
| 103 | // on carlos-edge.service chowns this file to carlos, so DAC passes |
| 104 | // and only the mount's ro flag can refuse the write, making EROFS |
| 105 | // evidence rather than an artifact of ownership (final review, |
| 106 | // plan 2, item 1). This depends on the edge being loaded, which |
| 107 | // every instance unit's BindsTo=carlos-edge.service already |
| 108 | // assumes. |
| 109 | cgroupProcs: "/sys/fs/cgroup/carlos.slice/carlos-edge.service/cgroup.procs", |
| 110 | pid: os.Getpid, |
| 111 | writable: accessWritable, |
| 112 | sendUDP: sendOneDatagram, |
| 113 | connectTCP: connectWithTimeout, |
| 114 | mountNoSym: instancefs.MountNoSymfollow, |
| 115 | mountOption: instancefs.MountOptionOf, |
| 116 | symlinkCheck: realSymlinkCheck, |
| 117 | |
| 118 | optRoot: "/opt/carlos", resolvConf: "/etc/resolv.conf", devMqueue: "/dev/mqueue", |
| 119 | ipcInode: ipcNamespaceInode, |
| 120 | abstractScoped: realAbstractCheck, |
| 121 | selfCgroup: ownCgroupDir, |
| 122 | ioEnforced: ioWeightEnforced, |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | // ownCgroupDir is /sys/fs/cgroup plus this process's unified-hierarchy |
| 127 | // path from /proc/self/cgroup. |
| 128 | func ownCgroupDir() (string, error) { |
| 129 | b, err := os.ReadFile("/proc/self/cgroup") |
| 130 | if err != nil { |
| 131 | return "", err |
| 132 | } |
| 133 | rel, ok := cgroup.ParseSelfCgroup(string(b)) |
| 134 | if !ok { |
| 135 | return "", errors.New("no cgroup v2 path in /proc/self/cgroup") |
| 136 | } |
| 137 | return filepath.Join(cgroupMountpoint, rel), nil |
| 138 | } |
| 139 | |
| 140 | func listNames(dir string) ([]string, error) { |
| 141 | entries, err := os.ReadDir(dir) |
| 142 | if err != nil { |
| 143 | return nil, err |
| 144 | } |
| 145 | names := make([]string, 0, len(entries)) |
| 146 | for _, e := range entries { |
| 147 | names = append(names, e.Name()) |
| 148 | } |
| 149 | sort.Strings(names) |
| 150 | return names, nil |
| 151 | } |
| 152 | |
| 153 | func exactly(dir string, want ...string) (string, bool) { |
| 154 | got, err := listNames(dir) |
| 155 | if err != nil { |
| 156 | return err.Error(), false |
| 157 | } |
| 158 | sort.Strings(want) |
| 159 | if fmt.Sprint(got) != fmt.Sprint(want) { |
| 160 | return fmt.Sprintf("%s lists %v, want %v", dir, got, want), false |
| 161 | } |
| 162 | return "", true |
| 163 | } |
| 164 | |
| 165 | // runChecks is the verdict. Each line names the property it checks; "fail" |
| 166 | // blocks readiness, "unverified" does not, "ok" is what a ready box shows |
| 167 | // on every line. |
| 168 | func runChecks(e probeEnv) instancectl.ProbeReport { |
| 169 | var r instancectl.ProbeReport |
| 170 | add := func(check, verdict, detail string) { |
| 171 | r.Lines = append(r.Lines, instancectl.VerdictLine{Check: check, Verdict: verdict, Detail: detail}) |
| 172 | } |
| 173 | pass := func(check string, detail string, ok bool) { |
| 174 | if ok { |
| 175 | add(check, "ok", "") |
| 176 | } else { |
| 177 | add(check, "fail", detail) |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | // The two trees hold exactly the probe's own directory. |
| 182 | d1, ok1 := exactly(e.dataRoot, "instances") |
| 183 | d2, ok2 := exactly(filepath.Join(e.dataRoot, "instances"), instancectl.ProbeDirName) |
| 184 | pass("data-tree", d1+d2, ok1 && ok2) |
| 185 | r1, ok1 := exactly(e.runRoot, "instances") |
| 186 | r2, ok2 := exactly(filepath.Join(e.runRoot, "instances"), instancectl.ProbeDirName) |
| 187 | pass("run-tree", r1+r2, ok1 && ok2) |
| 188 | |
| 189 | // The two trees other carlos-uid units write are empty. |
| 190 | v1, ok1 := exactly(e.varCache) |
| 191 | v2, ok2 := exactly(e.varLib) |
| 192 | pass("var-empty", v1+v2, ok1 && ok2) |
| 193 | |
| 194 | // A private pid namespace: this process is pid 1 and /proc lists no |
| 195 | // other pid. |
| 196 | pids, err := listNames(e.procRoot) |
| 197 | others := "" |
| 198 | for _, n := range pids { |
| 199 | if v, err := strconv.Atoi(n); err == nil && v != 1 { |
| 200 | others += n + " " |
| 201 | } |
| 202 | } |
| 203 | pid := e.pid() |
| 204 | pass("pid1", fmt.Sprintf("pid=%d others=[%s] err=%v", pid, others, err), err == nil && pid == 1 && others == "") |
| 205 | |
| 206 | // cgroupfs is read-only, checked two ways. access(W_OK) on the edge's |
| 207 | // delegated cgroup.procs must answer EROFS: DAC is checked before the |
| 208 | // mount's read-only flag, so EACCES here means the file is not |
| 209 | // delegated to carlos (not a pass, whatever the mount says) and ENOENT |
| 210 | // means the edge unit is not loaded at all, neither of which the mount |
| 211 | // flag could produce on its own. And /sys/fs/cgroup itself must carry |
| 212 | // the "ro" mount option, so the EROFS above is the mount flag's doing |
| 213 | // and not some other cause (final review, plan 2, item 1). |
| 214 | werr := e.writable(e.cgroupProcs) |
| 215 | detail, ok := cgroupROVerdict(werr) |
| 216 | pass("cgroup-ro", detail, ok) |
| 217 | mro, mroErr := e.mountOption(cgroupMountpoint, "ro") |
| 218 | pass("cgroup-ro-mount", fmt.Sprintf("MountOptionOf(%s, ro) = %v, %v", cgroupMountpoint, mro, mroErr), mro && mroErr == nil) |
| 219 | |
| 220 | // The control socket is unreachable from in here: InaccessiblePaths= |
| 221 | // in the shared namespace block (design §5.2) makes a connect from |
| 222 | // inside any instance impossible outright, not merely refused by the |
| 223 | // peer check on the other end. The path itself still answers Lstat, |
| 224 | // because systemd hides it under a mode 000 root-owned node rather |
| 225 | // than removing it, so the check is what a tenant could actually do: |
| 226 | // list the directory, or dial the socket (#126, first roll). |
| 227 | // Both must be refused for permission or absence: a refused |
| 228 | // connection or a timeout means the path was reachable. |
| 229 | _, cerr := os.ReadDir(e.controlDir) |
| 230 | kerr := dialUnix(e.controlSock) |
| 231 | pass("control-unreachable", fmt.Sprintf("readdir=%v dial=%v", cerr, kerr), unreachable(cerr) && unreachable(kerr)) |
| 232 | |
| 233 | // /etc/carlos cannot be opened and its fixture cannot be read. |
| 234 | _, derr := os.ReadDir(e.etcCarlos) |
| 235 | _, ferr := os.ReadFile(filepath.Join(e.etcCarlos, "probe-secret")) |
| 236 | pass("etc-carlos", fmt.Sprintf("readdir=%v readfile=%v", derr, ferr), derr != nil && ferr != nil) |
| 237 | |
| 238 | // /dev/shm is this unit's own tmpfs, not the host's, which PrivateDevices= |
| 239 | // would bind in: the host's holds the fixture /dev/shm/carlos-probe, so |
| 240 | // an empty listing here means a sibling's shared memory is out of reach |
| 241 | // (#126, R15). |
| 242 | s1, sok := exactly(e.devShm) |
| 243 | pass("shm-private", s1, sok) |
| 244 | |
| 245 | // A symlink planted in the probe's own directory is refused with ELOOP. |
| 246 | serr := e.symlinkCheck(filepath.Join(e.dataRoot, "instances", instancectl.ProbeDirName)) |
| 247 | pass("symlink-eloop", fmt.Sprint(serr), serr == nil) |
| 248 | |
| 249 | // Both bound directories are on nosymfollow mounts. |
| 250 | n1, err1 := e.mountNoSym(filepath.Join(e.dataRoot, "instances", instancectl.ProbeDirName)) |
| 251 | n2, err2 := e.mountNoSym(filepath.Join(e.runRoot, "instances", instancectl.ProbeDirName)) |
| 252 | pass("nosymfollow", fmt.Sprintf("data=%v/%v run=%v/%v", n1, err1, n2, err2), n1 && n2 && err1 == nil && err2 == nil) |
| 253 | |
| 254 | // IMDS, for a packet filter: a datagram's verdict comes straight back |
| 255 | // from sendmsg. v4 must be EPERM. v6 is EPERM (enforced) or ENETUNREACH |
| 256 | // (no route to that address: unverified, not blocking). The loopback |
| 257 | // control proves the stack is live. A TCP connect must not succeed. |
| 258 | v4 := e.sendUDP("udp4", "169.254.169.254:80") |
| 259 | pass("imds-v4", fmt.Sprintf("sendto = %v", v4), errors.Is(v4, syscall.EPERM)) |
| 260 | v6 := e.sendUDP("udp6", "[fd00:ec2::254]:80") |
| 261 | switch { |
| 262 | case errors.Is(v6, syscall.EPERM): |
| 263 | add("imds-v6", "ok", "") |
| 264 | case errors.Is(v6, syscall.ENETUNREACH): |
| 265 | add("imds-v6", "unverified", "no route to the address; deny not exercised") |
| 266 | default: |
| 267 | add("imds-v6", "fail", fmt.Sprintf("sendto = %v", v6)) |
| 268 | } |
| 269 | tcp := e.connectTCP("169.254.169.254:80", 2*time.Second) |
| 270 | pass("imds-tcp", fmt.Sprintf("connect = %v", tcp), tcp != nil) |
| 271 | |
| 272 | hardeningChecks(e, add, pass) |
| 273 | |
| 274 | r.Ready = true |
| 275 | for _, l := range r.Lines { |
| 276 | if l.Verdict == "fail" { |
| 277 | r.Ready = false |
| 278 | } |
| 279 | } |
| 280 | return r |
| 281 | } |
| 282 | |
| 283 | // cgroupROVerdict turns access(W_OK) on the edge's delegated cgroup.procs |
| 284 | // into the cgroup-ro check's detail and pass/fail: EROFS is the only |
| 285 | // answer the read-only mount flag alone can produce, since the kernel |
| 286 | // checks DAC first and Delegate= has already given carlos write access on |
| 287 | // the file itself (final review, plan 2, item 1). |
| 288 | func cgroupROVerdict(werr error) (detail string, ok bool) { |
| 289 | switch { |
| 290 | case errors.Is(werr, syscall.EROFS): |
| 291 | return "", true |
| 292 | case errors.Is(werr, syscall.EACCES): |
| 293 | return "not delegated", false |
| 294 | case errors.Is(werr, syscall.ENOENT): |
| 295 | return "edge not loaded", false |
| 296 | case werr == nil: |
| 297 | return "writable", false |
| 298 | default: |
| 299 | return werr.Error(), false |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | // randomName returns a 16-hex-digit name. A test overrides it to force a |
| 304 | // name collision; production draws 8 random bytes so two probe runs, or a |
| 305 | // probe run and a flat-tenant's own writes, never collide by accident. |
| 306 | var randomName = func() string { |
| 307 | var b [8]byte |
| 308 | if _, err := rand.Read(b[:]); err != nil { |
| 309 | // crypto/rand.Read only fails if the OS's random source is |
| 310 | // unavailable, which is a box-level emergency worth stopping the |
| 311 | // probe over rather than silently falling back to a fixed name |
| 312 | // that reintroduces the property this check exists to catch. |
| 313 | panic(fmt.Sprintf("instance-probe: crypto/rand: %v", err)) |
| 314 | } |
| 315 | return hex.EncodeToString(b[:]) |
| 316 | } |
| 317 | |
| 318 | // realSymlinkCheck plants a symlink under a random name inside dir, |
| 319 | // pointing at a real file under another random name, and tries to open |
| 320 | // the link through its name; on a nosymfollow mount the open fails with |
| 321 | // ELOOP. Random names mean a same-uid tenant that shares this directory |
| 322 | // cannot pre-plant either one (final review, plan 2, item 5); an EEXIST |
| 323 | // planting either name is reported as a failure and nothing at that name |
| 324 | // is removed, since it was not ours to remove. It returns nil for ELOOP, |
| 325 | // errNotELOOP if the link was followed, and any other error. |
| 326 | func realSymlinkCheck(dir string) error { |
| 327 | target := randomName() |
| 328 | f, err := instancefs.OpenBeneath(dir, target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) |
| 329 | if err != nil { |
| 330 | if errors.Is(err, os.ErrExist) { |
| 331 | return fmt.Errorf("a planted entry already occupies %s", filepath.Join(dir, target)) |
| 332 | } |
| 333 | return err |
| 334 | } |
| 335 | _, werr := f.Write([]byte("x")) |
| 336 | cerr := f.Close() |
| 337 | targetPath := filepath.Join(dir, target) |
| 338 | defer os.Remove(targetPath) |
| 339 | if werr != nil { |
| 340 | return werr |
| 341 | } |
| 342 | if cerr != nil { |
| 343 | return cerr |
| 344 | } |
| 345 | |
| 346 | link := randomName() |
| 347 | linkPath := filepath.Join(dir, link) |
| 348 | if err := os.Symlink(targetPath, linkPath); err != nil { |
| 349 | if errors.Is(err, os.ErrExist) { |
| 350 | return fmt.Errorf("a planted entry already occupies %s", linkPath) |
| 351 | } |
| 352 | return err |
| 353 | } |
| 354 | defer os.Remove(linkPath) |
| 355 | |
| 356 | // A plain open, deliberately: the whole point of this check is that |
| 357 | // the MOUNT's nosymfollow option refuses the follow. A resolve flag of |
| 358 | // our own (RESOLVE_NO_SYMLINKS) would answer ELOOP whether or not the |
| 359 | // mount carries the option, making the line vacuous either way (ledger |
| 360 | // ruling, final review, plan 2, item 5). |
| 361 | lf, err := os.Open(linkPath) |
| 362 | if err == nil { |
| 363 | lf.Close() |
| 364 | return errNotELOOP |
| 365 | } |
| 366 | if errors.Is(err, syscall.ELOOP) { |
| 367 | return nil |
| 368 | } |
| 369 | return err |
| 370 | } |
| 371 | |
| 372 | // dialUnix connects to a unix socket and closes the connection at once, |
| 373 | // returning nil only if the connect succeeded. |
| 374 | func dialUnix(path string) error { |
| 375 | c, err := net.DialTimeout("unix", path, 2*time.Second) |
| 376 | if err != nil { |
| 377 | return err |
| 378 | } |
| 379 | return c.Close() |
| 380 | } |
| 381 | |
| 382 | // unreachable reports whether err is a refusal for permission or absence: |
| 383 | // the answers a hidden path gives. Any other error, a refused connection |
| 384 | // among them, means the path itself was reached. |
| 385 | func unreachable(err error) bool { |
| 386 | return errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) || |
| 387 | errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.ENOTDIR) |
| 388 | } |
| 389 | |
amadan