| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "path/filepath" |
| 15 | "slices" |
| 16 | "strconv" |
| 17 | "strings" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/rootstate" |
| 20 | "golang.org/x/sys/unix" |
| 21 | ) |
| 22 | |
| 23 | type generationService struct { |
| 24 | group, executable, selectedPath string |
| 25 | main int |
| 26 | selected rootstate.FileIdentity |
| 27 | } |
| 28 | |
| 29 | // generationServiceEnv is what the service lookup reads besides the task's |
| 30 | // own /proc entries: systemd's view of the unit (show), the unit files' |
| 31 | // ownership (trust), the main process's cgroup (under proc), and, for a |
| 32 | // task still running a renamed-over binary, the box's signed router images |
| 33 | // (binary, under opt) and the owner every trusted file must have. The |
| 34 | // production values are generationServiceReader's; tests replace the parts |
| 35 | // a non-root test cannot provide. |
| 36 | type generationServiceEnv struct { |
| 37 | proc, opt string |
| 38 | owner uint32 |
| 39 | show func(ctx context.Context, unit string) (map[string]string, error) |
| 40 | trust func(path string) error |
| 41 | binary func(ctx context.Context, opt, path string) (rootFloorBinary, error) |
| 42 | } |
| 43 | |
| 44 | // generationServiceRefusal is a refusal of the service lookup. It names the |
| 45 | // check, the unit, the task's cgroup and its exe link, so that refusals |
| 46 | // that used to read as a bare "permission denied" can be told apart, and |
| 47 | // it wraps the cause, so errors.Is still sees os.ErrPermission or a task |
| 48 | // that exited. |
| 49 | type generationServiceRefusal struct { |
| 50 | check, unit, group, exe string |
| 51 | err error |
| 52 | } |
| 53 | |
| 54 | func (r *generationServiceRefusal) Error() string { |
| 55 | return fmt.Sprintf("%s check refused: unit %s, cgroup %s, exe %s: %v", r.check, r.unit, r.group, r.exe, r.err) |
| 56 | } |
| 57 | |
| 58 | func (r *generationServiceRefusal) Unwrap() error { return r.err } |
| 59 | |
| 60 | // generationServiceReader is the production service lookup: classify a |
| 61 | // task, and recheck the executable binding of a task it trusted |
| 62 | // (generationKernel.recheck). |
| 63 | func generationServiceReader(ctx context.Context) (func(string, []byte) (bool, error), func(string, bool) error) { |
| 64 | return generationServiceEnv{proc: "/proc", opt: "/opt/carlos", show: generationServiceShow, trust: generationServiceTrust, binary: readRootFloorBinaryContext}.reader(ctx) |
| 65 | } |
| 66 | |
| 67 | // generationExeID is what binds a task to the image it was proven to run: |
| 68 | // the device and inode of the descriptor opened through /proc/<pid>/exe, |
| 69 | // the kernel's file handle for it where the filesystem gives one, and its |
| 70 | // mount ID. rootstate.ObjectIdentity carries the same handle but also |
| 71 | // resolves the mount in root's own mount table, which a service in a |
| 72 | // private mount namespace does not share, so only its kernel facts are |
| 73 | // used here. |
| 74 | type generationExeID struct { |
| 75 | device, inode, mount uint64 |
| 76 | handle string |
| 77 | handleType int32 |
| 78 | } |
| 79 | |
| 80 | func generationExeIdentity(f *os.File) (generationExeID, error) { |
| 81 | var st unix.Stat_t |
| 82 | if err := unix.Fstat(int(f.Fd()), &st); err != nil { |
| 83 | return generationExeID{}, err |
| 84 | } |
| 85 | var sx unix.Statx_t |
| 86 | if err := unix.Statx(int(f.Fd()), "", unix.AT_EMPTY_PATH, unix.STATX_MNT_ID, &sx); err != nil { |
| 87 | return generationExeID{}, err |
| 88 | } |
| 89 | id := generationExeID{device: uint64(st.Dev), inode: st.Ino, mount: sx.Mnt_id} |
| 90 | handle, _, err := unix.NameToHandleAt(int(f.Fd()), "", unix.AT_EMPTY_PATH) |
| 91 | if err == nil { |
| 92 | id.handle, id.handleType = hex.EncodeToString(handle.Bytes()), handle.Type() |
| 93 | } else if !errors.Is(err, unix.EOPNOTSUPP) { |
| 94 | return generationExeID{}, err |
| 95 | } |
| 96 | return id, nil |
| 97 | } |
| 98 | |
| 99 | // generationExeBinding is a task the lookup trusted because the image it |
| 100 | // ran was proven: that image's descriptor, kept open for the inspection so |
| 101 | // its inode cannot be reused, and its identity. |
| 102 | type generationExeBinding struct { |
| 103 | file *os.File |
| 104 | id generationExeID |
| 105 | unit, group, exe string |
| 106 | } |
| 107 | |
| 108 | func generationServiceShow(ctx context.Context, unit string) (map[string]string, error) { |
| 109 | out, err := exec.CommandContext(ctx, "systemctl", "show", unit, "-p", "ControlGroup", "-p", "MainPID", "-p", "ExecStart", "-p", "FragmentPath", "-p", "DropInPaths").Output() |
| 110 | if err != nil { |
| 111 | return nil, err |
| 112 | } |
| 113 | props := map[string]string{} |
| 114 | for _, line := range strings.Split(string(out), "\n") { |
| 115 | if k, v, ok := strings.Cut(line, "="); ok { |
| 116 | props[k] = v |
| 117 | } |
| 118 | } |
| 119 | return props, nil |
| 120 | } |
| 121 | |
| 122 | func generationServiceTrust(file string) error { |
| 123 | file, err := filepath.EvalSymlinks(file) |
| 124 | if err != nil { |
| 125 | return err |
| 126 | } |
| 127 | f, err := rootstate.OpenTrustedFile(file) |
| 128 | if err != nil { |
| 129 | return err |
| 130 | } |
| 131 | f.Close() |
| 132 | return nil |
| 133 | } |
| 134 | |
| 135 | func (env generationServiceEnv) reader(ctx context.Context) (func(string, []byte) (bool, error), func(string, bool) error) { |
| 136 | units := map[string]generationService{} |
| 137 | bound := map[string]*generationExeBinding{} |
| 138 | var proven []string |
| 139 | var provenErr error |
| 140 | proved := false |
| 141 | // The digests of the current and the previous router binary, each |
| 142 | // proven against a signed identity the box holds, read once per census |
| 143 | // pass and only if a task's exe has been deleted. |
| 144 | prove := func() ([]string, error) { |
| 145 | if !proved { |
| 146 | proved = true |
| 147 | var errs []error |
| 148 | for _, name := range []string{"carlos", "carlos.prev"} { |
| 149 | b, err := env.binary(ctx, env.opt, filepath.Join(env.opt, name)) |
| 150 | if err != nil { |
| 151 | if name == "carlos" || !errors.Is(err, os.ErrNotExist) { |
| 152 | errs = append(errs, fmt.Errorf("%s: %w", name, err)) |
| 153 | } |
| 154 | continue |
| 155 | } |
| 156 | proven = append(proven, b.Digest) |
| 157 | } |
| 158 | if len(proven) == 0 { |
| 159 | provenErr = errors.Join(append(errs, os.ErrPermission)...) |
| 160 | } |
| 161 | } |
| 162 | return proven, provenErr |
| 163 | } |
| 164 | classify := func(path string, status []byte) (bool, error) { |
| 165 | var held *os.File |
| 166 | defer func() { |
| 167 | if held != nil { |
| 168 | held.Close() |
| 169 | } |
| 170 | }() |
| 171 | if old := bound[path]; old != nil { |
| 172 | old.file.Close() |
| 173 | delete(bound, path) |
| 174 | } |
| 175 | name, nameErr := os.Readlink(path + "/exe") |
| 176 | exe := name |
| 177 | if nameErr != nil { |
| 178 | exe = "unreadable" |
| 179 | } |
| 180 | unit, group := "unknown", "unknown" |
| 181 | refuse := func(check string, err error) error { |
| 182 | return &generationServiceRefusal{check: check, unit: unit, group: group, exe: exe, err: err} |
| 183 | } |
| 184 | raw, err := generationRead(path + "/cgroup") |
| 185 | if err != nil { |
| 186 | return false, refuse("cgroup", err) |
| 187 | } |
| 188 | group = strings.TrimPrefix(strings.TrimSpace(string(raw)), "0::") |
| 189 | if !strings.HasPrefix(group, "/") || strings.Contains(group, "\n") || strings.Contains(group, "/carlos-instances.slice/") { |
| 190 | return false, refuse("cgroup", os.ErrPermission) |
| 191 | } |
| 192 | unit = filepath.Base(group) |
| 193 | for _, part := range strings.Split(group, "/") { |
| 194 | if strings.HasSuffix(part, ".service") { |
| 195 | unit = part |
| 196 | break |
| 197 | } |
| 198 | } |
| 199 | if !strings.HasSuffix(unit, ".service") && group != "/init.scope" { |
| 200 | return false, refuse("unit", os.ErrPermission) |
| 201 | } |
| 202 | s, ok := units[unit] |
| 203 | if !ok { |
| 204 | if group == "/init.scope" { |
| 205 | s = generationService{group: group, main: 1} |
| 206 | } else { |
| 207 | props, err := env.show(ctx, unit) |
| 208 | if err != nil { |
| 209 | return false, refuse("systemctl show", err) |
| 210 | } |
| 211 | s.group, s.main = props["ControlGroup"], 0 |
| 212 | s.main, err = strconv.Atoi(props["MainPID"]) |
| 213 | if err != nil || s.main <= 0 || props["FragmentPath"] == "" { |
| 214 | return false, refuse("main pid", os.ErrPermission) |
| 215 | } |
| 216 | for _, file := range append([]string{props["FragmentPath"]}, strings.Fields(props["DropInPaths"])...) { |
| 217 | if err := env.trust(file); err != nil { |
| 218 | return false, refuse("unit file "+file, err) |
| 219 | } |
| 220 | } |
| 221 | _, tail, found := strings.Cut(props["ExecStart"], "path=") |
| 222 | if !found { |
| 223 | return false, refuse("exec start", os.ErrPermission) |
| 224 | } |
| 225 | s.executable, _, _ = strings.Cut(tail, " ;") |
| 226 | if unit == "carlos-edge.service" && s.executable == "/opt/carlos/system/guard/carlos" { |
| 227 | if !generationGuardEdgeStart(unit, props["ExecStart"]) { |
| 228 | return false, refuse("guard start", os.ErrPermission) |
| 229 | } |
| 230 | if _, err := env.binary(ctx, env.opt, s.executable); err != nil { |
| 231 | return false, refuse("guard binary", err) |
| 232 | } |
| 233 | selected, err := env.binary(ctx, env.opt, filepath.Join(env.opt, "carlos")) |
| 234 | if err != nil { |
| 235 | return false, refuse("selected binary", err) |
| 236 | } |
| 237 | s.selected, s.selectedPath = selected.File, filepath.Join(env.opt, "carlos") |
| 238 | } |
| 239 | } |
| 240 | units[unit] = s |
| 241 | } |
| 242 | supervisor := generationServiceSupervisor(unit, group, s.group, status, s.main) |
| 243 | if !generationServiceGroup(group, s.group) && !supervisor { |
| 244 | return false, refuse("group", os.ErrPermission) |
| 245 | } |
| 246 | if nameErr != nil { |
| 247 | return false, refuse("exe link", nameErr) |
| 248 | } |
| 249 | |
| 250 | // Non-root services may execute only the program selected by their |
| 251 | // root-owned unit. UID and a service-looking cgroup are insufficient. |
| 252 | _, rootErr := generationRootTask(status) |
| 253 | checked := rootErr != nil || s.selected != (rootstate.FileIdentity{}) |
| 254 | if program, deleted := strings.CutSuffix(name, " (deleted)"); deleted && checked { |
| 255 | want := s.executable |
| 256 | if s.selected != (rootstate.FileIdentity{}) { |
| 257 | want = s.selectedPath |
| 258 | } |
| 259 | image, err := env.deletedProgram(ctx, path, program, want, prove) |
| 260 | if err != nil { |
| 261 | return false, refuse("deleted exe", err) |
| 262 | } |
| 263 | held = image |
| 264 | } else { |
| 265 | actual, err := generationServiceExecutable(path, name) |
| 266 | if err != nil { |
| 267 | return false, refuse("exe", fmt.Errorf("%s: %w", name, err)) |
| 268 | } |
| 269 | if checked { |
| 270 | want, e := generationIdentity(s.executable) |
| 271 | if s.selected != (rootstate.FileIdentity{}) { |
| 272 | want, e = s.selected, nil |
| 273 | } |
| 274 | if e != nil || want != actual { |
| 275 | return false, refuse("selected program", os.ErrPermission) |
| 276 | } |
| 277 | } |
| 278 | } |
| 279 | mainGroup, err := generationRead(env.proc + "/" + strconv.Itoa(s.main) + "/cgroup") |
| 280 | mainPath := strings.TrimPrefix(strings.TrimSpace(string(mainGroup)), "0::") |
| 281 | if err != nil || (!generationServiceGroup(mainPath, s.group) && !(supervisor && mainPath == group)) { |
| 282 | if err == nil { |
| 283 | err = os.ErrPermission |
| 284 | } |
| 285 | return false, refuse("main cgroup", err) |
| 286 | } |
| 287 | if held != nil { |
| 288 | id, err := generationExeIdentity(held) |
| 289 | if err != nil { |
| 290 | return false, refuse("exe binding", err) |
| 291 | } |
| 292 | bound[path] = &generationExeBinding{file: held, id: id, unit: unit, group: group, exe: exe} |
| 293 | held = nil |
| 294 | } |
| 295 | return true, nil |
| 296 | } |
| 297 | return classify, func(path string, final bool) error { return env.recheck(bound, path, final) } |
| 298 | } |
| 299 | |
| 300 | // recheck holds a trusted task to the image it was proven to run, at each |
| 301 | // later use of that trust and at the end of its inspection (final). |
| 302 | // /proc/<pid>/exe, opened again, must be the very inode that was hashed (device, inode, mount and file handle), |
| 303 | // still a root-owned regular file with no links left and no group or other |
| 304 | // write. A task that exec'd anything else since, the same PID with the same |
| 305 | // start time, refuses; a later census pass may classify it afresh. The |
| 306 | // bytes are not hashed again: the identity is the hashed descriptor's, which |
| 307 | // stays open, and the kernel refuses writes to an image being executed |
| 308 | // (ETXTBSY). final releases the binding whatever the result. A task the |
| 309 | // lookup trusted without a proven image (its live exe was the unit's |
| 310 | // selected program, as on main) has no binding and passes. |
| 311 | func (env generationServiceEnv) recheck(bound map[string]*generationExeBinding, path string, final bool) error { |
| 312 | b := bound[path] |
| 313 | if b == nil { |
| 314 | return nil |
| 315 | } |
| 316 | if final { |
| 317 | defer func() { |
| 318 | b.file.Close() |
| 319 | delete(bound, path) |
| 320 | }() |
| 321 | } |
| 322 | refuse := func(err error) error { |
| 323 | return &generationServiceRefusal{check: "exe binding", unit: b.unit, group: b.group, exe: b.exe, err: err} |
| 324 | } |
| 325 | f, err := os.Open(path + "/exe") |
| 326 | if err != nil { |
| 327 | return refuse(err) |
| 328 | } |
| 329 | defer f.Close() |
| 330 | id, err := generationExeIdentity(f) |
| 331 | if err != nil { |
| 332 | return refuse(err) |
| 333 | } |
| 334 | var st unix.Stat_t |
| 335 | if err := unix.Fstat(int(f.Fd()), &st); err != nil { |
| 336 | return refuse(err) |
| 337 | } |
| 338 | if id != b.id || st.Uid != env.owner || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Mode&0022 != 0 || st.Nlink != 0 { |
| 339 | return refuse(fmt.Errorf("the task's image is no longer the one proven: %w", os.ErrPermission)) |
| 340 | } |
| 341 | return nil |
| 342 | } |
| 343 | |
| 344 | // deletedProgram accepts a non-root service task still running a carlos |
| 345 | // binary that an apply has since renamed over, whose exe link reads |
| 346 | // "<program> (deleted)". The program must be the one the unit selects and |
| 347 | // the box's router binary, <opt>/carlos. The task's own image, opened |
| 348 | // through /proc/<pid>/exe, must be a root-owned regular file with no links |
| 349 | // left that no group or other can write. Its bytes must hash to the router |
| 350 | // artifact of the signed identity of the current or the previous binary |
| 351 | // (<opt>/carlos, <opt>/carlos.prev), proven by readRootFloorBinary, the |
| 352 | // proof the guard makes before it execs one. Anything else refuses, and |
| 353 | // the census's reference checks still run on an accepted task. |
| 354 | // |
| 355 | // On success it returns the image's descriptor, open: the lookup keeps it |
| 356 | // to bind the task to that image for the rest of its inspection (recheck). |
| 357 | func (env generationServiceEnv) deletedProgram(ctx context.Context, task, program, want string, prove func() ([]string, error)) (*os.File, error) { |
| 358 | if program != want || want != filepath.Join(env.opt, "carlos") { |
| 359 | return nil, fmt.Errorf("%s is not this unit's router binary: %w", program, os.ErrPermission) |
| 360 | } |
| 361 | f, err := os.Open(task + "/exe") |
| 362 | if err != nil { |
| 363 | return nil, err |
| 364 | } |
| 365 | keep := false |
| 366 | defer func() { |
| 367 | if !keep { |
| 368 | f.Close() |
| 369 | } |
| 370 | }() |
| 371 | var st unix.Stat_t |
| 372 | if err := unix.Fstat(int(f.Fd()), &st); err != nil { |
| 373 | return nil, err |
| 374 | } |
| 375 | if st.Uid != env.owner || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Mode&0022 != 0 || st.Nlink != 0 || st.Size > 512<<20 { |
| 376 | return nil, fmt.Errorf("image is not an unlinked root-owned file: %w", os.ErrPermission) |
| 377 | } |
| 378 | sum := sha256.New() |
| 379 | if n, err := io.Copy(sum, generationContextReader{ctx: ctx, r: io.LimitReader(f, 512<<20+1)}); err != nil { |
| 380 | return nil, err |
| 381 | } else if n > 512<<20 { |
| 382 | return nil, os.ErrPermission |
| 383 | } |
| 384 | digests, err := prove() |
| 385 | if err != nil { |
| 386 | return nil, err |
| 387 | } |
| 388 | if !slices.Contains(digests, hex.EncodeToString(sum.Sum(nil))) { |
| 389 | return nil, fmt.Errorf("image is neither the current nor the previous signed router build: %w", os.ErrPermission) |
| 390 | } |
| 391 | keep = true |
| 392 | return f, nil |
| 393 | } |
| 394 | |
| 395 | func generationGuardEdgeStart(unit, start string) bool { |
| 396 | if unit != "carlos-edge.service" || strings.Count(start, "{") != 1 || strings.Count(start, "}") != 1 { |
| 397 | return false |
| 398 | } |
| 399 | fields := strings.Split(start, " ; ") |
| 400 | return len(fields) >= 4 && fields[0] == "{ path=/opt/carlos/system/guard/carlos" && |
| 401 | fields[1] == "argv[]=/opt/carlos/system/guard/carlos system guard-exec edge" && fields[2] == "ignore_errors=no" |
| 402 | } |
| 403 | |
| 404 | func generationServiceGroup(group, base string) bool { |
| 405 | if group == base && base != "" { |
| 406 | return true |
| 407 | } |
| 408 | if filepath.Clean(group) != group || base == "" || !strings.HasPrefix(group, base+"/") { |
| 409 | return false |
| 410 | } |
| 411 | for p := group; ; p = filepath.Dir(p) { |
| 412 | var st unix.Stat_t |
| 413 | if unix.Lstat(filepath.Join("/sys/fs/cgroup", p), &st) != nil || st.Uid != 0 || st.Mode&0022 != 0 || st.Mode&unix.S_IFMT != unix.S_IFDIR { |
| 414 | return false |
| 415 | } |
| 416 | if p == base { |
| 417 | return true |
| 418 | } |
| 419 | } |
| 420 | } |
| 421 | |
| 422 | func generationServiceExecutable(task, name string) (rootstate.FileIdentity, error) { |
| 423 | var zero rootstate.FileIdentity |
| 424 | if !filepath.IsAbs(name) || filepath.Clean(name) != name || name == "/" { |
| 425 | return zero, os.ErrPermission |
| 426 | } |
| 427 | root, err := os.Open(task + "/root") |
| 428 | if err != nil { |
| 429 | return zero, err |
| 430 | } |
| 431 | defer root.Close() |
| 432 | fd, err := unix.FcntlInt(root.Fd(), unix.F_DUPFD_CLOEXEC, 0) |
| 433 | if err != nil { |
| 434 | return zero, err |
| 435 | } |
| 436 | defer func() { unix.Close(fd) }() |
| 437 | parts := strings.Split(strings.TrimPrefix(name, "/"), "/") |
| 438 | for i := 0; ; i++ { |
| 439 | var st unix.Stat_t |
| 440 | if err := unix.Fstat(fd, &st); err != nil { |
| 441 | return zero, err |
| 442 | } |
| 443 | kind := uint32(unix.S_IFDIR) |
| 444 | if i == len(parts) { |
| 445 | kind = unix.S_IFREG |
| 446 | } |
| 447 | if st.Uid != 0 || st.Mode&unix.S_IFMT != kind || st.Mode&0022 != 0 || (kind == unix.S_IFREG && st.Nlink != 1) { |
| 448 | return zero, os.ErrPermission |
| 449 | } |
| 450 | if i == len(parts) { |
| 451 | id := rootstate.FileIdentity{Device: uint64(st.Dev), Inode: st.Ino} |
| 452 | actual, err := generationIdentity(task + "/exe") |
| 453 | if err != nil || actual != id { |
| 454 | return zero, os.ErrPermission |
| 455 | } |
| 456 | return id, nil |
| 457 | } |
| 458 | next, err := unix.Openat(fd, parts[i], unix.O_RDONLY|unix.O_NONBLOCK|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) |
| 459 | if err != nil { |
| 460 | return zero, err |
| 461 | } |
| 462 | unix.Close(fd) |
| 463 | fd = next |
| 464 | } |
| 465 | } |
| 466 | |
| 467 | func generationServiceSupervisor(unit, group, base string, status []byte, main int) bool { |
| 468 | if unit != "carlos-edge.service" || base == "" || filepath.Clean(base) != base || group != base+"/supervise" || main <= 0 { |
| 469 | return false |
| 470 | } |
| 471 | tgid := "" |
| 472 | for _, line := range strings.Split(string(status), "\n") { |
| 473 | if value, ok := strings.CutPrefix(line, "Tgid:"); ok { |
| 474 | if tgid != "" { |
| 475 | return false |
| 476 | } |
| 477 | tgid = strings.TrimSpace(value) |
| 478 | } |
| 479 | } |
| 480 | return tgid == strconv.Itoa(main) |
| 481 | } |
| 482 | |
amadan