| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "runtime" |
| 14 | "strconv" |
| 15 | "syscall" |
| 16 | "testing" |
| 17 | "time" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/instancectl" |
| 20 | "github.com/carlosframework/platform/internal/instanceexec" |
| 21 | "github.com/carlosframework/platform/internal/registry" |
| 22 | "github.com/carlosframework/platform/internal/release" |
| 23 | "github.com/carlosframework/platform/internal/rootstate" |
| 24 | "golang.org/x/sys/unix" |
| 25 | ) |
| 26 | |
| 27 | func TestPlan14GenerationContainedRunningRootRunner(t *testing.T) { |
| 28 | const key = "PLAN14_E45_CONTAINED" |
| 29 | dir, mode := os.Getenv("PLAN14_E45_DIR"), os.Getenv(key) |
| 30 | if mode == "mount" { |
| 31 | runtime.LockOSThread() |
| 32 | if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | if err := unix.Mount("proc", "/proc", "proc", 0, ""); err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | executable := release.VersionPath(filepath.Join(dir, "opt"), "bab", "v1", "shop") |
| 39 | if err := unix.Mount(executable, filepath.Join(dir, "bound-app"), "", unix.MS_BIND, ""); err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | if err := unix.Mount("tmpfs", filepath.Join(dir, "opt"), "tmpfs", unix.MS_RDONLY, "mode=0755"); err != nil { |
| 43 | t.Fatal(err) |
| 44 | } |
| 45 | group := instancectl.ClaimUnitControlGroup(filepath.Base(dir) + ".example") |
| 46 | if err := os.WriteFile(filepath.Join("/sys/fs/cgroup", group, "cgroup.procs"), []byte("0"), 0600); err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | if err := os.Setenv(key, "init"); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | env := os.Environ() |
| 53 | if err := syscall.Exec("/usr/bin/setpriv", []string{"setpriv", "--reuid=65534", "--regid=65534", "--clear-groups", "--bounding-set=-all", "--no-new-privs", filepath.Join(dir, "helper.test"), "-test.run=^TestPlan14GenerationContainedRunningRootRunner$", "-test.timeout=90s"}, env); err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | return |
| 57 | } |
| 58 | if mode == "init" { |
| 59 | runtime.LockOSThread() |
| 60 | if err := instanceexec.ScopeAbstractUnix(); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | if err := instanceexec.ScopeClaimGeneration(); err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | for _, call := range []uintptr{unix.SYS_RECVMSG, unix.SYS_RECVMMSG, unix.SYS_PIDFD_GETFD, unix.SYS_IO_URING_SETUP} { |
| 67 | _, _, e := unix.RawSyscall6(call, ^uintptr(0), 0, 0, 0, 0, 0) |
| 68 | if e != unix.EPERM { |
| 69 | t.Fatal("descriptor boundary missing", call, e) |
| 70 | } |
| 71 | } |
| 72 | tenant := exec.Command(filepath.Join(dir, "bound-app"), "90") |
| 73 | if err := tenant.Start(); err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | if err := tenant.Wait(); err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | return |
| 80 | } |
| 81 | if mode == "" { |
| 82 | dir = generationRootRunnerDir(t) |
| 83 | binary, err := os.ReadFile(os.Args[0]) |
| 84 | if err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if err := os.WriteFile(filepath.Join(dir, "helper.test"), binary, 0755); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | cmd := exec.CommandContext(t.Context(), "unshare", "--mount", "--pid", "--fork", "--mount-proc", filepath.Join(dir, "helper.test"), "-test.run=^TestPlan14GenerationContainedRunningRootRunner$", "-test.v", "-test.timeout=2m") |
| 91 | cmd.Dir = dir |
| 92 | cmd.Env = append(os.Environ(), key+"=census", "PLAN14_E45_DIR="+dir) |
| 93 | if out, err := cmd.CombinedOutput(); err != nil { |
| 94 | t.Fatalf("running containment: %v\n%s", err, out) |
| 95 | } else { |
| 96 | t.Logf("%s", out) |
| 97 | } |
| 98 | return |
| 99 | } |
| 100 | if mode != "census" || os.Geteuid() != 0 { |
| 101 | t.Fatal("invalid runner entry") |
| 102 | } |
| 103 | t.Setenv("CARLOS_OPT_DIR", filepath.Join(dir, "opt")) |
| 104 | body, err := os.ReadFile("/bin/sleep") |
| 105 | if err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | for _, version := range []string{"v1", "v2", "v3", "v4"} { |
| 109 | if err := release.StageArtifact(filepath.Join(dir, "opt"), "bab", version, "shop", body); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | old := time.Now().Add(-48 * time.Hour) |
| 113 | if err := os.Chtimes(release.VersionPath(filepath.Join(dir, "opt"), "bab", version, "shop"), old, old); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | } |
| 117 | if err := os.WriteFile(filepath.Join(dir, "bound-app"), nil, 0755); err != nil { |
| 118 | t.Fatal(err) |
| 119 | } |
| 120 | host := filepath.Base(dir) + ".example" |
| 121 | group := instancectl.ClaimUnitControlGroup(host) |
| 122 | var made []string |
| 123 | for p := filepath.Join("/sys/fs/cgroup", group); p != "/sys/fs/cgroup"; p = filepath.Dir(p) { |
| 124 | made = append(made, p) |
| 125 | } |
| 126 | var owned []string |
| 127 | for i := len(made) - 1; i >= 0; i-- { |
| 128 | if err := os.Mkdir(made[i], 0755); err == nil { |
| 129 | owned = append(owned, made[i]) |
| 130 | } else if !os.IsExist(err) { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | } |
| 134 | defer func() { |
| 135 | for i := len(owned) - 1; i >= 0; i-- { |
| 136 | if err := os.Remove(owned[i]); err != nil { |
| 137 | t.Error(err) |
| 138 | } |
| 139 | } |
| 140 | }() |
| 141 | cmd := exec.Command(filepath.Join(dir, "helper.test"), "-test.run=^TestPlan14GenerationContainedRunningRootRunner$", "-test.timeout=90s") |
| 142 | cmd.Dir = dir |
| 143 | cmd.Env = append(os.Environ(), key+"=mount") |
| 144 | cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: unix.CLONE_NEWNS | unix.CLONE_NEWPID} |
| 145 | cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr |
| 146 | if err := cmd.Start(); err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | defer func() { cmd.Process.Kill(); cmd.Wait() }() |
| 150 | pid := cmd.Process.Pid |
| 151 | k := linuxClaimLaunchKernel{ProcRoot: "/proc", CgroupRoot: "/sys/fs/cgroup", BoundApp: filepath.Join(dir, "bound-app")} |
| 152 | var observed ClaimLaunchObserved |
| 153 | deadline := time.Now().Add(10 * time.Second) |
| 154 | for { |
| 155 | observed, err = k.Observe(instancectl.ClaimUnitStarted{MainPID: pid, ControlGroup: group}) |
| 156 | if err == nil { |
| 157 | break |
| 158 | } |
| 159 | if time.Now().After(deadline) { |
| 160 | t.Fatalf("tenant observation: %+v %v", observed, err) |
| 161 | } |
| 162 | time.Sleep(10 * time.Millisecond) |
| 163 | } |
| 164 | boot, err := (generationKernel{proc: "/proc"}).boot() |
| 165 | if err != nil { |
| 166 | t.Fatal(err) |
| 167 | } |
| 168 | controller, err := claimLaunchProcessStart("/proc", os.Getpid()) |
| 169 | if err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | state, err := rootstate.Open(filepath.Join(dir, "generations"), true, false) |
| 173 | if err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | defer state.Close() |
| 177 | ledger := &release.RootGenerations{Store: state} |
| 178 | x := &release.ReleaseExclusion{Open: func() (rootstate.Store, error) { return rootstate.Open(filepath.Join(dir, "exclusion"), true, false) }} |
| 179 | identity := func(path string) rootstate.FileIdentity { |
| 180 | id, err := generationIdentity(path) |
| 181 | if err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | return id |
| 185 | } |
| 186 | digest := sha256.Sum256(body) |
| 187 | reserve := func(version string) release.GenerationReservation { |
| 188 | path := release.VersionPath(filepath.Join(dir, "opt"), "bab", version, "shop") |
| 189 | b := release.GenerationBinding{Binding: release.HistoryBinding{Host: host, Account: "bab", App: "shop"}, Target: release.GenerationOwner{Host: host, Incarnation: "fixture", Stamp: 1, Session: 1, Box: "runner", Enrollment: "fixture", BootID: boot, ControllerID: "fixture"}, Installation: "fixture", Attempt: version, Inputs: "fixture", Invocation: version, Record: "fixture", RecordETag: "fixture", ControllerPID: os.Getpid(), ControllerStart: controller, Version: version, Digest: hex.EncodeToString(digest[:]), Path: path, Tree: filepath.Dir(path), Executable: identity(path), TreeIdentity: identity(filepath.Dir(path))} |
| 190 | r, err := ledger.Reserve(b) |
| 191 | if err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | return r |
| 195 | } |
| 196 | var v1, v3 release.GenerationReservation |
| 197 | if err := x.With(t.Context(), func(context.Context) error { |
| 198 | v1, v3 = reserve("v1"), reserve("v3") |
| 199 | return ledger.Started(v1, release.GenerationProcess{PID: pid, Start: observed.ProcessStart, TenantPID: observed.TenantPID, TenantStart: observed.TenantStart, Group: group, Invocation: "fixture", GroupIdentity: identity(filepath.Join("/sys/fs/cgroup", group)), LauncherIdentity: identity(filepath.Join("/proc", strconv.Itoa(pid), "exe")), Containment: 1}) |
| 200 | }); err != nil { |
| 201 | t.Fatal(err) |
| 202 | } |
| 203 | v1, err = ledger.Read(v1.Binding) |
| 204 | if err != nil { |
| 205 | t.Fatal(err) |
| 206 | } |
| 207 | source := generationPinSource{open: func() (rootstate.Store, error) { |
| 208 | return rootstate.Open(filepath.Join(dir, "generations"), false, false) |
| 209 | }, verify: func(ctx context.Context, rs []release.GenerationReservation) error { |
| 210 | // Enrollment/launch-journal agreement is fixture authority in this witness. |
| 211 | kernel := generationKernel{proc: "/proc", cgroups: "/sys/fs/cgroup", boundApp: filepath.Join(dir, "bound-app"), contained: rs} |
| 212 | _, _, err := kernel.census(ctx, filepath.Join(dir, "opt", "versions")) |
| 213 | if err != nil { |
| 214 | tasks, _ := filepath.Glob("/proc/[0-9]*/task/[0-9]*") |
| 215 | for _, task := range tasks { |
| 216 | status, _ := generationRead(task + "/status") |
| 217 | ok, e := kernel.containedTask(ctx, task, status) |
| 218 | if e != nil { |
| 219 | t.Logf("containment %s ok=%v err=%v status=%s", task, ok, e, status) |
| 220 | } |
| 221 | } |
| 222 | } |
| 223 | return err |
| 224 | }} |
| 225 | for _, attack := range []string{"none", "erased legacy", "renamed legacy"} { |
| 226 | legacy := filepath.Join(dir, "zdd") |
| 227 | if attack == "none" { |
| 228 | if err := os.Mkdir(legacy, 0755); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | if err := release.WriteGenerations(filepath.Join(legacy, "generations.json"), release.Generations{}); err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | } |
| 235 | for _, name := range []string{"generations.json", "generations.seq"} { |
| 236 | p := filepath.Join(legacy, name) |
| 237 | if attack == "erased legacy" { |
| 238 | if err := os.Remove(p); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | } |
| 242 | } |
| 243 | if attack == "renamed legacy" { |
| 244 | if err := os.Rename(legacy, legacy+".old"); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | } |
| 248 | if err := x.With(t.Context(), func(ctx context.Context) error { |
| 249 | pins, err := source.StagedPins(ctx, filepath.Join(dir, "opt")) |
| 250 | if err != nil { |
| 251 | return fmt.Errorf("%s: %w", attack, err) |
| 252 | } |
| 253 | for _, r := range []release.GenerationReservation{v1, v3} { |
| 254 | if ok, err := pins.Refers(r.Binding.Path); err != nil || !ok { |
| 255 | t.Fatal("missing pin", err) |
| 256 | } |
| 257 | } |
| 258 | sweepStagedVersions(fakeLister{routes: []registry.Route{{Host: host, Account: "bab", App: "shop", Version: "v2"}}}, filepath.Join(dir, "opt"), filepath.Join(legacy, "generations.json"), time.Hour, time.Now(), source) |
| 259 | return nil |
| 260 | }); err != nil { |
| 261 | t.Fatal(err) |
| 262 | } |
| 263 | if !staged(t, filepath.Join(dir, "opt"), "v1") || !staged(t, filepath.Join(dir, "opt"), "v2") || !staged(t, filepath.Join(dir, "opt"), "v3") || staged(t, filepath.Join(dir, "opt"), "v4") { |
| 264 | t.Fatal("running/pending pins or permitted sweep incorrect") |
| 265 | } |
| 266 | } |
| 267 | if got, err := ledger.Read(v3.Binding); err != nil || got != v3 { |
| 268 | t.Fatal("pending reservation changed", err) |
| 269 | } |
| 270 | } |
| 271 | |
amadan