| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "os" |
| 12 | "os/exec" |
| 13 | "path/filepath" |
| 14 | "strings" |
| 15 | "syscall" |
| 16 | "testing" |
| 17 | "time" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/hostbox" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/provision" |
| 22 | "github.com/carlosframework/platform/internal/rootstate" |
| 23 | "golang.org/x/sys/unix" |
| 24 | ) |
| 25 | |
| 26 | func servingRootRunner(t *testing.T) { |
| 27 | t.Helper() |
| 28 | mapping, err := os.ReadFile("/proc/self/uid_map") |
| 29 | if err != nil || os.Geteuid() != 0 || strings.Join(strings.Fields(string(mapping)), " ") != "0 0 4294967295" { |
| 30 | t.Fatalf("root runner needs real root in the initial user namespace: %v", err) |
| 31 | } |
| 32 | if t.Name() == "TestRootServingParkUnitPayload" || os.Getenv("PLAN14_ROOT_FIXTURE") == t.Name() { |
| 33 | // Unit and chroot children use the parent-owned fixture filesystem. |
| 34 | return |
| 35 | } |
| 36 | // The runner's /tmp is a small RAM filesystem. Serving now enforces the |
| 37 | // real data-disk floor, so positive fixtures need the instance's disk. |
| 38 | data, err := os.MkdirTemp("/var/lib", "carlos-serving-test-") |
| 39 | if err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | t.Cleanup(func() { |
| 43 | if err := os.RemoveAll(data); err != nil { |
| 44 | t.Error(err) |
| 45 | } |
| 46 | }) |
| 47 | if !strings.HasPrefix(t.Name(), "TestRootReboot") { |
| 48 | scratch := servingRunnerScratchFilesystem(t, data, 2<<30) |
| 49 | t.Setenv("TMPDIR", scratch) |
| 50 | } else { |
| 51 | t.Setenv("TMPDIR", data) |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | // This includes loopback S3 and Unix-socket fixtures; it makes no remote calls. |
| 56 | // Installation, peer/census admission and prelaunch cgroups remain test seams. |
| 57 | func TestRootServingParkNestedBindRestore(t *testing.T) { |
| 58 | servingRootRunner(t) |
| 59 | if os.Getenv("PLAN14_ROOT_FIXTURE") == t.Name() { |
| 60 | if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | if !plan14RootFixture(t) { |
| 64 | t.Fatal("root child did not enter its scratch root") |
| 65 | } |
| 66 | for _, failure := range []string{"", "enrolled-refusal", "missing-marker", "cancel-after-enrollment", "disk-after-enrollment", "invalid-sqlite", "cancel-after-output", "restore-checksum", "restore-enospc"} { |
| 67 | for _, restart := range []string{"same-controller", "controller", "reboot"} { |
| 68 | t.Run(failure+"/"+restart, func(t *testing.T) { runFilesSeedOrdinaryPreparation(t, failure, restart, true) }) |
| 69 | } |
| 70 | } |
| 71 | return |
| 72 | } |
| 73 | root := t.TempDir() |
| 74 | for _, name := range []string{"scratch", "proc", "foreign-dir"} { |
| 75 | if err := os.Mkdir(filepath.Join(root, name), 0755); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | } |
| 79 | if err := os.WriteFile(filepath.Join(root, "foreign"), nil, 0644); err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | binary, err := os.Executable() |
| 83 | if err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | cmd := exec.CommandContext(t.Context(), binary, "-test.run=^"+t.Name()+"$", "-test.v", "-test.timeout=3m") |
| 87 | cmd.Env = append(os.Environ(), "PLAN14_ROOT_FIXTURE="+t.Name(), "PLAN14_ROOT_DIR="+root) |
| 88 | cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: unix.CLONE_NEWNS} |
| 89 | if out, err := cmd.CombinedOutput(); err != nil { |
| 90 | t.Fatalf("real-root mounted restore: %v\n%s", err, out) |
| 91 | } else { |
| 92 | t.Log(string(out)) |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | func servingRunnerMount(t *testing.T, data string) { |
| 97 | t.Helper() |
| 98 | instances := filepath.Join(data, "instances") |
| 99 | if err := os.MkdirAll(instances, 0700); err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | if err := unix.Mount(instances, instances, "", unix.MS_BIND, ""); err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | if err := unix.Mount("", instances, "", unix.MS_PRIVATE|unix.MS_REC, ""); err != nil { |
| 106 | _ = unix.Unmount(instances, 0) |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | t.Cleanup(func() { |
| 110 | if err := unix.Unmount(instances, 0); err != nil { |
| 111 | t.Error(err) |
| 112 | } |
| 113 | }) |
| 114 | if err := unix.Mount("", instances, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_NOSYMFOLLOW|unix.MS_NOSUID|unix.MS_NODEV|unix.MS_NOEXEC, ""); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // Exercise the production filesystem and private store. The authorization is |
| 120 | // a fixture, so this helper proves no peer, census or installation admission. |
| 121 | func servingRunnerPark(t *testing.T, data string, state rootstate.Store) servingParkRecord { |
| 122 | t.Helper() |
| 123 | current := servingArtifactIdentity{Host: "runner.example", Box: "runner", Region: "test", Incarnation: "inc", Location: "instances/runner.example", Generation: "G2", Stamp: 2, Session: 2} |
| 124 | db := provision.NestedDBPath(data, current.Host) |
| 125 | if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | for suffix, body := range map[string]string{"": "database", "-wal": "WAL", "-shm": "SHM"} { |
| 129 | if err := os.WriteFile(db+suffix, []byte(body), 0600); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | } |
| 133 | dir, err := os.Open(filepath.Dir(db)) |
| 134 | if err != nil { |
| 135 | t.Fatal(err) |
| 136 | } |
| 137 | id, err := servingParkDirID(dir) |
| 138 | dir.Close() |
| 139 | if err != nil { |
| 140 | t.Fatal(err) |
| 141 | } |
| 142 | fs := newServingParkFS(data, state) |
| 143 | a := servingParkAuthorization{Successor: current, OriginalSuccessor: current, Directory: id, Legacy: true} |
| 144 | a.ID = servingRetirementID(a.Predecessor, current, true) |
| 145 | // The helper's installation authorization remains a fixture. Capture the |
| 146 | // parent through the production recorder so reboot can inspect its proof. |
| 147 | if err := fs.recordNamespaceParent(t.Context(), current, id); err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | source, err := fs.Capture(a) |
| 151 | if err != nil { |
| 152 | t.Fatal(err) |
| 153 | } |
| 154 | // Stop authorization remains explicitly fixture-backed. It is encoded in |
| 155 | // the production record format so the reboot inspector reads the ledger. |
| 156 | at := time.Now().UTC() |
| 157 | target := hostguard.FenceTarget{Host: current.Host, Box: current.Box, Region: current.Region, Incarnation: current.Incarnation, Stamp: 1, Session: 1, FenceID: "runner-stop", Enrollment: "runner-enrollment", BootID: source.BootID, ControllerID: "runner-controller", DirectoryRevision: "1", RosterRevision: 1, ClaimETag: "runner-claim", ClaimGen: 1} |
| 158 | receipt := StopReceipt{V: 1, ID: target.FenceID, Kind: StopReceiptStopped, Region: target.Region, Host: target.Host, Box: target.Box, Incarnation: target.Incarnation, InstallationStamp: target.Stamp, WriterSession: target.Session, FenceID: target.FenceID, Enrollment: target.Enrollment, BootID: target.BootID, ControllerID: target.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: target.DirectoryRevision, ClaimETag: target.ClaimETag, ClaimGen: target.ClaimGen, EvidenceAt: at} |
| 159 | r := servingParkRecord{Version: 1, Authorization: a, Source: source, Disposition: servingParkRetain, Receipts: []hostguard.FenceReceipt{{Target: target, Bytes: mustMarshalStopReceipt(receipt), EvidenceAt: at}}} |
| 160 | save := func() error { |
| 161 | l := servingParkLedger{state: state} |
| 162 | if err := l.save(r); err != nil { |
| 163 | return err |
| 164 | } |
| 165 | return l.write("fixture.json", r) |
| 166 | } |
| 167 | if err = save(); err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | if err = fs.Prepare(&r, save); err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | if err = fs.Move(&r, save); err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | if err = fs.CheckParked(&r); err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | r.Parked, r.ParkedAt = true, time.Now().UTC() |
| 180 | r.ReapAfter = r.ParkedAt.Add(7 * 24 * time.Hour) |
| 181 | if err = save(); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | ledger := servingParkLedger{state: state} |
| 185 | if err := ledger.write("slot-"+servingParkKey(current.Box, current.Host)+".json", servingParkSlot{Version: 1, Box: current.Box, Host: current.Host, ID: a.ID, Pending: a.ID, Last: a.ID}); err != nil { |
| 186 | t.Fatal(err) |
| 187 | } |
| 188 | if err := ledger.write(servingLegacyKey("legacy", current), servingLegacyConsumption{Authorization: a, Source: source, Receipts: r.Receipts}); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | return r |
| 192 | } |
| 193 | |
| 194 | // This child runs only through the installed test unit below. The production |
| 195 | // hardening is retained, with a test binary and scratch deployment roots. |
| 196 | func TestRootServingParkUnitPayload(t *testing.T) { |
| 197 | servingRootRunner(t) |
| 198 | data, authority := os.Getenv("SERVING_RUNNER_DATA"), os.Getenv("SERVING_RUNNER_AUTHORITY") |
| 199 | mode := os.Getenv("SERVING_RUNNER_MODE") |
| 200 | if data == "" || authority == "" || (mode != "on" && mode != "off") { |
| 201 | t.Fatal("invoke through TestRootServingParkInstalledUnit") |
| 202 | } |
| 203 | if mode == "off" { |
| 204 | if err := os.WriteFile(filepath.Join(data, "must-not-write"), nil, 0600); !errors.Is(err, syscall.EROFS) { |
| 205 | t.Fatalf("claims-off data write: %v", err) |
| 206 | } |
| 207 | if err := os.WriteFile(filepath.Join(authority, "must-not-write"), nil, 0600); !errors.Is(err, syscall.EROFS) { |
| 208 | t.Fatalf("claims-off authority write: %v", err) |
| 209 | } |
| 210 | return |
| 211 | } |
| 212 | state, err := rootstate.Open(authority, false, false) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | defer state.Close() |
| 217 | r := servingRunnerPark(t, data, state) |
| 218 | fs := newServingParkFS(data, state) |
| 219 | // Exercise persistent handles inside the unchanged hardened unit. A |
| 220 | // simulated boot lets the read-only checker run; it enables no effects. |
| 221 | fs.boot = func() (string, error) { return servingOtherBoot, nil } |
| 222 | if _, err := fs.inspectRebootTopology(t.Context()); err != nil { |
| 223 | t.Fatal("installed-unit topology inspection", err) |
| 224 | } |
| 225 | beforeRecord, err := state.Read("retirement-" + r.Authorization.ID + ".json") |
| 226 | if err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | if _, err := fs.inspectRebootRecord(t.Context(), r.Authorization.ID); err != nil { |
| 230 | t.Fatal("installed-unit record inspection", err) |
| 231 | } |
| 232 | servingRunnerPrepareReboot(t, fs, r) |
| 233 | afterRecord, err := state.Read("retirement-" + r.Authorization.ID + ".json") |
| 234 | if err != nil || !bytes.Equal(beforeRecord, afterRecord) { |
| 235 | t.Fatal("record inspection changed history", err) |
| 236 | } |
| 237 | fs.boot = servingParkBootID |
| 238 | r.Deleting, r.Cause = true, "expiry" |
| 239 | save := func() error { return (&servingParkLedger{state: state}).write("fixture.json", r) } |
| 240 | fs.unlink = func(fd int, name string, flags int) error { |
| 241 | err := unix.Unlinkat(fd, name, flags) |
| 242 | if err == nil && flags == unix.AT_REMOVEDIR { |
| 243 | return unix.EIO |
| 244 | } |
| 245 | return err |
| 246 | } |
| 247 | if err := fs.Delete(&r, save); !errors.Is(err, unix.EIO) { |
| 248 | t.Fatalf("lost empty-directory removal reply: %v", err) |
| 249 | } |
| 250 | body, err := state.Read("fixture.json") |
| 251 | if err != nil || servingParkDecode(body, &r) != nil || !r.EventRemoveIntent || r.EventRemoved { |
| 252 | t.Fatal("missing durable event removal intent", err) |
| 253 | } |
| 254 | fs = newServingParkFS(data, state) |
| 255 | if err = fs.Delete(&r, save); err != nil || !r.EventRemoved { |
| 256 | t.Fatal("installed unit did not recover empty event removal", err) |
| 257 | } |
| 258 | if _, err = os.Lstat(filepath.Join(data, "instances", ".serving-parked", r.Authorization.ID)); !errors.Is(err, os.ErrNotExist) { |
| 259 | t.Fatal("recorded empty event survived", err) |
| 260 | } |
| 261 | for _, item := range r.Source.Files { |
| 262 | if item.Present && !item.Deleted { |
| 263 | t.Fatal("installed unit did not delete recorded artifact") |
| 264 | } |
| 265 | } |
| 266 | } |
| 267 | |
| 268 | func servingRunnerSystemctl(t *testing.T, args ...string) string { |
| 269 | t.Helper() |
| 270 | ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) |
| 271 | defer cancel() |
| 272 | out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() |
| 273 | if err != nil { |
| 274 | if len(args) > 1 && args[0] == "start" { |
| 275 | journal, _ := exec.CommandContext(ctx, "journalctl", "-u", args[1], "--no-pager", "-n", "60").CombinedOutput() |
| 276 | t.Log(string(journal)) |
| 277 | } |
| 278 | t.Fatalf("systemctl %v: %v\n%s", args, err, out) |
| 279 | } |
| 280 | return string(out) |
| 281 | } |
| 282 | |
| 283 | func TestRootServingParkInstalledUnit(t *testing.T) { |
| 284 | servingRootRunner(t) |
| 285 | pid1, err := os.ReadFile("/proc/1/comm") |
| 286 | if err != nil || strings.TrimSpace(string(pid1)) != "systemd" { |
| 287 | t.Fatal("root runner needs systemd as PID 1") |
| 288 | } |
| 289 | root, err := os.MkdirTemp("/var/lib", "carlos-rootrunner-") |
| 290 | if err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | t.Cleanup(func() { |
| 294 | if err := os.RemoveAll(root); err != nil { |
| 295 | t.Error(err) |
| 296 | } |
| 297 | }) |
| 298 | data := filepath.Join(root, "data") |
| 299 | opt := filepath.Join(root, "opt") |
| 300 | // Create only absent deployment directories and remove only those on exit. |
| 301 | for _, path := range []string{data, opt, "/run/carlos-control", "/run/carlos", "/var/lib/carlos-claims", "/var/lib/carlos-release", rootReleasePrivate} { |
| 302 | if err := os.Mkdir(path, 0700); err == nil { |
| 303 | if path == data || path == opt { |
| 304 | continue |
| 305 | } |
| 306 | p := path |
| 307 | t.Cleanup(func() { |
| 308 | if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 309 | t.Error(err) |
| 310 | } |
| 311 | }) |
| 312 | } else if !errors.Is(err, os.ErrExist) { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | } |
| 316 | authority := filepath.Join("/var/lib/carlos-claims", filepath.Base(root)) |
| 317 | if err := os.Mkdir(authority, 0700); err != nil { |
| 318 | t.Fatal(err) |
| 319 | } |
| 320 | t.Cleanup(func() { |
| 321 | if err := os.RemoveAll(authority); err != nil { |
| 322 | t.Error(err) |
| 323 | } |
| 324 | }) |
| 325 | servingRunnerMount(t, data) |
| 326 | binary, err := os.Executable() |
| 327 | if err != nil { |
| 328 | t.Fatal(err) |
| 329 | } |
| 330 | body, err := os.ReadFile(binary) |
| 331 | if err != nil { |
| 332 | t.Fatal(err) |
| 333 | } |
| 334 | installed := filepath.Join(root, "runner.test") |
| 335 | if err := os.WriteFile(installed, body, 0700); err != nil { |
| 336 | t.Fatal(err) |
| 337 | } |
| 338 | raw, err := unitsFS.ReadFile("units/" + claimControlUnit) |
| 339 | if err != nil { |
| 340 | t.Fatal(err) |
| 341 | } |
| 342 | for _, mode := range []string{"off", "on"} { |
| 343 | t.Run(mode, func(t *testing.T) { |
| 344 | unit := filepath.Base(root) + "-" + mode + ".service" |
| 345 | content := claimUnitContent(claimControlUnit, raw, claimUnitSettings{enabled: mode == "on", data: data, opt: opt}) |
| 346 | var lines []string |
| 347 | for _, line := range strings.Split(string(content), "\n") { |
| 348 | if strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") || strings.HasPrefix(line, "ExecStart=") || strings.HasPrefix(line, "EnvironmentFile=") { |
| 349 | continue |
| 350 | } |
| 351 | lines = append(lines, line) |
| 352 | } |
| 353 | content = []byte(strings.Join(lines, "\n") + fmt.Sprintf("\nType=oneshot\nTimeoutStartSec=45s\nExecStart=%s -test.run=^TestRootServingParkUnitPayload$ -test.v -test.timeout=30s\nEnvironment=SERVING_RUNNER_DATA=%s SERVING_RUNNER_AUTHORITY=%s SERVING_RUNNER_MODE=%s\nIPAddressDeny=any\n", installed, data, authority, mode)) |
| 354 | path := filepath.Join("/run/systemd/system", unit) |
| 355 | file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) |
| 356 | if err != nil { |
| 357 | t.Fatal(err) |
| 358 | } |
| 359 | _, writeErr := file.Write(content) |
| 360 | closeErr := file.Close() |
| 361 | t.Cleanup(func() { |
| 362 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
| 363 | defer cancel() |
| 364 | if out, err := exec.CommandContext(ctx, "systemctl", "stop", unit).CombinedOutput(); err != nil { |
| 365 | t.Errorf("stop test unit: %v %s", err, out) |
| 366 | } |
| 367 | if err := os.Remove(path); err != nil { |
| 368 | t.Error(err) |
| 369 | } |
| 370 | if out, err := exec.CommandContext(ctx, "systemctl", "daemon-reload").CombinedOutput(); err != nil { |
| 371 | t.Errorf("reload after cleanup: %v %s", err, out) |
| 372 | } |
| 373 | }) |
| 374 | if err := errors.Join(writeErr, closeErr); err != nil { |
| 375 | t.Fatal(err) |
| 376 | } |
| 377 | servingRunnerSystemctl(t, "daemon-reload") |
| 378 | servingRunnerSystemctl(t, "start", unit) |
| 379 | if got := strings.TrimSpace(servingRunnerSystemctl(t, "show", unit, "--property=Result", "--value")); got != "success" { |
| 380 | t.Fatal(got) |
| 381 | } |
| 382 | if got := strings.TrimSpace(servingRunnerSystemctl(t, "show", unit, "--property=ExecMainStatus", "--value")); got != "0" { |
| 383 | t.Fatal(got) |
| 384 | } |
| 385 | }) |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | const servingRebootRoot = "/var/lib/carlos-rootrunner-serving-reboot" |
| 390 | |
| 391 | type servingRebootWitness struct { |
| 392 | Boot string |
| 393 | Record servingParkRecord |
| 394 | } |
| 395 | |
| 396 | // The runner reboots between these phases. Raw history stays unchanged while |
| 397 | // fresh persistent evidence permits current-boot completion and reaping. |
| 398 | func TestRootRebootServingParkChangedMountPhase1(t *testing.T) { |
| 399 | servingRootRunner(t) |
| 400 | if err := os.Mkdir(servingRebootRoot, 0700); err != nil { |
| 401 | t.Fatal(err) |
| 402 | } |
| 403 | data := filepath.Join(servingRebootRoot, "data") |
| 404 | var grow func() |
| 405 | if strings.Contains(t.Name(), "GrownDisk") { |
| 406 | grow = servingRunnerGrownDisk(t, data, true) |
| 407 | } |
| 408 | servingRunnerMount(t, data) |
| 409 | state, err := rootstate.Open(filepath.Join(servingRebootRoot, "authority"), true, false) |
| 410 | if err != nil { |
| 411 | t.Fatal(err) |
| 412 | } |
| 413 | defer state.Close() |
| 414 | r := servingRunnerPark(t, data, state) |
| 415 | servingRunnerTopologyDiagnostic(t, newServingParkFS(data, state)) |
| 416 | boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") |
| 417 | if err != nil { |
| 418 | t.Fatal(err) |
| 419 | } |
| 420 | if err := (&servingParkLedger{state: state}).write("reboot.json", servingRebootWitness{Boot: strings.TrimSpace(string(boot)), Record: r}); err != nil { |
| 421 | t.Fatal(err) |
| 422 | } |
| 423 | if grow != nil { |
| 424 | grow() |
| 425 | } |
| 426 | } |
| 427 | |
| 428 | func TestRootRebootServingParkChangedMountPhase2(t *testing.T) { |
| 429 | servingRootRunner(t) |
| 430 | state, err := rootstate.Open(filepath.Join(servingRebootRoot, "authority"), false, false) |
| 431 | if err != nil { |
| 432 | t.Fatal(err) |
| 433 | } |
| 434 | defer state.Close() |
| 435 | body, err := state.Read("reboot.json") |
| 436 | if err != nil { |
| 437 | t.Fatal(err) |
| 438 | } |
| 439 | var saved servingRebootWitness |
| 440 | if err := json.Unmarshal(body, &saved); err != nil { |
| 441 | t.Fatal(err) |
| 442 | } |
| 443 | boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") |
| 444 | if err != nil || saved.Boot == "" || strings.TrimSpace(string(boot)) == saved.Boot { |
| 445 | t.Fatal("phase 2 requires an actual intervening reboot") |
| 446 | } |
| 447 | t.Cleanup(func() { |
| 448 | if err := os.RemoveAll(servingRebootRoot); err != nil { |
| 449 | t.Error(err) |
| 450 | } |
| 451 | }) |
| 452 | data := filepath.Join(servingRebootRoot, "data") |
| 453 | if strings.Contains(t.Name(), "GrownDisk") { |
| 454 | servingRunnerGrownDisk(t, data, false) |
| 455 | } |
| 456 | // Mount IDs can be reused across boots. Add bind layers until this fixture |
| 457 | // has a different ID, instead of assuming reboot alone changes the number. |
| 458 | fs := newServingParkFS(data, state) |
| 459 | for i := 0; i < 8; i++ { |
| 460 | servingRunnerMount(t, data) |
| 461 | parent, id, err := fs.parent() |
| 462 | if err != nil { |
| 463 | t.Fatal(err) |
| 464 | } |
| 465 | parent.Close() |
| 466 | if id.Mount != saved.Record.Source.Parent.Mount { |
| 467 | break |
| 468 | } |
| 469 | if i == 7 { |
| 470 | t.Fatal("could not create changed mount topology") |
| 471 | } |
| 472 | } |
| 473 | before, err := state.Read("namespace.json") |
| 474 | if err != nil { |
| 475 | t.Fatal(err) |
| 476 | } |
| 477 | originalNamespace := append([]byte(nil), before...) |
| 478 | |
| 479 | // Prove independent recognition after a real reboot before exercising the |
| 480 | // old-authority refusals below. No namespace or retirement is rewritten. |
| 481 | parentBefore, err := state.Read("namespace-parent.json") |
| 482 | if err != nil { |
| 483 | t.Fatal(err) |
| 484 | } |
| 485 | servingRunnerTopologyDiagnostic(t, fs) |
| 486 | observation, err := fs.inspectRebootTopology(t.Context()) |
| 487 | if err != nil || observation.BootID != strings.TrimSpace(string(boot)) || observation.Previous.BootID != saved.Boot || observation.CurrentParent.Mount == observation.Previous.Parent.Mount { |
| 488 | t.Fatal("persistent topology inspection after actual reboot", observation, err) |
| 489 | } |
| 490 | for _, replacement := range []string{"directory-bind", "filesystem"} { |
| 491 | t.Run(replacement, func(t *testing.T) { |
| 492 | path := filepath.Join(data, "instances", ".serving-parked") |
| 493 | if replacement == "directory-bind" { |
| 494 | other := filepath.Join(data, "replacement") |
| 495 | if err := os.Mkdir(other, 0700); err != nil { |
| 496 | t.Fatal(err) |
| 497 | } |
| 498 | if err := unix.Mount(other, path, "", unix.MS_BIND, ""); err != nil { |
| 499 | t.Fatal(err) |
| 500 | } |
| 501 | } else if err := unix.Mount("tmpfs", path, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, "size=1m,mode=0700"); err != nil { |
| 502 | t.Fatal(err) |
| 503 | } |
| 504 | t.Cleanup(func() { |
| 505 | if err := unix.Unmount(path, 0); err != nil { |
| 506 | t.Error(err) |
| 507 | } |
| 508 | }) |
| 509 | if _, err := fs.inspectRebootTopology(t.Context()); err == nil { |
| 510 | t.Fatal("replacement mount accepted across reboot") |
| 511 | } |
| 512 | if body, err := state.Read("namespace.json"); err != nil || !bytes.Equal(body, before) { |
| 513 | t.Fatal("replacement refusal changed authority", err) |
| 514 | } |
| 515 | }) |
| 516 | } |
| 517 | if _, err := fs.inspectRebootTopology(t.Context()); err != nil { |
| 518 | t.Fatal("original mount after replacement controls", err) |
| 519 | } |
| 520 | parentAfter, err := state.Read("namespace-parent.json") |
| 521 | if err != nil || !bytes.Equal(parentBefore, parentAfter) { |
| 522 | t.Fatal("inspection rewrote parent authority", err) |
| 523 | } |
| 524 | beforeRecord, err := state.Read("retirement-" + saved.Record.Authorization.ID + ".json") |
| 525 | if err != nil { |
| 526 | t.Fatal(err) |
| 527 | } |
| 528 | recordObservation, err := fs.inspectRebootRecord(t.Context(), saved.Record.Authorization.ID) |
| 529 | if err != nil || recordObservation.Source.Mount != observation.CurrentParent.Mount || recordObservation.Event.Mount != observation.CurrentRoot.Mount { |
| 530 | t.Fatal("persistent source/event/artifacts after reboot", recordObservation, err) |
| 531 | } |
| 532 | // Model a reused file inode by changing only the saved persistent handle. |
| 533 | // The same device/inode/size must not suffice to identify an old artifact. |
| 534 | corrupted := saved.Record |
| 535 | corrupted.Source.Files[0].Persistent.Handle = "aa" |
| 536 | ledger := servingParkLedger{state: state} |
| 537 | if err := ledger.save(corrupted); err != nil { |
| 538 | t.Fatal(err) |
| 539 | } |
| 540 | if _, err := fs.inspectRebootRecord(t.Context(), corrupted.Authorization.ID); err == nil { |
| 541 | t.Fatal("changed persistent artifact handle accepted") |
| 542 | } |
| 543 | if err := state.Write("retirement-"+saved.Record.Authorization.ID+".json", beforeRecord); err != nil { |
| 544 | t.Fatal(err) |
| 545 | } |
| 546 | if _, err := fs.inspectRebootRecord(t.Context(), saved.Record.Authorization.ID); err != nil { |
| 547 | t.Fatal("restored read-only control", err) |
| 548 | } |
| 549 | servingRunnerPrepareReboot(t, fs, saved.Record) |
| 550 | afterRecord, err := state.Read("retirement-" + saved.Record.Authorization.ID + ".json") |
| 551 | if err != nil || !bytes.Equal(beforeRecord, afterRecord) { |
| 552 | t.Fatal("record inspection changed history", err) |
| 553 | } |
| 554 | t.Log("persistent parent/root/source/event/file handles match across actual reboot; retirement authority remains unchanged") |
| 555 | if err := fs.CheckParked(&saved.Record); !errors.Is(err, hostbox.ErrChanged) { |
| 556 | t.Fatalf("old mount authority reused: %v", err) |
| 557 | } |
| 558 | r := saved.Record |
| 559 | r.Deleting, r.Cause = true, "expiry" |
| 560 | writes := 0 |
| 561 | if err := fs.Delete(&r, func() error { writes++; return nil }); !errors.Is(err, hostbox.ErrChanged) || writes != 0 { |
| 562 | t.Fatalf("stale mount deleted data: writes=%d err=%v", writes, err) |
| 563 | } |
| 564 | after, err := state.Read("namespace.json") |
| 565 | if err != nil || !bytes.Equal(before, after) { |
| 566 | t.Fatal("refusal changed namespace authority", err) |
| 567 | } |
| 568 | for suffix, want := range map[string]string{"": "database", "-wal": "WAL", "-shm": "SHM"} { |
| 569 | path := filepath.Join(data, "instances", ".serving-parked", r.Authorization.ID, r.Authorization.Successor.Host+".db"+suffix) |
| 570 | got, err := os.ReadFile(path) |
| 571 | if err != nil || string(got) != want { |
| 572 | t.Fatalf("reboot lost retained %s: %q %v", suffix, got, err) |
| 573 | } |
| 574 | } |
| 575 | |
| 576 | // The kernel need not recycle the number in this run. Normalize only this |
| 577 | // fixture's saved device and mount numbers to the actual new mount, leaving the real |
| 578 | // pre-reboot BootID untouched. A mount-number-only guard would now accept it. |
| 579 | parent, actual, err := fs.parent() |
| 580 | if err != nil { |
| 581 | t.Fatal(err) |
| 582 | } |
| 583 | parent.Close() |
| 584 | var namespace servingParkNamespace |
| 585 | if servingParkDecode(before, &namespace) != nil || namespace.BootID != saved.Boot { |
| 586 | t.Fatal("phase 1 did not persist its actual boot") |
| 587 | } |
| 588 | if actual.Inode != saved.Record.Source.Parent.Inode { |
| 589 | t.Fatal("reboot fixture changed its underlying parent identity") |
| 590 | } |
| 591 | namespace.Parent.Mount, namespace.Root.Mount = actual.Mount, actual.Mount |
| 592 | namespace.Parent.Device, namespace.Root.Device = actual.Device, actual.Device |
| 593 | if err := (&servingParkLedger{state: state}).write("namespace.json", namespace); err != nil { |
| 594 | t.Fatal(err) |
| 595 | } |
| 596 | r = saved.Record |
| 597 | r.Authorization.Directory.Mount = actual.Mount |
| 598 | r.Source.Directory.Mount, r.Source.Parent.Mount = actual.Mount, actual.Mount |
| 599 | r.Event.Mount, r.Namespace.Mount = actual.Mount, actual.Mount |
| 600 | r.Authorization.Directory.Device = actual.Device |
| 601 | r.Source.Directory.Device, r.Source.Parent.Device = actual.Device, actual.Device |
| 602 | r.Event.Device, r.Namespace.Device = actual.Device, actual.Device |
| 603 | for i := range r.Source.Files { |
| 604 | if r.Source.Files[i].Present { |
| 605 | r.Source.Files[i].Mount = actual.Mount |
| 606 | r.Source.Files[i].Device = actual.Device |
| 607 | } |
| 608 | } |
| 609 | before, err = state.Read("namespace.json") |
| 610 | if err != nil { |
| 611 | t.Fatal(err) |
| 612 | } |
| 613 | if err := fs.CheckParked(&r); !errors.Is(err, hostbox.ErrChanged) { |
| 614 | t.Fatal("recycled mount number accepted across actual reboot", err) |
| 615 | } |
| 616 | writes = 0 |
| 617 | if err := fs.Delete(&r, func() error { writes++; return nil }); !errors.Is(err, hostbox.ErrChanged) || writes != 0 { |
| 618 | t.Fatal("recycled mount number authorized deletion", writes, err) |
| 619 | } |
| 620 | after, err = state.Read("namespace.json") |
| 621 | if err != nil || !bytes.Equal(before, after) { |
| 622 | t.Fatal("recycled-number refusal rewrote authority", err) |
| 623 | } |
| 624 | for suffix, want := range map[string]string{"": "database", "-wal": "WAL", "-shm": "SHM"} { |
| 625 | body, err := os.ReadFile(filepath.Join(data, "instances", ".serving-parked", r.Authorization.ID, r.Authorization.Successor.Host+".db"+suffix)) |
| 626 | if err != nil || string(body) != want { |
| 627 | t.Fatal("recycled-number refusal changed parked data", suffix, err) |
| 628 | } |
| 629 | } |
| 630 | // Read-only control: with only the fixture boot changed, every physical |
| 631 | // identity must pass. Otherwise the previous refusals could have another |
| 632 | // cause and would not witness the boot boundary. Never delete this control. |
| 633 | namespace.BootID = strings.TrimSpace(string(boot)) |
| 634 | if err := (&servingParkLedger{state: state}).write("namespace.json", namespace); err != nil { |
| 635 | t.Fatal(err) |
| 636 | } |
| 637 | r.Source.BootID = strings.TrimSpace(string(boot)) |
| 638 | controlErr := fs.CheckParked(&r) |
| 639 | if err := state.Write("namespace.json", before); err != nil { |
| 640 | t.Fatal(err) |
| 641 | } |
| 642 | if controlErr != nil { |
| 643 | t.Fatal("numeric-reuse control has another identity mismatch", controlErr) |
| 644 | } |
| 645 | if err := state.Write("namespace.json", originalNamespace); err != nil { |
| 646 | t.Fatal(err) |
| 647 | } |
| 648 | ledger.fs, ledger.now = fs, time.Now |
| 649 | observe, err := fs.inspectRebootRecord(t.Context(), saved.Record.Authorization.ID) |
| 650 | if err != nil { |
| 651 | t.Fatal(err) |
| 652 | } |
| 653 | a := saved.Record.activeAuthorization() |
| 654 | active := &servingParkRebootAuthority{ClaimETag: "runner-claim", Actor: hostbox.Actor{Box: a.Successor.Box, Region: a.Successor.Region, Enrollment: "runner-enrollment", Revision: 1, Binding: "1", Eligible: true}, Directory: observe.Source, Tags: a.TagsFrom} |
| 655 | if err := ledger.withNamespace(t.Context(), func() error { return ledger.commitRebootLocked(t.Context(), saved.Record, active) }); err != nil { |
| 656 | t.Fatal("commit after actual reboot", err) |
| 657 | } |
| 658 | current, err := ledger.read(a.ID) |
| 659 | if err != nil { |
| 660 | t.Fatal(err) |
| 661 | } |
| 662 | if current.Source.BootID != strings.TrimSpace(string(boot)) || current.Source.Parent.Mount != actual.Mount { |
| 663 | t.Fatal("wrong committed topology") |
| 664 | } |
| 665 | if err := fs.CheckParked(¤t); err != nil { |
| 666 | t.Fatal("current park check", err) |
| 667 | } |
| 668 | if err := ledger.completeLocked(t.Context(), a.ID, a.Successor); err != nil { |
| 669 | t.Fatal("complete", err) |
| 670 | } |
| 671 | current, err = ledger.read(a.ID) |
| 672 | if err != nil { |
| 673 | t.Fatal(err) |
| 674 | } |
| 675 | ledger.now = func() time.Time { return current.ReapAfter } |
| 676 | reapErr := ledger.reapLocked(t.Context(), a.ID) |
| 677 | if strings.Contains(t.Name(), "GrownDisk") { |
| 678 | if !errors.Is(reapErr, hostbox.ErrChanged) || current.Source.Volume != "" { |
| 679 | t.Fatal("serial-less loop gained deletion authority", reapErr) |
| 680 | } |
| 681 | for _, item := range current.Source.Files { |
| 682 | if item.Present { |
| 683 | if _, err := os.Stat(filepath.Join(data, "instances", ".serving-parked", a.ID, item.Name)); err != nil { |
| 684 | t.Fatal("serial-less park lost bytes", err) |
| 685 | } |
| 686 | } |
| 687 | } |
| 688 | t.Log("grown filesystem recovered and completed; no independent loop volume serial, so reaping retained every artifact") |
| 689 | } else if reapErr != nil { |
| 690 | t.Fatal("verified reap after reboot", reapErr) |
| 691 | } |
| 692 | historical, err := ledger.readHistorical(a.ID) |
| 693 | if err != nil || historical.Authorization != saved.Record.Authorization || historical.Source.BootID != saved.Record.Source.BootID || historical.ReapAfter != saved.Record.ReapAfter { |
| 694 | t.Fatal("changed history", err) |
| 695 | } |
| 696 | if !strings.Contains(t.Name(), "GrownDisk") { |
| 697 | t.Log("committed current mount identities and reaped the exact parked artifacts after an actual reboot") |
| 698 | } |
| 699 | |
| 700 | } |
| 701 | |
| 702 | // Installation, census and stopped-writer permission remain fixture evidence. |
| 703 | // This checks durable preparation and replay under real unit/mount constraints. |
| 704 | func servingRunnerPrepareReboot(t *testing.T, fs *servingParkFS, r servingParkRecord) { |
| 705 | t.Helper() |
| 706 | ledger := servingParkLedger{state: fs.state, fs: fs, now: time.Now} |
| 707 | observe, err := fs.inspectRebootRecord(t.Context(), r.Authorization.ID) |
| 708 | if err != nil { |
| 709 | t.Fatal(err) |
| 710 | } |
| 711 | a := r.activeAuthorization() |
| 712 | active := &servingParkRebootAuthority{ClaimETag: "runner-claim", Actor: hostbox.Actor{Box: a.Successor.Box, Region: a.Successor.Region, Enrollment: "runner-enrollment", Revision: 1, Binding: "1", Eligible: true}, Directory: observe.Source, Tags: a.TagsFrom} |
| 713 | prepare := func() error { |
| 714 | return ledger.withNamespace(t.Context(), func() error { return ledger.prepareRebootLocked(t.Context(), r, active) }) |
| 715 | } |
| 716 | if err := prepare(); err != nil { |
| 717 | t.Fatal("durable reboot preparation", err) |
| 718 | } |
| 719 | name := servingNamespacePreparationName(a.ID, observe.Topology) |
| 720 | before, err := fs.state.Read(name) |
| 721 | if err != nil { |
| 722 | t.Fatal(err) |
| 723 | } |
| 724 | if err := prepare(); err != nil { |
| 725 | t.Fatal("durable reboot preparation replay", err) |
| 726 | } |
| 727 | after, err := fs.state.Read(name) |
| 728 | if err != nil || !bytes.Equal(before, after) { |
| 729 | t.Fatal("preparation replay changed evidence", err) |
| 730 | } |
| 731 | if err := fs.recoverRecordedNamespace(t.Context()); err == nil { |
| 732 | t.Fatal("preparation granted old-boot namespace authority") |
| 733 | } |
| 734 | } |
| 735 | |
| 736 | // Run the existing privileged acceptance batch and the actual reboot in one |
| 737 | // runner allocation. Each batch test still runs by its original top-level name. |
| 738 | func TestRootRebootServingParkAcceptancePhase1(t *testing.T) { |
| 739 | servingRootRunner(t) |
| 740 | binary, err := os.Executable() |
| 741 | if err != nil { |
| 742 | t.Fatal(err) |
| 743 | } |
| 744 | pattern := "^(TestRootServingParkAuthenticatedPeer|TestRootServingParkAuthenticatedHandover|TestRootServingParkStoppedWriterHandover|TestRootServingParkColdHandoverBoundary|TestRootServingParkNestedBindRestore|TestRootServingParkInstalledUnit|TestRootServingParkQuarantine|TestRootServingNamespaceStartup|TestRootServingParkUnrecordedReplacement|TestRootServingNamespaceUnrecordedStartup|TestRootServingParkDisplacedRoot|TestRootServingParkCollection)$" |
| 745 | cmd := exec.CommandContext(t.Context(), binary, "-test.run="+pattern, "-test.v", "-test.count=1", "-test.timeout=8m") |
| 746 | cmd.Env = os.Environ() |
| 747 | out, err := cmd.CombinedOutput() |
| 748 | t.Log(string(out)) |
| 749 | if err != nil { |
| 750 | t.Error(err) |
| 751 | } |
| 752 | TestRootRebootServingParkChangedMountPhase1(t) |
| 753 | } |
| 754 | |
| 755 | func TestRootRebootServingParkAcceptancePhase2(t *testing.T) { |
| 756 | TestRootRebootServingParkChangedMountPhase2(t) |
| 757 | } |
| 758 | |
| 759 | func servingRunnerTopologyDiagnostic(t *testing.T, fs *servingParkFS) { |
| 760 | t.Helper() |
| 761 | body, err := fs.state.Read("namespace.json") |
| 762 | if err != nil { |
| 763 | t.Fatal(err) |
| 764 | } |
| 765 | t.Logf("recorded topology: %s", body) |
| 766 | parent, pid, err := fs.parent() |
| 767 | if err != nil { |
| 768 | t.Fatal(err) |
| 769 | } |
| 770 | defer parent.Close() |
| 771 | root, rid, err := fs.privateDir(parent, ".serving-parked") |
| 772 | if err != nil { |
| 773 | t.Fatal(err) |
| 774 | } |
| 775 | defer root.Close() |
| 776 | boot, err := fs.currentBoot() |
| 777 | if err != nil { |
| 778 | t.Fatal(err) |
| 779 | } |
| 780 | topology, err := fs.captureTopology(parent, root, servingParkNamespace{Parent: pid, Root: rid, BootID: boot}) |
| 781 | if err != nil { |
| 782 | t.Fatal(err) |
| 783 | } |
| 784 | t.Logf("current topology: boot=%s parent=%+v root=%+v persistent=%+v", boot, pid, rid, topology) |
| 785 | out, err := exec.CommandContext(t.Context(), "findmnt", "-n", "-o", "UUID,MAJ:MIN,SOURCE", "-T", filepath.Join(fs.data, "instances")).CombinedOutput() |
| 786 | if err != nil { |
| 787 | t.Fatal(err, string(out)) |
| 788 | } |
| 789 | t.Logf("current filesystem UUID/device/source: %s", out) |
| 790 | } |
| 791 | |
| 792 | func TestRootFilesSeedRealOriginPreparation(t *testing.T) { |
| 793 | runRootFilesSeedRealOriginPreparation(t, false) |
| 794 | } |
| 795 | func TestRootFilesSeedRealOriginTenant(t *testing.T) { runRootFilesSeedRealOriginPreparation(t, true) } |
| 796 | func TestRootFilesSeedWholeExecutorKilledRerun(t *testing.T) { |
| 797 | runRootFilesSeedRealOriginPreparation(t, true) |
| 798 | } |
| 799 | func runRootFilesSeedRealOriginPreparation(t *testing.T, live bool) { |
| 800 | servingRootRunner(t) |
| 801 | if os.Getenv("PLAN14_ROOT_FIXTURE") == t.Name() { |
| 802 | if err := unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, ""); err != nil { |
| 803 | t.Fatal(err) |
| 804 | } |
| 805 | if !plan14RootFixture(t) { |
| 806 | t.Fatal("root child did not enter its scratch root") |
| 807 | } |
| 808 | if live { |
| 809 | runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{rootFilesSeedWitnessFor(t, "/tenant.test")}) |
| 810 | } else { |
| 811 | runFilesSeedOrdinaryPreparation(t, "real-seed", "same-controller", true) |
| 812 | } |
| 813 | return |
| 814 | } |
| 815 | root := t.TempDir() |
| 816 | for _, name := range []string{"scratch", "proc", "foreign-dir"} { |
| 817 | if err := os.Mkdir(filepath.Join(root, name), 0755); err != nil { |
| 818 | t.Fatal(err) |
| 819 | } |
| 820 | } |
| 821 | if err := os.WriteFile(filepath.Join(root, "foreign"), nil, 0644); err != nil { |
| 822 | t.Fatal(err) |
| 823 | } |
| 824 | binary, err := os.Executable() |
| 825 | if err != nil { |
| 826 | t.Fatal(err) |
| 827 | } |
| 828 | if live { |
| 829 | body, err := os.ReadFile(binary) |
| 830 | if err != nil { |
| 831 | t.Fatal(err) |
| 832 | } |
| 833 | if err := os.WriteFile(filepath.Join(root, "tenant.test"), body, 0755); err != nil { |
| 834 | t.Fatal(err) |
| 835 | } |
| 836 | } |
| 837 | cmd := exec.CommandContext(t.Context(), binary, "-test.run=^"+t.Name()+"$", "-test.v", "-test.timeout=3m") |
| 838 | cmd.Env = append(os.Environ(), "PLAN14_ROOT_FIXTURE="+t.Name(), "PLAN14_ROOT_DIR="+root) |
| 839 | cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: unix.CLONE_NEWNS} |
| 840 | if out, err := cmd.CombinedOutput(); err != nil { |
| 841 | t.Fatalf("real-root mounted restore: %v\n%s", err, out) |
| 842 | } else { |
| 843 | t.Log(string(out)) |
| 844 | } |
| 845 | } |
| 846 | |
amadan