| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "crypto/ed25519" |
| 9 | "encoding/base64" |
| 10 | "encoding/json" |
| 11 | "errors" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "path/filepath" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/hostops" |
| 22 | "github.com/carlosframework/platform/internal/moves" |
| 23 | "github.com/carlosframework/platform/internal/registry" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 26 | "github.com/carlosframework/platform/internal/rootstate" |
| 27 | ) |
| 28 | |
| 29 | func TestPlan14RootMigrationBootstrapRootRunner(t *testing.T) { |
| 30 | testRootMigrationBootstrap(t, false, false) |
| 31 | } |
| 32 | func TestPlan14RootMigrationHistoricRootRunner(t *testing.T) { |
| 33 | testRootMigrationBootstrap(t, true, false) |
| 34 | } |
| 35 | func TestPlan14RootMigrationFreshStopRootRunner(t *testing.T) { |
| 36 | testRootMigrationBootstrap(t, true, true) |
| 37 | } |
| 38 | func testRootMigrationBootstrap(t *testing.T, historic, fresh bool) { |
| 39 | if os.Getenv("PLAN14_P3_BOOTSTRAP_CHILD") == "" { |
| 40 | if !historic { |
| 41 | generationRunnerRegistryUser(t) |
| 42 | } |
| 43 | for _, name := range rootMigrationOfflineUnits { |
| 44 | path := filepath.Join("/run/systemd/system", name) |
| 45 | if err := os.Symlink("/dev/null", path); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | t.Cleanup(func() { os.Remove(path); exec.Command("systemctl", "daemon-reload").Run() }) |
| 49 | } |
| 50 | unit := "/usr/lib/systemd/system/carlos-edge.service" |
| 51 | for _, p := range []string{unit, "/etc/systemd/system/carlos-edge.service", "/run/systemd/system/carlos-edge.service"} { |
| 52 | if _, err := os.Lstat(p); !os.IsNotExist(err) { |
| 53 | t.Fatal("existing unit", p, err) |
| 54 | } |
| 55 | } |
| 56 | if err := os.WriteFile(unit, []byte("[Service]\nType=oneshot\nExecStart=/usr/bin/true\n[Install]\nWantedBy=multi-user.target\n"), 0644); err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | t.Cleanup(func() { |
| 60 | exec.Command("systemctl", "unmask", "--runtime", "carlos-edge.service").Run() |
| 61 | os.Remove(unit) |
| 62 | exec.Command("systemctl", "daemon-reload").Run() |
| 63 | }) |
| 64 | if out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput(); err != nil { |
| 65 | t.Fatal(err, string(out)) |
| 66 | } |
| 67 | if out, err := exec.Command("systemctl", "mask", "--runtime", "carlos-edge.service").CombinedOutput(); err != nil { |
| 68 | t.Fatal(err, string(out)) |
| 69 | } |
| 70 | |
| 71 | if err := verifyOfflineRootMigration(t.Context()); err != nil { |
| 72 | t.Fatal("offline root units", err) |
| 73 | } |
| 74 | properties := map[string]string{} |
| 75 | for _, name := range rootMigrationOfflineUnits { |
| 76 | out, err := claimSystemctl(t.Context(), "show", name, "--property=LoadState,ActiveState", "--no-pager") |
| 77 | if err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | properties[name] = string(out) |
| 81 | if out, err := exec.Command("systemctl", "start", name).CombinedOutput(); err == nil || !bytes.Contains(out, []byte("masked")) { |
| 82 | t.Fatal(name, string(out), err) |
| 83 | } |
| 84 | } |
| 85 | savedProperties, _ := json.Marshal(properties) |
| 86 | oldOwner := claimQuiesceOwner |
| 87 | claimQuiesceOwner = func() (int, int, error) { return 65534, 65534, nil } |
| 88 | defer func() { claimQuiesceOwner = oldOwner }() |
| 89 | release, err := quiesceLegacyClaimJournals(t.Context()) |
| 90 | if err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | defer release() |
| 94 | // The existing generation census supports a controlled all-root namespace; |
| 95 | // E owns the wider running-tenant/foreign-namespace production integration. |
| 96 | name := "TestPlan14RootMigrationBootstrapRootRunner" |
| 97 | if historic { |
| 98 | name = "TestPlan14RootMigrationHistoricRootRunner" |
| 99 | } |
| 100 | if fresh { |
| 101 | name = "TestPlan14RootMigrationFreshStopRootRunner" |
| 102 | } |
| 103 | if t.Name() == "TestRootRebootPlan14MigrationInstalledPhase1" || t.Name() == "TestPlan14InitialMigrationExclusionRootRunner" || t.Name() == "TestPlan14RetiredMigrationPayloadRootRunner" { |
| 104 | name = t.Name() |
| 105 | } |
| 106 | cmd := exec.CommandContext(t.Context(), "unshare", "--mount", "--pid", "--fork", "--mount-proc", os.Args[0], "-test.run=^"+name+"$", "-test.v") |
| 107 | cmd.Env = append(os.Environ(), "PLAN14_P3_BOOTSTRAP_CHILD=1", "PLAN14_OFFLINE_UNITS="+string(savedProperties)) |
| 108 | if out, err := cmd.CombinedOutput(); err != nil { |
| 109 | t.Fatal(err, string(out)) |
| 110 | } else { |
| 111 | t.Log(string(out)) |
| 112 | } |
| 113 | return |
| 114 | } |
| 115 | for _, path := range []string{"/run/carlos", "/run/carlos/hostguard"} { |
| 116 | err := os.Mkdir(path, 0755) |
| 117 | if err == nil { |
| 118 | t.Cleanup(func() { os.RemoveAll(path) }) |
| 119 | } else if !os.IsExist(err) { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | } |
| 123 | claimSystemctl = func(ctx context.Context, args ...string) ([]byte, error) { |
| 124 | var properties map[string]string |
| 125 | if json.Unmarshal([]byte(os.Getenv("PLAN14_OFFLINE_UNITS")), &properties) != nil || len(args) < 2 || args[0] != "show" || properties[args[1]] == "" { |
| 126 | return nil, os.ErrPermission |
| 127 | } |
| 128 | return []byte(properties[args[1]]), nil |
| 129 | } |
| 130 | rootMigrationQuiesce = func(context.Context) (func(), error) { return func() {}, nil } |
| 131 | for _, path := range []string{"/var/lib/carlos-claims", "/var/lib/carlos-release"} { |
| 132 | if err := os.Mkdir(path, 0700); err != nil { |
| 133 | t.Fatal("existing authority", path, err) |
| 134 | } |
| 135 | t.Cleanup(func() { |
| 136 | if err := os.RemoveAll(path); err != nil { |
| 137 | t.Error(err) |
| 138 | } |
| 139 | }) |
| 140 | } |
| 141 | scratch := generationRootRunnerDir(t) |
| 142 | t.Setenv("TMPDIR", scratch) |
| 143 | store, r, owners, _ := fix4JournalFixture(t) |
| 144 | // Use the production private authority, replacing only the older fixture's opener. |
| 145 | openClaimRootAuthority = func() (rootstate.Store, error) { |
| 146 | return rootstate.Open("/var/lib/carlos-claims/authority/fences", true, false) |
| 147 | } |
| 148 | actor, err := owners.Identity.Verify(t.Context()) |
| 149 | if err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | for path := r.OptDir; path != scratch; path = filepath.Dir(path) { |
| 153 | if err := os.Chmod(path, 0755); err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | } |
| 157 | if err := os.Mkdir(filepath.Join(r.OptDir, "versions"), 0755); err != nil { |
| 158 | t.Fatal(err) |
| 159 | } |
| 160 | t.Setenv("CARLOS_OPT_DIR", r.OptDir) |
| 161 | t.Setenv("CARLOS_DATA_DIR", r.DataDir) |
| 162 | t.Setenv("CARLOS_DEPLOYMENT_BUCKET", actor.ControlBucket) |
| 163 | key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) |
| 164 | t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey))) |
| 165 | plan := hostops.BackfillPlan{V: 1, ID: "migration", Phase: "collecting", DirectoryRevision: 1, CreatedAt: time.Now(), Hosts: map[string]hostops.BackfillHost{}} |
| 166 | var target hostguard.FenceTarget |
| 167 | var historicBytes []byte |
| 168 | if historic { |
| 169 | target, historicBytes = migrationHistoricFixture(t, store, r, owners.ClaimOwnerBindings, actor, &plan) |
| 170 | } |
| 171 | raw, err := release.SignRootEnvelope(release.RootSigner(key), plan) |
| 172 | if err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | if _, err := store.Put(t.Context(), hostops.BackfillKey, raw, ""); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | id := rootFloorIdentity{Deployment: actor.ControlBucket, Box: actor.Box, Enrollment: actor.Enrollment} |
| 179 | floors, err := openRootMovesFloors(r.OptDir, id, true) |
| 180 | if err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | defer floors.close() |
| 184 | floor := moves.Floor{Min: 1, MinRelease: "v2.0.0"} |
| 185 | if err := floors.reconstruct(t.Context(), rootFloorReconstruction{Identity: id, Floor: &floor, Source: "enrolled root history", Digest: floorTestDigest, Authorization: floorTestAuthorization(t, floors)}); err != nil { |
| 186 | t.Fatal(err) |
| 187 | } |
| 188 | rootFloorRunnerBinary(t, r.OptDir, systemBinaryPath(r.OptDir), "v2.0.0", key) |
| 189 | rootFloorRunnerBinary(t, r.OptDir, filepath.Join(systemGuardDir(r.OptDir), "carlos"), "v2.0.0", key) |
| 190 | journal, err := hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") |
| 191 | if err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | defer journal.Close() |
| 195 | service, err := hostbox.NewJournalService(journal) |
| 196 | if err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | |
| 200 | if t.Name() == "TestPlan14InitialMigrationExclusionRootRunner" { |
| 201 | plan14InitialMigrationExclusion(t, store, r, owners.ClaimOwnerBindings, actor, service, journal) |
| 202 | } |
| 203 | observedAfter := time.Now().UTC() |
| 204 | oldOwner := claimQuiesceOwner |
| 205 | claimQuiesceOwner = func() (int, int, error) { return 65534, 65534, nil } |
| 206 | defer func() { claimQuiesceOwner = oldOwner }() |
| 207 | if historic { |
| 208 | held, err := os.Open(filepath.Join(r.DataDir, "historic-data")) |
| 209 | if err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | err = migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal) |
| 213 | held.Close() |
| 214 | if err == nil { |
| 215 | t.Fatal("migration completed with an open historic payload") |
| 216 | } |
| 217 | if fresh { |
| 218 | key := stopEvidenceKey(target.Host, "", actor.Box, "historic-stop") |
| 219 | if err := store.Delete(t.Context(), key); err != nil { |
| 220 | t.Fatal(err) |
| 221 | } |
| 222 | if err := store.Delete(t.Context(), key+".sig"); err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | held, err := os.Open(filepath.Join(r.DataDir, "historic-data")) |
| 226 | if err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | err = migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal) |
| 230 | held.Close() |
| 231 | if err == nil { |
| 232 | t.Fatal("fresh stop accepted held payload") |
| 233 | } |
| 234 | |
| 235 | } |
| 236 | if err := r.Registry.UpsertTimer(registry.Timer{Host: target.Host, Name: "unrecorded", Kind: "cron", Spec: "0 0 * * *", Path: "/extra"}); err != nil { |
| 237 | t.Fatal(err) |
| 238 | } |
| 239 | if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err == nil { |
| 240 | t.Fatal("unrecorded preclaim timer accepted") |
| 241 | } |
| 242 | if err := r.Registry.DeleteTimer(target.Host, "unrecorded"); err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | } |
| 246 | if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { |
| 247 | t.Fatal("production bootstrap", err) |
| 248 | } |
| 249 | if !historic { |
| 250 | uid, gid := generationRunnerRegistryUser(t) |
| 251 | for path := r.DataDir; path != scratch; path = filepath.Dir(path) { |
| 252 | if err := os.Chmod(path, 0755); err != nil { |
| 253 | t.Fatal(err) |
| 254 | } |
| 255 | } |
| 256 | if err := os.Chown(r.DataDir, uid, gid); err != nil { |
| 257 | t.Fatal(err) |
| 258 | } |
| 259 | reg, err := openRegistryChild(filepath.Join(r.DataDir, "registry.db"), uid, gid) |
| 260 | if err != nil { |
| 261 | t.Fatal(err) |
| 262 | } |
| 263 | if err := reg.Close(); err != nil { |
| 264 | t.Fatal(err) |
| 265 | } |
| 266 | if err := withClaimRootRelease(t.Context(), true, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, true, false) }, func() error { return rootFloorDowngradeIdle(t.Context()) }); err != nil { |
| 267 | t.Fatal("offline floor idle", err) |
| 268 | } |
| 269 | } |
| 270 | state, err := rootstate.Open(rootparticipation.Namespace, false, false) |
| 271 | if err != nil { |
| 272 | t.Fatal(err) |
| 273 | } |
| 274 | defer state.Close() |
| 275 | c, err := (rootparticipation.Ledger{State: state}).Census() |
| 276 | if err != nil || !c.Complete || c.Box != actor.Box || c.NeverEnrolled { |
| 277 | t.Fatal("invalid root census", c, err) |
| 278 | } |
| 279 | keys, err := store.List(t.Context(), "control/host-census/") |
| 280 | if err != nil || len(keys) != 1 { |
| 281 | t.Fatal(keys, err) |
| 282 | } |
| 283 | body, _, err := store.Get(t.Context(), keys[0]) |
| 284 | if err != nil { |
| 285 | t.Fatal(err) |
| 286 | } |
| 287 | var envelope hostops.CensusEnvelope |
| 288 | if json.Unmarshal(body, &envelope) != nil || envelope.Report.Seeds == nil { |
| 289 | t.Fatal("no Seeds in actual census") |
| 290 | } |
| 291 | if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { |
| 292 | t.Fatal("completed migration replay", err) |
| 293 | } |
| 294 | if err := verifyRootMigrationStartup(t.Context(), id, service.Inventory); err != nil { |
| 295 | t.Fatal("production startup", err) |
| 296 | } |
| 297 | if historic { |
| 298 | fences, err := openClaimRootAuthority() |
| 299 | if err != nil { |
| 300 | t.Fatal(err) |
| 301 | } |
| 302 | defer fences.Close() |
| 303 | root := hostguard.NewRootAuthority(fences) |
| 304 | g := hostguard.Guard{Root: root, DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}} |
| 305 | if fresh { |
| 306 | view, err := root.View(t.Context(), target.Host, hostguard.Captured{Incarnation: target.Incarnation, Stamp: target.Stamp}) |
| 307 | if err != nil || len(view.Receipts) != 1 { |
| 308 | t.Fatal("fresh receipt", view, err) |
| 309 | } |
| 310 | target = view.Receipts[0].Target |
| 311 | } |
| 312 | receipt, err := g.FenceReceiptFor(t.Context(), target) |
| 313 | if err != nil || !fresh && !bytes.Equal(receipt.Bytes, historicBytes) { |
| 314 | t.Fatal("historic bytes changed", err) |
| 315 | } |
| 316 | if fresh { |
| 317 | if !receipt.EvidenceAt.After(observedAfter) { |
| 318 | t.Fatal("fresh stop inherited historic time", receipt.EvidenceAt) |
| 319 | } |
| 320 | var body StopReceipt |
| 321 | if json.Unmarshal(receipt.Bytes, &body) != nil || body.RequestLimitNS != int64(hostbox.RequestLimit) || body.ObservedBootID == target.BootID { |
| 322 | t.Fatal("fresh observation missing horizon or current boot") |
| 323 | } |
| 324 | historicBytes = receipt.Bytes |
| 325 | } |
| 326 | view, err := root.View(t.Context(), target.Host, hostguard.Captured{Incarnation: target.Incarnation, Stamp: target.Stamp}) |
| 327 | if err != nil || view.Installation.Artifacts.Timers != "refresh" || len(view.MigrationDenies) != 1 { |
| 328 | t.Fatal("historic root inventory", view, err) |
| 329 | } |
| 330 | legacy := filepath.Join(r.DataDir, ".hostclaims") |
| 331 | if err := os.Rename(legacy, filepath.Join(scratch, "legacy-moved")); err != nil { |
| 332 | t.Fatal(err) |
| 333 | } |
| 334 | if err := os.Mkdir(legacy, 0700); err != nil { |
| 335 | t.Fatal(err) |
| 336 | } |
| 337 | if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { |
| 338 | t.Fatal("legacy rename retry", err) |
| 339 | } |
| 340 | if err := os.RemoveAll(filepath.Join(r.DataDir, ".hostclaims")); err != nil { |
| 341 | t.Fatal(err) |
| 342 | } |
| 343 | if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { |
| 344 | t.Fatal("legacy erased retry", err) |
| 345 | } |
| 346 | again, err := g.FenceReceiptFor(t.Context(), target) |
| 347 | if err != nil || !bytes.Equal(again.Bytes, historicBytes) || !again.EvidenceAt.Equal(receipt.EvidenceAt) { |
| 348 | t.Fatal("historic replay renewed evidence", err) |
| 349 | } |
| 350 | captured := hostguard.Captured{Incarnation: target.Incarnation, Stamp: target.Stamp} |
| 351 | effects := 0 |
| 352 | launch := func() error { effects++; return exec.Command("/usr/bin/true").Run() } |
| 353 | allow := func(*hostclaim.Claim) error { return nil } |
| 354 | if err := g.With(t.Context(), target.Host, captured, allow, launch); err == nil || effects != 0 { |
| 355 | t.Fatal("old writer admitted", err, effects) |
| 356 | } |
| 357 | observed, err := hostclaim.Read(t.Context(), store, target.Host) |
| 358 | if err != nil { |
| 359 | t.Fatal(err) |
| 360 | } |
| 361 | observed.Claim.Holder.Session++ |
| 362 | observed.Claim.Gen++ |
| 363 | raw, err := json.Marshal(observed.Claim) |
| 364 | if err != nil { |
| 365 | t.Fatal(err) |
| 366 | } |
| 367 | key, _ := hostclaim.Key(target.Host) |
| 368 | if _, err = store.Put(t.Context(), key, raw, observed.ETag); err != nil { |
| 369 | t.Fatal(err) |
| 370 | } |
| 371 | if err := g.With(t.Context(), target.Host, captured, allow, launch); err != nil || effects != 1 { |
| 372 | t.Fatal("successor writer", err, effects) |
| 373 | } |
| 374 | if t.Name() == "TestPlan14RetiredMigrationPayloadRootRunner" { |
| 375 | if err := g.RetireInstallationWith(t.Context(), target.Host, captured, allow, func() error { return os.ErrPermission }); !errors.Is(err, os.ErrPermission) { |
| 376 | t.Fatal("retirement interrupt", err) |
| 377 | } |
| 378 | fences.Close() |
| 379 | if err := verifyRootMigrationStartup(t.Context(), id, service.Inventory); err != nil { |
| 380 | t.Fatal("retired migration startup", err) |
| 381 | } |
| 382 | read := func(ctx context.Context) (generationInventory, error) { |
| 383 | return generationInventoryAt(ctx, rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", "/var/lib/carlos-claims/journals.db") |
| 384 | } |
| 385 | snapshot, err := read(t.Context()) |
| 386 | if err != nil { |
| 387 | t.Fatal(err) |
| 388 | } |
| 389 | if len(snapshot.Installations) != 1 || snapshot.Installations[0].Installation.Stamp != captured.Stamp || len(snapshot.Payloads) == 0 || len(snapshot.Payloads[0].Inventory.Entries) == 0 { |
| 390 | t.Fatal("retired payload lost", snapshot) |
| 391 | } |
| 392 | verifier := generationReconciler{read: read, kernel: productionGenerationKernelCensus} |
| 393 | if _, err := verifier.reconcile(t.Context(), nil); err != nil { |
| 394 | t.Fatal("retired generation verification", err) |
| 395 | } |
| 396 | if _, err := os.Stat(filepath.Join(r.DataDir, "historic-data")); err != nil { |
| 397 | t.Fatal("payload removed", err) |
| 398 | } |
| 399 | } |
| 400 | if t.Name() == "TestRootRebootPlan14MigrationInstalledPhase1" || t.Name() == "TestPlan14InitialMigrationExclusionRootRunner" { |
| 401 | if err := journal.Close(); err != nil { |
| 402 | t.Fatal(err) |
| 403 | } |
| 404 | saveMigrationInstalled(t, store, r, scratch, target, receipt.Bytes) |
| 405 | } |
| 406 | } |
| 407 | |
| 408 | } |
| 409 | |
amadan