| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "strconv" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/hostguard" |
| 20 | "github.com/carlosframework/platform/internal/instancectl" |
| 21 | "github.com/carlosframework/platform/internal/provision" |
| 22 | "github.com/carlosframework/platform/internal/registry" |
| 23 | "github.com/carlosframework/platform/internal/release" |
| 24 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 25 | "github.com/carlosframework/platform/internal/rootstate" |
| 26 | ) |
| 27 | |
| 28 | func TestPlan14GenerationExactRetirementRootRunner(t *testing.T) { |
| 29 | const childKey = "PLAN14_E35_EXACT_RETIREMENT_CHILD" |
| 30 | dir := os.Getenv(childKey) |
| 31 | if dir == "" { |
| 32 | dir = generationRootRunnerDir(t) |
| 33 | generationRunnerRegistryUser(t) |
| 34 | binary, err := os.Executable() |
| 35 | if err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | cmd := exec.CommandContext(t.Context(), "unshare", "--mount", "--pid", "--fork", "--mount-proc", binary, "-test.run=^TestPlan14GenerationExactRetirementRootRunner$", "-test.v", "-test.timeout=2m") |
| 39 | cmd.Dir = dir |
| 40 | cmd.Env = append(os.Environ(), childKey+"="+dir) |
| 41 | if out, err := cmd.CombinedOutput(); err != nil { |
| 42 | t.Fatalf("isolated exact retirement: %v\n%s", err, out) |
| 43 | } else { |
| 44 | t.Logf("%s", out) |
| 45 | } |
| 46 | return |
| 47 | } |
| 48 | if os.Geteuid() != 0 { |
| 49 | t.Fatal("designated runner must execute as root") |
| 50 | } |
| 51 | t.Setenv("CARLOS_OPT_DIR", dir) |
| 52 | t.Setenv("CARLOS_DATA_DIR", filepath.Join(dir, "data")) |
| 53 | for _, path := range []string{dataDir(), filepath.Join(dir, "host-locks")} { |
| 54 | if err := os.Mkdir(path, 0755); err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | } |
| 58 | ctx := t.Context() |
| 59 | open := func(name string) rootstate.Store { |
| 60 | t.Helper() |
| 61 | s, err := rootstate.Open(filepath.Join(dir, name), true, false) |
| 62 | if err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | t.Cleanup(func() { s.Close() }) |
| 66 | return s |
| 67 | } |
| 68 | fences, census, generations, exclusion := open("fences"), open("census"), open("generations"), open("exclusion") |
| 69 | write := func(s rootstate.Store, name string, value any) { |
| 70 | t.Helper() |
| 71 | raw, err := json.Marshal(value) |
| 72 | if err == nil { |
| 73 | err = s.Write(name, raw) |
| 74 | } |
| 75 | if err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | } |
| 79 | identity := func(path string) rootstate.FileIdentity { |
| 80 | t.Helper() |
| 81 | f, err := os.Open(path) |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | defer f.Close() |
| 86 | id, err := rootstate.Identity(f) |
| 87 | if err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | return id |
| 91 | } |
| 92 | start := func(path string) (int, string, func()) { |
| 93 | t.Helper() |
| 94 | cmd := exec.CommandContext(ctx, path, "90") |
| 95 | cmd.Dir = dir |
| 96 | if err := cmd.Start(); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | joined := false |
| 100 | join := func() { |
| 101 | if !joined { |
| 102 | _ = cmd.Process.Kill() |
| 103 | _ = cmd.Wait() |
| 104 | joined = true |
| 105 | } |
| 106 | } |
| 107 | t.Cleanup(join) |
| 108 | raw, err := generationRead(filepath.Join("/proc", strconv.Itoa(cmd.Process.Pid), "stat")) |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | stamp, err := generationStart(raw) |
| 113 | if err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | return cmd.Process.Pid, stamp, join |
| 117 | } |
| 118 | boot, err := (generationKernel{proc: "/proc"}).boot() |
| 119 | if err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | rec := provision.Record{Schema: 1, Host: filepath.Base(dir) + ".example", Account: "bab", App: "shop", Kind: "service", Addr: "127.0.0.1:8080", Channel: "stable", CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)} |
| 123 | store := release.NewDirStore(filepath.Join(dir, "deployment")) |
| 124 | plan14PutRecord(t, store, rec) |
| 125 | inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) |
| 126 | if err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | in := hostguard.RootInstallation{Artifacts: hostguard.InstallationArtifacts{Schema: 1}, Host: rec.Host, Account: rec.Account, App: rec.App, Record: provision.RecordKey(rec.Account, rec.App, rec.Host), Incarnation: inc, Stamp: 1, Box: "runner", Enrollment: "en", Migration: 1, InventoryRevision: 1} |
| 130 | // Enrollment, installation, journal closure and group identities are fixtures. |
| 131 | // Kernel references, private stores, reconciliation and sweep are production. |
| 132 | write(fences, in.Host+".json", map[string]any{"Schema": 1, "Installation": in, "State": map[string]any{"v": 1, "host": in.Host, "incarnation": inc, "active_stamp": in.Stamp}}) |
| 133 | write(census, "census.json", rootparticipation.Census{Schema: 1, Box: in.Box, Enrollment: in.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) |
| 134 | journalPath := filepath.Join(dir, "journals.db") |
| 135 | journal, err := hostbox.OpenLaunchJournal(journalPath) |
| 136 | if err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | defer journal.Close() |
| 140 | service, err := hostbox.NewJournalService(journal) |
| 141 | if err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | payload := hostbox.StopInventory{Complete: true, Host: in.Host, Incarnation: inc, InstallationStamp: 1, Box: in.Box, Enrollment: in.Enrollment, DirectoryRevision: "1", Entries: []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: in.Host, RowIdentity: "row", Ownership: "owner", Materialization: "materialized"}}} |
| 145 | if err := service.Inventory.WriteStopInventory(ctx, payload); err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | uid, gid := generationRunnerRegistryUser(t) |
| 149 | if err := os.Chown(dataDir(), uid, gid); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | reg, err := openRegistryChild(filepath.Join(dataDir(), "registry.db"), uid, gid) |
| 153 | if err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | defer reg.Close() |
| 157 | row := registry.Route{Host: in.Host, Account: in.Account, App: in.App, Version: "v2", ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: inc, ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"}} |
| 158 | if err := reg.Upsert(row); err != nil { |
| 159 | t.Fatal(err) |
| 160 | } |
| 161 | if err := reg.Close(); err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | rows := fakeLister{routes: []registry.Route{row}} |
| 165 | body, err := os.ReadFile("/bin/sleep") |
| 166 | if err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | sum := sha256.Sum256(body) |
| 170 | for _, version := range []string{"v1", "v2", "v3"} { |
| 171 | if err := release.StageArtifact(dir, in.Account, version, in.App, body); err != nil { |
| 172 | t.Fatal(err) |
| 173 | } |
| 174 | when := time.Now().Add(-48 * time.Hour) |
| 175 | if err := os.Chtimes(release.VersionPath(dir, in.Account, version, in.App), when, when); err != nil { |
| 176 | t.Fatal(err) |
| 177 | } |
| 178 | } |
| 179 | controllerPID, controllerStart, joinController := start("/bin/sleep") |
| 180 | ledger := &release.RootGenerations{Store: generations} |
| 181 | reserve := func(version string) (release.GenerationReservation, hostbox.LaunchReservation) { |
| 182 | t.Helper() |
| 183 | target := hostguard.FenceTarget{Host: in.Host, Incarnation: inc, Stamp: 1, Session: 1, Box: in.Box, Enrollment: in.Enrollment, BootID: boot, ControllerID: version} |
| 184 | path := release.VersionPath(dir, in.Account, version, in.App) |
| 185 | parent := hostbox.LaunchGroup{RelativePath: "fixture-prelaunch", Device: 1, Inode: 1} |
| 186 | if err := journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: controllerPID, ProcessStart: controllerStart, Groups: []hostbox.LaunchGroup{parent}, NoState: true}); err != nil { |
| 187 | t.Fatal(err) |
| 188 | } |
| 189 | b := release.GenerationBinding{Binding: release.HistoryBinding{Host: in.Host, Account: in.Account, App: in.App}, Target: generationOwner(target), Installation: rootHistoryInstallation(in), Record: in.Record, RecordETag: "fixture", Attempt: "main", Inputs: "fixture", Invocation: version, ControllerPID: controllerPID, ControllerStart: controllerStart, Version: version, Digest: hex.EncodeToString(sum[:]), Path: path, Tree: filepath.Dir(path), Executable: identity(path), TreeIdentity: identity(filepath.Dir(path))} |
| 190 | var r release.GenerationReservation |
| 191 | if err := rootstate.WithRelease(ctx, exclusion, func() error { var err error; r, err = ledger.Reserve(b); return err }); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | unit := hostbox.LaunchReservation{ID: b.Attempt, Role: "main", Target: target, Inputs: b.Inputs, InvocationID: b.Invocation, Executable: hostbox.LaunchFile{Device: b.Executable.Device, Inode: b.Executable.Inode}, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(in.Host)}} |
| 195 | return r, unit |
| 196 | } |
| 197 | v1, unit := reserve("v1") |
| 198 | v3, _ := reserve("v3") |
| 199 | if err := journal.IntendUnit(ctx, unit); err != nil { |
| 200 | t.Fatal(err) |
| 201 | } |
| 202 | _, _, joinTenant := start(v1.Binding.Path) |
| 203 | // Leave Started absent to exercise a process launched before that commit. |
| 204 | verifier := generationRetirement{guard: hostguard.Guard{DataDir: dataDir(), LockDir: filepath.Join(dir, "host-locks"), Root: hostguard.NewRootAuthority(fences)}, journal: journal, kernel: productionGenerationAbsence} |
| 205 | source := generationPinSource{open: func() (rootstate.Store, error) { |
| 206 | return rootstate.Open(filepath.Join(dir, "generations"), false, false) |
| 207 | }, retire: func(ctx context.Context, g *release.RootGenerations) error { |
| 208 | records, err := g.Reservations(ctx) |
| 209 | if err != nil { |
| 210 | return err |
| 211 | } |
| 212 | return verifier.retireAll(ctx, g, records) |
| 213 | }} |
| 214 | source.reconcile = (generationReconciler{read: func(ctx context.Context) (generationInventory, error) { |
| 215 | snapshot, err := generationInventoryAt(ctx, filepath.Join(dir, "census"), filepath.Join(dir, "fences"), journalPath) |
| 216 | if err == nil { |
| 217 | snapshot, err = generationRequiredInventory(ctx, store, snapshot, nil) |
| 218 | } |
| 219 | if err == nil { |
| 220 | snapshot, err = productionGenerationRegistryInventory(ctx, snapshot) |
| 221 | } |
| 222 | return snapshot, err |
| 223 | }, kernel: productionGenerationKernelCensus}).reconcile |
| 224 | readV1 := func() release.GenerationReservation { |
| 225 | t.Helper() |
| 226 | r, err := ledger.Read(v1.Binding) |
| 227 | if err != nil { |
| 228 | t.Fatal(err) |
| 229 | } |
| 230 | return r |
| 231 | } |
| 232 | refuse := func(label string) { |
| 233 | t.Helper() |
| 234 | err := rootstate.WithRelease(ctx, exclusion, func() error { |
| 235 | pins, err := source.StagedPins(ctx, dir) |
| 236 | if err == nil || pins != nil { |
| 237 | t.Fatalf("%s certified: %v %v", label, pins, err) |
| 238 | } |
| 239 | sweepStagedVersions(rows, dir, filepath.Join(dir, "legacy", "generations.json"), 6*time.Hour, time.Now(), source) |
| 240 | return nil |
| 241 | }) |
| 242 | if err != nil { |
| 243 | t.Fatal(err) |
| 244 | } |
| 245 | if readV1().Retirement != "" || !staged(t, dir, "v1") || !staged(t, dir, "v3") { |
| 246 | t.Fatalf("%s lost an outstanding pin", label) |
| 247 | } |
| 248 | raw, err := os.ReadFile(v1.Binding.Path) |
| 249 | if err != nil || sha256.Sum256(raw) != sum || identity(v1.Binding.Path) != v1.Binding.Executable { |
| 250 | t.Fatalf("%s changed V1's bytes or inode: %v", label, err) |
| 251 | } |
| 252 | } |
| 253 | refuse("live launch") |
| 254 | if err := journal.SettleUnit(ctx, unit.Target, unit.ID); err != nil { |
| 255 | t.Fatal(err) |
| 256 | } |
| 257 | if err := journal.CloseWriters(ctx, unit.Target); err != nil { |
| 258 | t.Fatal(err) |
| 259 | } |
| 260 | refuse("closed journal with live controller") |
| 261 | joinController() |
| 262 | refuse("joined controller with live executable") |
| 263 | joinTenant() |
| 264 | file, err := os.Open(v1.Binding.Path) |
| 265 | if err != nil { |
| 266 | t.Fatal(err) |
| 267 | } |
| 268 | defer file.Close() |
| 269 | refuse("joined processes with open descriptor") |
| 270 | if err := file.Close(); err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | if err := rootstate.WithRelease(ctx, exclusion, func() error { |
| 274 | pins, err := source.StagedPins(ctx, dir) |
| 275 | if err != nil { |
| 276 | return err |
| 277 | } |
| 278 | for path, want := range map[string]bool{v1.Binding.Path: false, v3.Binding.Path: true} { |
| 279 | if got, err := pins.Refers(path); err != nil || got != want { |
| 280 | t.Fatalf("exact retirement pin %s: %v %v", path, got, err) |
| 281 | } |
| 282 | } |
| 283 | sweepStagedVersions(rows, dir, filepath.Join(dir, "legacy", "generations.json"), 72*time.Hour, time.Now(), source) |
| 284 | if !staged(t, dir, "v1") { |
| 285 | t.Fatal("retirement bypassed grace") |
| 286 | } |
| 287 | sweepStagedVersions(rows, dir, filepath.Join(dir, "legacy", "generations.json"), 6*time.Hour, time.Now(), source) |
| 288 | return nil |
| 289 | }); err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | if staged(t, dir, "v1") || !staged(t, dir, "v2") || !staged(t, dir, "v3") { |
| 293 | t.Fatal("sweep did not remove only retired V1") |
| 294 | } |
| 295 | if got, err := ledger.Read(v3.Binding); err != nil || got != v3 { |
| 296 | t.Fatalf("pending V3 changed: %+v %v", got, err) |
| 297 | } |
| 298 | if err := journal.CloseWriters(ctx, hostguard.FenceTarget{Host: v3.Binding.Target.Host, Incarnation: v3.Binding.Target.Incarnation, Stamp: v3.Binding.Target.Stamp, Session: v3.Binding.Target.Session, Box: v3.Binding.Target.Box, Enrollment: v3.Binding.Target.Enrollment, BootID: v3.Binding.Target.BootID, ControllerID: v3.Binding.Target.ControllerID}); err != nil { |
| 299 | t.Fatal(err) |
| 300 | } |
| 301 | if err := rootstate.WithRelease(ctx, exclusion, func() error { |
| 302 | pins, err := source.StagedPins(ctx, dir) |
| 303 | if err != nil { |
| 304 | return err |
| 305 | } |
| 306 | if pinned, err := pins.Refers(v3.Binding.Path); err != nil || pinned { |
| 307 | t.Fatalf("cancelled prelaunch pin: %v %v", pinned, err) |
| 308 | } |
| 309 | return nil |
| 310 | }); err != nil { |
| 311 | t.Fatal("prelaunch cancellation with production kernel", err) |
| 312 | } |
| 313 | if _, err := ledger.Reserve(v3.Binding); err == nil { |
| 314 | t.Fatal("cancelled prelaunch replayed") |
| 315 | } |
| 316 | if readV1().Retirement == "" { |
| 317 | t.Fatal("missing retirement tombstone") |
| 318 | } |
| 319 | if _, err := ledger.Reserve(v1.Binding); err == nil { |
| 320 | t.Fatal("retired reservation replayed") |
| 321 | } |
| 322 | if err := journal.IntendUnit(ctx, unit); err == nil { |
| 323 | t.Fatal("closed launch replayed") |
| 324 | } |
| 325 | } |
| 326 | |
amadan