| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "crypto/ed25519" |
| 9 | "encoding/base64" |
| 10 | "encoding/json" |
| 11 | "fmt" |
| 12 | "net" |
| 13 | "os" |
| 14 | "os/exec" |
| 15 | "os/user" |
| 16 | "path/filepath" |
| 17 | "strconv" |
| 18 | "syscall" |
| 19 | "testing" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/hostactors" |
| 23 | "github.com/carlosframework/platform/internal/hostbox" |
| 24 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 25 | "github.com/carlosframework/platform/internal/hostguard" |
| 26 | "github.com/carlosframework/platform/internal/hostops" |
| 27 | "github.com/carlosframework/platform/internal/release" |
| 28 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 29 | "github.com/carlosframework/platform/internal/rootstate" |
| 30 | ) |
| 31 | |
| 32 | func init() { |
| 33 | if len(os.Args) > 1 && os.Args[1] == registryChildCommand { |
| 34 | if err := RunRegistryChild(os.Args[2:]); err != nil { |
| 35 | fmt.Fprintln(os.Stderr, err) |
| 36 | os.Exit(1) |
| 37 | } |
| 38 | os.Exit(0) |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | func TestRootFilesSeedInstanceControlConstruction(t *testing.T) { |
| 43 | runServingParkPeerNamespace(t, false, false, "seed-construction") |
| 44 | } |
| 45 | |
| 46 | func filesSeedInstanceControlChild(t *testing.T, binary, root string, listener *net.UnixListener) { |
| 47 | t.Helper() |
| 48 | for _, path := range []string{"/data", "/var/lib/carlos-claims"} { |
| 49 | if err := os.Mkdir(path, 0755); err != nil { |
| 50 | t.Fatal("refuse existing fixture path", path, err) |
| 51 | } |
| 52 | t.Cleanup(func() { os.RemoveAll(path) }) |
| 53 | } |
| 54 | account, err := user.Lookup("carlos") |
| 55 | if err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | uid, _ := strconv.Atoi(account.Uid) |
| 59 | gid, _ := strconv.Atoi(account.Gid) |
| 60 | if err := os.Chown("/data", uid, gid); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | file, err := listener.File() |
| 64 | if err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | defer file.Close() |
| 68 | listener.SetUnlinkOnClose(false) |
| 69 | if err := listener.Close(); err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | output, err := os.Create(filepath.Join(root, "control.log")) |
| 73 | if err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | cmd := exec.Command(binary, "-test.run=^TestRootFilesSeedInstanceControlPayload$", "-test.v", "-test.timeout=90s") |
| 77 | cmd.Env = append(os.Environ(), "FILES_SEED_CONTROL=1", "TMPDIR="+root) |
| 78 | if os.Getenv("PLAN14_R9_PENDING") != "" || os.Getenv("PLAN14_R10_ACQUISITION") != "" || os.Getenv("PLAN14_R10_RETIRED") != "" { |
| 79 | cmd.SysProcAttr = &syscall.SysProcAttr{Cloneflags: syscall.CLONE_NEWNS} |
| 80 | for _, path := range []string{"/run/carlos", "/run/carlos/hostguard"} { |
| 81 | if err := os.Mkdir(path, 0755); err == nil { |
| 82 | t.Cleanup(func() { os.RemoveAll(path) }) |
| 83 | } else if !os.IsExist(err) { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | } |
| 87 | } |
| 88 | cmd.ExtraFiles = []*os.File{file} |
| 89 | cmd.Stdout, cmd.Stderr = output, output |
| 90 | if err := cmd.Start(); err != nil { |
| 91 | output.Close() |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | t.Cleanup(func() { |
| 95 | cmd.Process.Kill() |
| 96 | cmd.Wait() |
| 97 | output.Close() |
| 98 | body, _ := os.ReadFile(filepath.Join(root, "control.log")) |
| 99 | t.Log(string(body)) |
| 100 | }) |
| 101 | // The inherited listener stays open while RunInstanceControl constructs its owners. |
| 102 | } |
| 103 | |
| 104 | func TestRootFilesSeedInstanceControlPayload(t *testing.T) { |
| 105 | if os.Getenv("FILES_SEED_CONTROL") != "1" { |
| 106 | t.Fatal("invoke through construction witness") |
| 107 | } |
| 108 | t.Setenv("CARLOS_HOST_CLAIM", "1") |
| 109 | t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1") |
| 110 | t.Setenv("CARLOS_MOVES_PHASE1", "") |
| 111 | t.Setenv("CARLOS_REGION_HA", "") |
| 112 | t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "") |
| 113 | t.Setenv("CARLOS_DATA_DIR", "/data") |
| 114 | t.Setenv("LISTEN_FDS", "1") |
| 115 | t.Setenv("LISTEN_PID", strconv.Itoa(os.Getpid())) |
| 116 | store, resources, owners, _ := fix4JournalFixture(t) |
| 117 | if os.Getenv("PLAN14_R10_ACQUISITION") != "" || os.Getenv("PLAN14_R10_RETIRED") != "" { |
| 118 | openClaimRootAuthority = func() (rootstate.Store, error) { |
| 119 | return rootstate.Open("/var/lib/carlos-claims/authority/fences", true, false) |
| 120 | } |
| 121 | openClaimRootJournal = func() (*hostbox.LaunchJournal, error) { |
| 122 | return hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | resources.OptDir = generationRootRunnerDir(t) |
| 127 | dir := t.TempDir() |
| 128 | t.Setenv("CARLOS_DEPLOYMENT_DIR", dir) |
| 129 | t.Setenv("CARLOS_OPT_DIR", resources.OptDir) |
| 130 | body, _, err := store.Get(t.Context(), hostactors.Key) |
| 131 | if err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | if _, err := release.NewDirStore(dir).Put(t.Context(), hostactors.Key, body, ""); err != nil { |
| 135 | t.Fatal(err) |
| 136 | } |
| 137 | owners.Serving = plan9D5Fixture(t).owners.Sources |
| 138 | owners.Controllers = productionStopControllers{} |
| 139 | claimOwnerRegistration.Provider = seedConstructionOwners{owners} |
| 140 | state, err := rootstate.Open(rootparticipation.Namespace, true, false) |
| 141 | if err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | actor, err := owners.Identity.Verify(t.Context()) |
| 145 | if err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | filesSeedConstructionMigration(t, resources.OptDir, actor.ControlBucket, actor.Box, actor.Enrollment) |
| 149 | raw, _ := json.Marshal(rootparticipation.Census{Schema: 1, Box: actor.Box, Enrollment: actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) |
| 150 | if err := state.Write("census.json", raw); err != nil { |
| 151 | t.Fatal(err) |
| 152 | } |
| 153 | state.Close() |
| 154 | exclusion := &release.ReleaseExclusion{Open: func() (rootstate.Store, error) { |
| 155 | return rootstate.Open(filepath.Join(dir, "release-exclusion"), true, false) |
| 156 | }} |
| 157 | rootLaunchStartup.Do(func() { |
| 158 | verify := func(ctx context.Context, _ uint64) error { return exclusion.Check(ctx) } |
| 159 | if os.Getenv("PLAN14_R10_ACQUISITION") != "" || os.Getenv("PLAN14_R10_RETIRED") != "" { |
| 160 | generation, err := rootstate.Open(release.RootGenerationsDir, true, false) |
| 161 | if err != nil { |
| 162 | t.Fatal(err) |
| 163 | } |
| 164 | generation.Close() |
| 165 | read := func(ctx context.Context) (generationInventory, error) { |
| 166 | snapshot, err := productionLocalGenerationInventory(ctx) |
| 167 | if err != nil { |
| 168 | t.Logf("production generation read: %v", err) |
| 169 | return snapshot, err |
| 170 | } |
| 171 | if os.Getenv("PLAN14_R10_RETIRED") != "" { |
| 172 | if len(snapshot.Installations) != 2 || len(snapshot.Payloads) != 2 { |
| 173 | t.Fatalf("lost retained installation or payload: %d %d", len(snapshot.Installations), len(snapshot.Payloads)) |
| 174 | } |
| 175 | installs := map[string]hostguard.FenceView{} |
| 176 | for _, v := range snapshot.Installations { |
| 177 | installs[v.Host] = v |
| 178 | } |
| 179 | if err := generationPayloadBindings(snapshot.Payloads[1:], installs); err == nil { |
| 180 | t.Fatal("retirement dropped payload obligation") |
| 181 | } |
| 182 | } |
| 183 | return snapshot, nil |
| 184 | } |
| 185 | v := rootLaunchVerifier{exclude: exclusion, open: func() (rootstate.Store, error) { return rootstate.Open(release.RootGenerationsDir, false, false) }, reconcile: generationReconciler{read: read, kernel: func(context.Context) (string, string, error) { |
| 186 | boot, err := execlease.BootID() |
| 187 | return boot, floorTestDigest, err |
| 188 | }}} |
| 189 | verify = v.verify |
| 190 | } |
| 191 | rootLaunchStartup.err = rootparticipation.RegisterLaunchVerifier(verify, exclusion.With) |
| 192 | }) |
| 193 | if rootLaunchStartup.err != nil { |
| 194 | t.Fatal(rootLaunchStartup.err) |
| 195 | } |
| 196 | if mode := os.Getenv("PLAN14_R9_PENDING"); mode != "" { |
| 197 | filesSeedPendingConstruction(t, release.NewDirStore(dir), resources, owners.ClaimOwnerBindings, actor) |
| 198 | if mode == "system" { |
| 199 | if err := recoverRootExecution(t.Context(), "/data", systemGuardDir(resources.OptDir)); err != nil { |
| 200 | t.Fatal("production system recovery", err) |
| 201 | } |
| 202 | } |
| 203 | } |
| 204 | if os.Getenv("PLAN14_R10_RETIRED") != "" { |
| 205 | for _, host := range []string{"sibling.example", "retired.example"} { |
| 206 | t.Setenv("PLAN14_R10_INSTALL_HOST", host) |
| 207 | filesSeedPendingConstruction(t, release.NewDirStore(dir), resources, owners.ClaimOwnerBindings, actor) |
| 208 | } |
| 209 | } |
| 210 | if os.Getenv("PLAN14_R10_ACQUISITION") != "" { |
| 211 | plan14PendingAcquisition(t, resources.OptDir, actor.ControlBucket) |
| 212 | } |
| 213 | RunInstanceControl(nil) |
| 214 | t.Fatal("claims-on instance-control returned before serving") |
| 215 | } |
| 216 | |
| 217 | type seedConstructionOwners struct{ fix4Owners } |
| 218 | |
| 219 | func (o seedConstructionOwners) OpenClaimOwners(_ context.Context, store release.Store, _ ClaimRuntimeResources) (ClaimOwnerBindings, error) { |
| 220 | owners := o.ClaimOwnerBindings |
| 221 | authority := *owners.RetirementAuthority |
| 222 | authority.Store = store |
| 223 | authority.Directory.Store = store |
| 224 | owners.RetirementAuthority = &authority |
| 225 | return owners, nil |
| 226 | } |
| 227 | |
| 228 | func filesSeedConstructionMigration(t *testing.T, opt, deployment, box, enrollment string) { |
| 229 | t.Helper() |
| 230 | boot, err := execlease.BootID() |
| 231 | if err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | record := rootMigrationRecord{Schema: 1, Identity: rootMigrationIdentity{Deployment: deployment, Box: box, Enrollment: enrollment, Generation: 1}, Phase: "complete", Snapshot: rootMigrationSnapshot{ |
| 235 | Complete: true, DirectoryRevision: 1, Exclusion: "construction-fixture", Items: []rootMigrationItem{}, |
| 236 | Report: hostops.CensusReport{Complete: true, Box: box, Enrollment: enrollment, DirectoryRevision: 1, Reservation: "construction-fixture", BootID: boot, CapturedAt: time.Now(), Installations: []hostops.CensusInstallation{}, Seeds: &hostops.CensusSeeds{}}, |
| 237 | }} |
| 238 | if err := record.Snapshot.preflight(record.Identity); err != nil { |
| 239 | t.Fatal("construction migration preflight", err) |
| 240 | } |
| 241 | migration, err := rootstate.Open(rootMigrationDir, true, false) |
| 242 | if err != nil { |
| 243 | t.Fatal("construction migration state", err) |
| 244 | } |
| 245 | defer migration.Close() |
| 246 | raw, err := json.Marshal(record) |
| 247 | if err != nil { |
| 248 | t.Fatal(err) |
| 249 | } |
| 250 | if err := migration.Write("migration.json", raw); err != nil { |
| 251 | t.Fatal(err) |
| 252 | } |
| 253 | key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) |
| 254 | t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey))) |
| 255 | if err := os.MkdirAll(systemGuardDir(opt), 0755); err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | rootFloorRunnerBinary(t, opt, systemBinaryPath(opt), "v2.0.0", key) |
| 259 | rootFloorRunnerBinary(t, opt, filepath.Join(systemGuardDir(opt), "carlos"), "v2.0.0", key) |
| 260 | } |
| 261 | |
amadan