| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "crypto/ed25519" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "net" |
| 13 | "net/http" |
| 14 | "os" |
| 15 | "path/filepath" |
| 16 | "testing" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/provision" |
| 22 | "github.com/carlosframework/platform/internal/registry" |
| 23 | "github.com/carlosframework/platform/internal/release" |
| 24 | "github.com/carlosframework/platform/internal/rollout" |
| 25 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 26 | "github.com/carlosframework/platform/internal/rootstate" |
| 27 | ) |
| 28 | |
| 29 | func TestPlan14DeletedRecordRetirementRootRunner(t *testing.T) { |
| 30 | t.Setenv("PLAN14_R11_DELETION", "1") |
| 31 | TestPlan14GenerationParkedRootRunner(t) |
| 32 | } |
| 33 | |
| 34 | func TestPlan14ServingHandoverPinsRootRunner(t *testing.T) { |
| 35 | if err := os.Mkdir("/var/lib/carlos-claims", 0700); err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | t.Cleanup(func() { os.RemoveAll("/var/lib/carlos-claims") }) |
| 39 | t.Setenv("PLAN14_R11_HANDOVER_ROOT", "1") |
| 40 | TestPlan14ServingHandoverPins(t) |
| 41 | } |
| 42 | |
| 43 | func r11DeletedRecordPins(t *testing.T, snapshot generationInventory, store release.Store) { |
| 44 | t.Helper() |
| 45 | ctx := t.Context() |
| 46 | open := func(path string) rootstate.Store { |
| 47 | s, err := rootstate.Open(path, true, false) |
| 48 | if err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | t.Cleanup(func() { s.Close() }) |
| 52 | return s |
| 53 | } |
| 54 | write := func(s rootstate.Store, key string, v any) { |
| 55 | raw, err := json.Marshal(v) |
| 56 | if err == nil { |
| 57 | err = s.Write(key, raw) |
| 58 | } |
| 59 | if err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | } |
| 63 | write(open(rootparticipation.Namespace), "census.json", snapshot.Census) |
| 64 | fences := open("/var/lib/carlos-claims/authority/fences") |
| 65 | in := snapshot.Installations[0].Installation |
| 66 | write(fences, in.Host+".json", map[string]any{"Schema": 1, "Installation": in, "Retired": []hostguard.Captured{{Incarnation: in.Incarnation, Stamp: in.Stamp}}, "State": map[string]any{"v": 1, "host": in.Host, "incarnation": in.Incarnation, "active_stamp": 0, "retired_stamps": []uint64{in.Stamp}, "fence_journal": snapshot.Installations[0].Receipts}}) |
| 67 | journal, err := hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") |
| 68 | if err != nil { |
| 69 | t.Fatal(err) |
| 70 | } |
| 71 | service, err := hostbox.NewJournalService(journal) |
| 72 | if err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | if err = service.Inventory.WriteStopInventory(ctx, snapshot.Payloads[0].Inventory); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | target := snapshot.Launches[0].Target |
| 79 | if err = journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "closed-fixture", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "fixture", Device: 1, Inode: 1}}}); err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | if err = journal.CloseWriters(ctx, target); err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | if err = journal.Close(); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | open(release.RootGenerationsDir) |
| 89 | snapshotRead, err := generationInventoryAt(ctx, rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", "/var/lib/carlos-claims/journals.db") |
| 90 | if err != nil { |
| 91 | t.Fatal("read production inventory", err) |
| 92 | } |
| 93 | snapshotRead.Retirements, err = productionGenerationRetirements(ctx) |
| 94 | if err != nil { |
| 95 | t.Fatal("read retirements", err) |
| 96 | } |
| 97 | if _, err = generationRequiredInventory(ctx, store, snapshotRead, productionGenerationCreation); err != nil { |
| 98 | t.Fatal("retained obligations", err) |
| 99 | } |
| 100 | if _, err = productionGenerationRegistryInventory(ctx, snapshotRead); err != nil { |
| 101 | t.Fatal("registry obligations", err) |
| 102 | } |
| 103 | if _, err = generationPhysicalPayloads(ctx, snapshotRead); err != nil { |
| 104 | t.Fatal("physical obligations", err) |
| 105 | } |
| 106 | if _, _, err = productionGenerationKernelCensus(ctx); err != nil { |
| 107 | t.Fatal("kernel census", err) |
| 108 | } |
| 109 | source := productionRootGenerationPins() |
| 110 | lock := open(rootReleasePrivate) |
| 111 | if err = rootstate.WithRelease(ctx, lock, func() error { |
| 112 | pins, err := source.StagedPins(ctx, release.OptDir()) |
| 113 | if err == nil && len(pins) != 0 { |
| 114 | t.Fatal("unexpected pins", pins) |
| 115 | } |
| 116 | return err |
| 117 | }); err != nil { |
| 118 | t.Fatal("production pins after record deletion", err) |
| 119 | } |
| 120 | pub, priv, err := ed25519.GenerateKey(nil) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | body := []byte("signed sibling") |
| 125 | m := release.Manifest{App: "sibling", Version: "v9", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| 126 | if err = release.Ship(ctx, store, "bab", m, map[string][]byte{release.AdoptPlatform(): body}); err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | if err = release.Promote(ctx, store, priv, "bab", "sibling", "stable", "v9"); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | if err = release.StageArtifact(release.OptDir(), "bab", "v9", "sibling", body); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | manifest := release.StagedManifestPath(release.OptDir(), "bab", "v9", "sibling") |
| 136 | if err = os.WriteFile(manifest, []byte("broken sibling manifest"), 0644); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | if err = rootstate.WithRelease(ctx, lock, func() error { |
| 140 | proof := release.NewRootProofGate(filepath.Join(t.TempDir(), "policy.json"), filepath.Join(t.TempDir(), "bakes")) |
| 141 | _, err := release.StageHeldVerifiedWithPins(ctx, store, pub, nil, release.OptDir(), registry.Route{Host: "sibling.example", Account: "bab", App: "sibling", Kind: "instance", Channel: "stable", Status: "landing", HoldOp: "repair"}, "v9", proof, source) |
| 142 | return err |
| 143 | }); err != nil { |
| 144 | t.Fatal("sibling repair after deleted host", err) |
| 145 | } |
| 146 | raw, err := os.ReadFile(manifest) |
| 147 | if err != nil || string(raw) == "broken sibling manifest" { |
| 148 | t.Fatal("sibling manifest not repaired", err) |
| 149 | } |
| 150 | if _, _, err = store.Get(ctx, provision.RecordKey(in.Account, in.App, in.Host)); !errors.Is(err, os.ErrNotExist) { |
| 151 | t.Fatal("record resurrected", err) |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | func TestPlan14SettingsSuccessorProductionRootRunner(t *testing.T) { |
| 156 | for _, path := range []string{"/var/lib/carlos-claims", "/var/lib/carlos-release", rootRetryProjection, "/run/carlos"} { |
| 157 | if err := os.Mkdir(path, 0755); err != nil { |
| 158 | t.Fatal("existing fixture", path, err) |
| 159 | } |
| 160 | t.Cleanup(func() { os.RemoveAll(path) }) |
| 161 | } |
| 162 | open := func(path string) rootstate.Store { |
| 163 | s, err := rootstate.Open(path, true, false) |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | t.Cleanup(func() { s.Close() }) |
| 168 | return s |
| 169 | } |
| 170 | c := rootparticipation.Census{Schema: 1, Box: "box-1", Enrollment: "en", Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true} |
| 171 | raw, _ := json.Marshal(c) |
| 172 | if err := open(rootparticipation.Namespace).Write("census.json", raw); err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | failures := open(rollout.RootFailuresDir) |
| 176 | if _, err := rollout.InitializeRootFailures(failures, rootUpdatesMigration(c), nil); err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | testInventoryRegistrationWith(t, true, nil, open("/var/lib/carlos-claims/authority/fences"), func(t *testing.T, f *recoveryFixture, root ClaimRootHTTP, row registry.Route, actor hostbox.Actor) { |
| 180 | t.Setenv("CARLOS_DATA_DIR", root.Roots["data"]) |
| 181 | t.Setenv("CARLOS_OPT_DIR", root.Roots["opt"]) |
| 182 | pub, priv, err := ed25519.GenerateKey(nil) |
| 183 | if err != nil { |
| 184 | t.Fatal(err) |
| 185 | } |
| 186 | m := release.Manifest{App: row.App, Version: "v1", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| 187 | body := []byte("signed settings witness") |
| 188 | if err = release.Ship(t.Context(), f.claims, row.Account, m, map[string][]byte{release.AdoptPlatform(): body}); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | if err = release.Promote(t.Context(), f.claims, priv, row.Account, row.App, "stable", "v1"); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | if err = os.MkdirAll(filepath.Dir(row.Socket), 0755); err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | listener, err := net.Listen("unix", row.Socket) |
| 198 | if err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ready")) })} |
| 202 | go server.Serve(listener) |
| 203 | defer server.Close() |
| 204 | source := productionRootGenerations(f.claims, root.Roots["opt"]) |
| 205 | source.pub = func() (ed25519.PublicKey, error) { return pub, nil } |
| 206 | digest := sha256.Sum256(body) |
| 207 | g := root.Stop.Coordinator.Guard |
| 208 | capture, err := g.Capture(row.Host) |
| 209 | if err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | prior, err := g.Root.Installation(t.Context(), row.Host, capture) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | for wake := 0; wake < 2; wake++ { |
| 217 | opened, err := productionRootUpdates(dataDir())(t.Context(), f.claims, pub) |
| 218 | if err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | resolved, err := opened.Resolve(t.Context(), row.Host) |
| 222 | if err != nil { |
| 223 | t.Fatal("settings wake", wake, err) |
| 224 | } |
| 225 | if wake == 1 && (resolved.Build.State != nil || resolved.Consumed != nil) { |
| 226 | t.Fatal("successor reused foreign completion") |
| 227 | } |
| 228 | if wake == 0 { |
| 229 | if err = resolved.Save(release.Consumed{Resolved: &release.ResolvedRequest{ID: 11, Target: "v1"}}); err != nil { |
| 230 | t.Fatal(err) |
| 231 | } |
| 232 | } |
| 233 | opened.Close() |
| 234 | current, err := g.Capture(row.Host) |
| 235 | if err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | installed, err := g.Root.Installation(t.Context(), row.Host, current) |
| 239 | if err != nil { |
| 240 | t.Fatal(err) |
| 241 | } |
| 242 | b := release.GenerationBinding{Binding: release.HistoryBinding{Host: row.Host, Account: row.Account, App: row.App}, Target: release.GenerationOwner{Host: installed.Host, Incarnation: installed.Incarnation, Stamp: installed.Stamp, Box: installed.Box, Enrollment: installed.Enrollment}, Installation: rootHistoryInstallation(installed), Record: installed.Record, Version: "v1", Digest: hex.EncodeToString(digest[:]), Attempt: fmt.Sprintf("wake-%d", wake)} |
| 243 | finish, close, err := source.beginUpdate(t.Context(), source, b) |
| 244 | if err != nil { |
| 245 | t.Fatal("production begin", wake, err) |
| 246 | } |
| 247 | if err = finish(fmt.Sprintf("invocation-%d", wake), nil, func() error { return nil }); err != nil { |
| 248 | close() |
| 249 | t.Fatal(err) |
| 250 | } |
| 251 | close() |
| 252 | if wake == 0 { |
| 253 | observed := f.read() |
| 254 | admitted, err := hostclaim.Update(t.Context(), f.claims, observed, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: observed.Claim.Holder.Session, ObservedGeneration: observed.Claim.Holder.Generation, Box: actor.Box, Region: actor.Region, Mode: "cold", NewGeneration: "second-generation"})) |
| 255 | if err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | next := prior |
| 259 | next.Stamp = admitted.Claim.Holder.Stamp |
| 260 | if next.Stamp <= prior.Stamp { |
| 261 | t.Fatal("cold admission did not advance installation") |
| 262 | } |
| 263 | intent, _ := json.Marshal(admitted) |
| 264 | if err = g.RebindRootInventory(t.Context(), capture, prior.InventoryRevision, next, intent, func(claim *hostclaim.Claim) error { |
| 265 | if !hostclaim.SameWriterIdentity(admitted, hostclaim.Observed{Host: row.Host, Claim: claim}) { |
| 266 | return os.ErrPermission |
| 267 | } |
| 268 | return nil |
| 269 | }, func() error { return nil }); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | } |
| 273 | } |
| 274 | var saved rootUpdateRecord |
| 275 | raw, err := failures.Read(rootSettingsKey(row.Host, rootHistoryInstallation(prior))) |
| 276 | if err != nil || json.Unmarshal(raw, &saved) != nil || saved.Build.State == nil || saved.Build.State.Completed != 11 { |
| 277 | t.Fatal("predecessor settings not retained", err) |
| 278 | } |
| 279 | }) |
| 280 | } |
| 281 | |
amadan