| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "crypto/ed25519" |
| 9 | "encoding/json" |
| 10 | "fmt" |
| 11 | "net" |
| 12 | "net/http/httptest" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "reflect" |
| 16 | "sort" |
| 17 | "strconv" |
| 18 | "strings" |
| 19 | "syscall" |
| 20 | "testing" |
| 21 | "time" |
| 22 | |
| 23 | "github.com/carlosframework/platform/internal/hostbox" |
| 24 | "github.com/carlosframework/platform/internal/hostguard" |
| 25 | "github.com/carlosframework/platform/internal/lease" |
| 26 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 27 | "github.com/carlosframework/platform/internal/registry" |
| 28 | "github.com/carlosframework/platform/internal/release" |
| 29 | "github.com/carlosframework/platform/internal/rootstate" |
| 30 | ) |
| 31 | |
| 32 | const seedRebootPath = "/var/lib/carlos-rootrunner-files-seed-reboot" |
| 33 | |
| 34 | type seedRebootObject struct { |
| 35 | Version lease.ObjectVersion |
| 36 | Body []byte |
| 37 | } |
| 38 | type seedRebootHost struct { |
| 39 | Host string |
| 40 | Fixture seedCrashFixture |
| 41 | Objects []seedRebootObject |
| 42 | Snapshot string |
| 43 | Records []servingParkRecord |
| 44 | } |
| 45 | type seedRebootSave struct { |
| 46 | Boot, Scratch, Install string |
| 47 | Hosts []seedRebootHost |
| 48 | } |
| 49 | |
| 50 | func TestRootRebootFilesSeedTwoHostPhase1(t *testing.T) { |
| 51 | if err := os.Mkdir(seedRebootPath, 0700); err != nil { |
| 52 | t.Fatal("refuse existing reboot fixture", err) |
| 53 | } |
| 54 | runServingParkPeerNamespace(t, false, false, "seed-two-host") |
| 55 | } |
| 56 | |
| 57 | func saveFilesSeedReboot(t *testing.T, binary string, hosts []*seedLiveHost) { |
| 58 | boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") |
| 59 | if err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | saved := seedRebootSave{Boot: strings.TrimSpace(string(boot)), Scratch: os.Getenv("TMPDIR"), Install: filepath.Dir(binary)} |
| 63 | for _, h := range hosts { |
| 64 | f := h.p.f |
| 65 | host := f.f.hostName() |
| 66 | cfg := seedCrashFixture{Claims: filepath.Join(seedRebootPath, host, "claims"), Data: f.f.guard.DataDir, Locks: f.f.guard.LockDir, Root: f.rootDisk.dir, Parks: "/var/lib/carlos-claims/authority/serving-parks", Journal: f.f.journalPath, Registry: f.p.root.Reconcile.Registry.Path(), Retirement: f.p.root.Reconcile.Retirement.JournalDir, Actor: f.actor, Now: f.f.now} |
| 67 | claims := release.NewDirStore(cfg.Claims) |
| 68 | keys, err := f.f.claims.List(t.Context(), "") |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | for _, key := range keys { |
| 73 | body, _, err := f.f.claims.Get(t.Context(), key) |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | if _, err := claims.Put(t.Context(), key, body, ""); err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | } |
| 81 | sh := seedRebootHost{Host: host, Fixture: cfg, Snapshot: f.f.data.(*leasetest.VersionedBucket).Snapshot()} |
| 82 | versions, err := f.f.data.(*leasetest.VersionedBucket).ListVersions(t.Context(), f.actor.Bucket, "") |
| 83 | if err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | for _, v := range versions { |
| 87 | var body []byte |
| 88 | if !v.DeleteMarker { |
| 89 | body, err = f.f.data.GetVersion(t.Context(), f.actor.Bucket, v.Key, v.VersionID) |
| 90 | if err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | } |
| 94 | sh.Objects = append(sh.Objects, seedRebootObject{v, body}) |
| 95 | } |
| 96 | names, err := f.p.ledger.state.Names() |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | for _, name := range names { |
| 101 | if !strings.HasPrefix(name, "retirement-") || !strings.HasSuffix(name, ".json") { |
| 102 | continue |
| 103 | } |
| 104 | r, err := f.p.ledger.read(strings.TrimSuffix(strings.TrimPrefix(name, "retirement-"), ".json")) |
| 105 | if err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | if r.Authorization.Successor.Host == host && r.Parked && !r.Closed { |
| 109 | |
| 110 | sh.Records = append(sh.Records, r) |
| 111 | } |
| 112 | } |
| 113 | if len(sh.Records) == 0 { |
| 114 | t.Fatal("no retirement retained", host) |
| 115 | } |
| 116 | if err := h.executor.registry.Close(); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | if err := f.f.journal.Close(); err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | if err := f.p.root.Reconcile.Registry.Close(); err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | saved.Hosts = append(saved.Hosts, sh) |
| 126 | } |
| 127 | raw, err := json.Marshal(saved) |
| 128 | if err != nil { |
| 129 | t.Fatal(err) |
| 130 | } |
| 131 | if err := os.WriteFile(filepath.Join(seedRebootPath, "fixture.json"), raw, 0600); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | syscall.Sync() |
| 135 | // Phase 2 owns these exact durable fixtures. All tenant units have stopped; |
| 136 | // process exit closes the loopback servers without running TempDir removal. |
| 137 | t.Log("two live hosts retired independently; durable fixture ready for actual reboot") |
| 138 | os.Exit(0) |
| 139 | } |
| 140 | |
| 141 | func TestRootRebootFilesSeedTwoHostPhase2(t *testing.T) { |
| 142 | servingRootRunner(t) |
| 143 | raw, err := os.ReadFile(filepath.Join(seedRebootPath, "fixture.json")) |
| 144 | if err != nil { |
| 145 | t.Fatal(err) |
| 146 | } |
| 147 | var saved seedRebootSave |
| 148 | if err := json.Unmarshal(raw, &saved); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") |
| 152 | if err != nil || strings.TrimSpace(string(boot)) == saved.Boot { |
| 153 | t.Fatal("actual reboot required", err) |
| 154 | } |
| 155 | t.Cleanup(func() { |
| 156 | for _, p := range []string{saved.Scratch, saved.Install, seedRebootPath} { |
| 157 | if !strings.HasPrefix(p, "/var/lib/carlos-rootrunner-") && !strings.HasPrefix(p, "/var/lib/carlos-serving-test-") { |
| 158 | t.Errorf("refuse cleanup path %q", p) |
| 159 | continue |
| 160 | } |
| 161 | if err := os.RemoveAll(p); err != nil { |
| 162 | t.Error(err) |
| 163 | } |
| 164 | } |
| 165 | }) |
| 166 | binary, err := os.Executable() |
| 167 | if err != nil { |
| 168 | t.Fatal(err) |
| 169 | } |
| 170 | witness := rootFilesSeedIsolatedWitness(t, binary, true, true) |
| 171 | servingRunnerMount(t, "/data") |
| 172 | state, err := openClaimServingParkAuthority() |
| 173 | if err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | t.Cleanup(func() { state.Close() }) |
| 177 | fs := newServingParkFS("/data", state) |
| 178 | ledger := &servingParkLedger{state: state, fs: fs, now: time.Now} |
| 179 | rebootRoot := &servingParkRoot{ledger: ledger} |
| 180 | for _, h := range saved.Hosts { |
| 181 | for _, r := range h.Records { |
| 182 | if err := ledger.fs.CheckParked(&r); err == nil { |
| 183 | t.Fatal("old boot authority accepted", h.Host) |
| 184 | } |
| 185 | if !r.Complete { |
| 186 | t.Fatal("unfinished reboot fixture", h.Host) |
| 187 | } |
| 188 | if err := rebootRoot.prepareServingReboot(t.Context(), r.Authorization.ID); err != nil { |
| 189 | topology, topErr := fs.inspectRebootTopology(t.Context()) |
| 190 | observation, inspectErr := fs.inspectRebootRecord(t.Context(), r.Authorization.ID) |
| 191 | rawRecord, readErr := ledger.readHistorical(r.Authorization.ID) |
| 192 | slot, slotErr := ledger.slot(r.Authorization) |
| 193 | t.Logf("reboot diagnosis topology=%+v err=%v inspection=%+v err=%v savedMatches=%v readErr=%v slot=%+v slotErr=%v saved=%+v", topology, topErr, observation, inspectErr, reflect.DeepEqual(rawRecord, r), readErr, slot, slotErr, r) |
| 194 | t.Fatal("prepare completed park", h.Host, err) |
| 195 | } |
| 196 | if err := rebootRoot.commitServingReboot(t.Context(), r.Authorization.ID); err != nil { |
| 197 | t.Fatal("recover park", h.Host, err) |
| 198 | } |
| 199 | } |
| 200 | } |
| 201 | retained := map[string][]byte{} |
| 202 | if strings.Contains(t.Name(), "Renumber") { |
| 203 | for _, host := range saved.Hosts { |
| 204 | for _, r := range host.Records { |
| 205 | for _, file := range r.Source.Files { |
| 206 | if !file.Present { |
| 207 | continue |
| 208 | } |
| 209 | path := filepath.Join("/data/instances/.serving-parked", r.Authorization.ID, file.Name) |
| 210 | body, err := os.ReadFile(path) |
| 211 | if err != nil { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | retained[path] = body |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | } |
| 219 | var hosts []*seedLiveHost |
| 220 | for _, savedHost := range saved.Hosts { |
| 221 | h := reopenSeedLiveHost(t, savedHost, witness, state) |
| 222 | hosts = append(hosts, h) |
| 223 | inventory, err := h.p.f.p.root.Journals.Inventory.ReadStopInventory(t.Context(), savedHost.Host) |
| 224 | if err != nil { |
| 225 | t.Fatal(err) |
| 226 | } |
| 227 | var current syscall.Stat_t |
| 228 | if err := syscall.Stat(filepath.Dir(h.p.f.db), ¤t); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | for _, entry := range inventory.Entries { |
| 232 | if entry.Kind == "path" { |
| 233 | if strings.Contains(t.Name(), "Renumber") && entry.Device == uint64(current.Dev) { |
| 234 | t.Fatal("real filesystem device did not renumber") |
| 235 | } |
| 236 | if !strings.Contains(t.Name(), "Renumber") && (entry.Device != uint64(current.Dev) || entry.Inode != current.Ino) { |
| 237 | t.Fatalf("unchanged-number reboot control changed identity: device %d -> %d, inode %d -> %d", entry.Device, current.Dev, entry.Inode, current.Ino) |
| 238 | } |
| 239 | t.Logf("reboot directory host=%s recordedDevice=%d currentDevice=%d recordedInode=%d currentInode=%d", savedHost.Host, entry.Device, current.Dev, entry.Inode, current.Ino) |
| 240 | } |
| 241 | } |
| 242 | h.wake(t) |
| 243 | } |
| 244 | for _, h := range hosts { |
| 245 | h.probe(t) |
| 246 | } |
| 247 | for i, h := range hosts { |
| 248 | hosts[1-i].probe(t) |
| 249 | h.stop() |
| 250 | h.wake(t) |
| 251 | hosts[1-i].probe(t) |
| 252 | } |
| 253 | for _, h := range hosts { |
| 254 | h.probe(t) |
| 255 | h.stop() |
| 256 | } |
| 257 | for path, want := range retained { |
| 258 | got, err := os.ReadFile(path) |
| 259 | if err != nil || !bytes.Equal(got, want) { |
| 260 | t.Fatalf("retained park changed after two wakes: %s %v", path, err) |
| 261 | } |
| 262 | } |
| 263 | if len(retained) != 0 { |
| 264 | ids, more, err := ledger.recoveryIDs(t.Context(), "", servingParkRecoveryLimit) |
| 265 | if err != nil || more { |
| 266 | t.Fatal("retained census", err) |
| 267 | } |
| 268 | count := 0 |
| 269 | for _, host := range saved.Hosts { |
| 270 | for _, old := range host.Records { |
| 271 | found := false |
| 272 | for _, id := range ids { |
| 273 | if id == old.Authorization.ID { |
| 274 | found = true |
| 275 | } |
| 276 | } |
| 277 | r, err := ledger.readHistorical(old.Authorization.ID) |
| 278 | if !found || err != nil || r.Closed || !r.ReapAfter.Equal(old.ReapAfter) { |
| 279 | t.Fatal("lost retained record", r, err) |
| 280 | } |
| 281 | count++ |
| 282 | } |
| 283 | } |
| 284 | t.Logf("%d unverifiable parks retained after two live wakes per host; %d records counted", count, len(ids)) |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | func reopenSeedLiveHost(t *testing.T, saved seedRebootHost, witness filesSeedLiveWitness, state rootstate.Store) *seedLiveHost { |
| 289 | t.Helper() |
| 290 | cfg := saved.Fixture |
| 291 | data := leasetest.NewVersionedBucket() |
| 292 | data.AddBucket(cfg.Actor.Bucket) |
| 293 | sort.Slice(saved.Objects, func(i, j int) bool { return saved.Objects[i].Version.VersionID < saved.Objects[j].Version.VersionID }) |
| 294 | next := 1 |
| 295 | for _, o := range saved.Objects { |
| 296 | n, err := strconv.Atoi(strings.TrimPrefix(o.Version.VersionID, "v")) |
| 297 | if err != nil { |
| 298 | t.Fatal(err) |
| 299 | } |
| 300 | for next < n { |
| 301 | id := data.Put(cfg.Actor.Bucket, "reboot-gap", nil) |
| 302 | if err := data.DeleteVersion(t.Context(), cfg.Actor.Bucket, "reboot-gap", id); err != nil { |
| 303 | t.Fatal(err) |
| 304 | } |
| 305 | next++ |
| 306 | } |
| 307 | var id string |
| 308 | if o.Version.DeleteMarker { |
| 309 | id = data.BareDelete(cfg.Actor.Bucket, o.Version.Key) |
| 310 | } else { |
| 311 | id = data.Put(cfg.Actor.Bucket, o.Version.Key, o.Body) |
| 312 | } |
| 313 | if id != o.Version.VersionID { |
| 314 | t.Fatal("version identity changed", id, o.Version.VersionID) |
| 315 | } |
| 316 | next++ |
| 317 | } |
| 318 | if data.Snapshot() != saved.Snapshot { |
| 319 | t.Fatal("remote fixture version snapshot changed") |
| 320 | } |
| 321 | handler, _ := leasetest.NewS3Server(data) |
| 322 | server := httptest.NewServer(handler) |
| 323 | t.Cleanup(server.Close) |
| 324 | for k, v := range map[string]string{"AWS_ACCESS_KEY_ID": "test-key", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "", "AWS_EC2_METADATA_DISABLED": "true", "AWS_REGION": "eu-west-1", "AWS_DEFAULT_REGION": "eu-west-1", "CARLOS_HOST_CLAIM": "1", "CARLOS_HOST_CLAIM_ORIGINS": "1", "CARLOS_MOVES_PHASE1": "", "CARLOS_S3_ENDPOINT": server.URL} { |
| 325 | t.Setenv(k, v) |
| 326 | } |
| 327 | claims := release.NewDirStore(cfg.Claims) |
| 328 | journal, err := hostbox.OpenLaunchJournal(cfg.Journal) |
| 329 | if err != nil { |
| 330 | t.Fatal(err) |
| 331 | } |
| 332 | t.Cleanup(func() { journal.Close() }) |
| 333 | roots := &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: cfg.Root}} |
| 334 | rf := &recoveryFixture{host: saved.Host, t: t, claims: claims, evidence: claims, journal: journal, journalPath: cfg.Journal, data: data, bucket: cfg.Actor.Bucket, signer: ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)), now: cfg.Now.Add(time.Minute), seq: 100000} |
| 335 | rf.guard = hostguard.Guard{DataDir: cfg.Data, LockDir: cfg.Locks, Claims: hostguard.StoreReader{Store: claims}, Root: hostguard.NewRootAuthority(roots)} |
| 336 | rf.kernel = &recoveryKernel{journal: journal} |
| 337 | reg, err := registry.Open(cfg.Registry) |
| 338 | if err != nil { |
| 339 | t.Fatal(err) |
| 340 | } |
| 341 | t.Cleanup(func() { reg.Close() }) |
| 342 | row, err := reg.Get(saved.Host) |
| 343 | if err != nil || row == nil { |
| 344 | t.Fatal("route", err) |
| 345 | } |
| 346 | journals, err := hostbox.NewJournalService(journal) |
| 347 | if err != nil { |
| 348 | t.Fatal(err) |
| 349 | } |
| 350 | f := &servingRootFixture{t: t, f: rf, rootDisk: roots, db: row.DBPath, actor: cfg.Actor} |
| 351 | f.p = &servingParkRoot{ledger: &servingParkLedger{state: state, fs: newServingParkFS(cfg.Data, state), journal: journal, now: func() time.Time { return rf.now }}, empty: plan9D5Fixture(t).owners} |
| 352 | coordinator := rf.root(nil) |
| 353 | coordinator.Kernel = &noProcessKernelFake{} |
| 354 | coordinator.Inventory = journals.Inventory |
| 355 | f.p.root = ClaimRootHTTP{Roots: map[string]string{"data": cfg.Data}, Journals: journals, Launches: journal, Stop: ClaimStopHTTP{Coordinator: coordinator, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return rf.peer, nil }}, InventoryActor: func(context.Context) (hostbox.Actor, error) { return f.actor, nil }, RequireNewWork: func(context.Context) error { return nil }} |
| 356 | f.p.root.Reconcile.Registry = reg |
| 357 | f.p.root.Reconcile.Inventory = journals.Inventory |
| 358 | f.p.root.Reconcile.Box, f.p.root.Reconcile.Enrollment = f.actor.Box, f.actor.Enrollment |
| 359 | f.p.root.Reconcile.Retirement.JournalDir = cfg.Retirement |
| 360 | source := &movesFilesSource{DataDir: cfg.Data, Routes: reg, newFiles: func(context.Context, string, string) (release.Store, error) { |
| 361 | return casBucketStore{b: data, bucket: cfg.Actor.Bucket}, nil |
| 362 | }} |
| 363 | opened, err := source.Open(t.Context(), claims, data, cfg.Actor) |
| 364 | if err != nil { |
| 365 | t.Fatal(err) |
| 366 | } |
| 367 | p := newServingPreparation(t, f, "boot", fmt.Sprintf("reboot-%d", os.Getpid())) |
| 368 | return &seedLiveHost{p: p, endpoint: server.URL, files: opened.(*movesFiles), witness: witness} |
| 369 | } |
| 370 | |
amadan