| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "net" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "reflect" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/genselect" |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/lease" |
| 22 | "github.com/carlosframework/platform/internal/lease/leasetest" |
| 23 | "github.com/carlosframework/platform/internal/provision" |
| 24 | ) |
| 25 | |
| 26 | func newServingLegacyRootSetup(t *testing.T, empty bool, dataRoot ...string) (*servingRootFixture, ClaimLaunchInputs, ClaimStopCoordinator) { |
| 27 | t.Helper() |
| 28 | var hosts []string |
| 29 | if len(dataRoot) > 1 { |
| 30 | hosts = dataRoot[1:] |
| 31 | } |
| 32 | f, _, _, in := servingRetirementFixture(t, hosts...) |
| 33 | if len(dataRoot) != 0 { |
| 34 | f.guard.DataDir = dataRoot[0] |
| 35 | in.DBPath = provision.NestedDBPath(f.guard.DataDir, f.hostName()) |
| 36 | for _, dir := range []string{filepath.Join(f.guard.DataDir, "hostguard"), filepath.Dir(in.DBPath)} { |
| 37 | if err := os.MkdirAll(dir, 0700); err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | } |
| 41 | } |
| 42 | if !empty { |
| 43 | _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { |
| 44 | c.Holder.WriterBox = "box-1" |
| 45 | c.Holder.Location = "instances/" + c.Host |
| 46 | return nil |
| 47 | }) |
| 48 | if err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | } |
| 52 | disk := &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: t.TempDir()}} |
| 53 | roots := &servingAuthorityDisk{rollbackDisk: &rollbackDisk{dir: t.TempDir()}} |
| 54 | f.guard.Root = hostguard.NewRootAuthority(roots) |
| 55 | journals, err := hostbox.NewJournalService(f.journal) |
| 56 | if err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | fx := &servingRootFixture{t: t, f: f, disk: disk, rootDisk: roots, db: in.DBPath, actor: hostbox.Actor{Box: "box-1", Region: "ie", Enrollment: "enrollment", Revision: 7, Binding: "7", Eligible: true}} |
| 60 | fs := newServingParkFS(f.guard.DataDir, disk) |
| 61 | fs.owner = uint32(os.Geteuid()) |
| 62 | fs.mount = func(string) (bool, error) { return true, nil } |
| 63 | fx.p = &servingParkRoot{ledger: &servingParkLedger{state: disk, fs: fs, journal: f.journal, now: func() time.Time { return f.now }}, empty: plan9D5Fixture(t).owners} |
| 64 | f.peer = StopPeer{UID: 1001, PID: 77, ProcessStart: "edge-start", BootID: "boot", ControllerID: "controller"} |
| 65 | coordinator := f.root(stopDrainFunc(func(context.Context, StopRequest, string) (DrainAcknowledgement, error) { |
| 66 | t.Fatal("session zero requested drain") |
| 67 | return DrainAcknowledgement{}, hostbox.ErrChanged |
| 68 | })) |
| 69 | coordinator.Kernel = &noProcessKernelFake{} |
| 70 | coordinator.Inventory = stopInventoryFunc(func(ctx context.Context, host string) (StopInventory, error) { |
| 71 | return journals.Inventory.ReadStopInventory(ctx, host) |
| 72 | }) |
| 73 | fx.p.root = ClaimRootHTTP{Roots: map[string]string{"data": f.guard.DataDir}, Journals: journals, Stop: ClaimStopHTTP{Coordinator: coordinator, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil }}, InventoryActor: func(context.Context) (hostbox.Actor, error) { return fx.actor, nil }, RequireNewWork: func(context.Context) error { return nil }} |
| 74 | fx.install(f.read()) |
| 75 | coordinator.RetainNoProcess = fx.p.retainLegacyNoProcess |
| 76 | if !empty { |
| 77 | for suffix, body := range map[string]string{"": "legacy bytes", "-wal": "legacy WAL", "-shm": "legacy SHM"} { |
| 78 | if err := os.WriteFile(fx.db+suffix, []byte(body), 0600); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | } |
| 82 | } |
| 83 | left, right := net.Pipe() |
| 84 | fx.conn = left |
| 85 | t.Cleanup(func() { left.Close(); right.Close() }) |
| 86 | return fx, in, coordinator |
| 87 | } |
| 88 | |
| 89 | func newServingLegacyRootFixture(t *testing.T, empty bool, beforeAdmission ...func(*servingRootFixture, *ClaimStopCoordinator, StopRequest)) (*servingRootFixture, ClaimLaunchInputs) { |
| 90 | t.Helper() |
| 91 | fx, in, coordinator := newServingLegacyRootSetup(t, empty) |
| 92 | f := fx.f |
| 93 | o := f.read() |
| 94 | h := o.Claim.Holder |
| 95 | req := StopRequest{Region: "ie", Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: "enrollment", BootID: "boot", ControllerID: "controller", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: "7", RosterRevision: 7} |
| 96 | if _, err := coordinator.FenceAndStop(t.Context(), req); err != nil { |
| 97 | t.Fatalf("no-process: %v", err) |
| 98 | } |
| 99 | for _, fn := range beforeAdmission { |
| 100 | fn(fx, &coordinator, req) |
| 101 | } |
| 102 | admission := hostclaim.Admission{ObservedSession: 0, Box: "box-1", Region: "ie", Mode: "cold", NewGeneration: "G"} |
| 103 | if !empty { |
| 104 | admission.PredecessorRegion = "ie" |
| 105 | } |
| 106 | admitted, err := hostclaim.Update(t.Context(), f.claims, f.read(), hostclaim.AdmitWriter(admission)) |
| 107 | if err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | fx.admitted = admitted |
| 111 | fx.current, err = servingWriterIdentity(admitted, "box-1", "ie") |
| 112 | if err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | fx.install(admitted) |
| 116 | in.Admission, in.Empty = admission, empty |
| 117 | return fx, in |
| 118 | } |
| 119 | |
| 120 | func TestServingLegacyRootAuthority(t *testing.T) { |
| 121 | for _, empty := range []bool{false, true} { |
| 122 | t.Run(fmt.Sprint(empty), func(t *testing.T) { |
| 123 | f, in := newServingLegacyRootFixture(t, empty) |
| 124 | r := &claimServingArtifactRetirement{Claims: f.f.claims, Guard: f.f.guard, DataDir: f.f.guard.DataDir, Box: f.actor.Box, Region: f.actor.Region, Root: servingRootClientFixture{f}} |
| 125 | work := servingRootWork{f.f.guard, f.admitted} |
| 126 | if err := r.BeforeLegacyInstall(t.Context(), f.admitted, in, work); err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | id := servingRetirementID(servingArtifactIdentity{}, f.current, true) |
| 130 | record, err := f.p.ledger.read(id) |
| 131 | if err != nil || !record.Complete { |
| 132 | t.Fatalf("record: %+v %v", record, err) |
| 133 | } |
| 134 | if record.LegacyEvidence == nil || record.LegacyEvidence.NoProcess == nil || len(record.Receipts) != 0 { |
| 135 | t.Fatal("invented fence for session zero") |
| 136 | } |
| 137 | if (record.Disposition == servingParkArtifactFree) != empty { |
| 138 | t.Fatal(record.Disposition) |
| 139 | } |
| 140 | if !empty { |
| 141 | b, err := os.ReadFile(filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", id, filepath.Base(f.db))) |
| 142 | if err != nil || string(b) != "legacy bytes" { |
| 143 | t.Fatalf("park %q %v", b, err) |
| 144 | } |
| 145 | } |
| 146 | if err := r.Recover(t.Context(), f.admitted, in); err != nil { |
| 147 | t.Fatalf("lost completion reply: %v", err) |
| 148 | } |
| 149 | if err := work.With(t.Context(), in.Identity, func() error { return r.EnrollLocked(t.Context(), f.admitted, in) }); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | }) |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | func TestServingLegacyRootRefusals(t *testing.T) { |
| 157 | for _, attack := range []string{"stop-missing", "stop-bytes", "stop-region", "first-creator", "marker", "half-tags", "tags", "peer", "census", "enrollment", "binding", "source", "main-intention", "empty-database", "empty-wal", "empty-remote", "empty-unavailable", "selector"} { |
| 158 | t.Run(attack, func(t *testing.T) { |
| 159 | empty := strings.HasPrefix(attack, "empty-") |
| 160 | f, _ := newServingLegacyRootFixture(t, empty) |
| 161 | requested := servingLegacyPredecessor{Empty: empty} |
| 162 | switch attack { |
| 163 | case "stop-missing": |
| 164 | if err := os.Remove(filepath.Join(f.disk.dir, servingLegacyKey("legacy-stop", f.current))); err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | case "stop-bytes", "stop-region": |
| 168 | s, err := f.p.ledger.legacyStop(f.current) |
| 169 | if err != nil { |
| 170 | t.Fatal(err) |
| 171 | } |
| 172 | if attack == "stop-bytes" { |
| 173 | s.Result.Bytes = []byte("forged") |
| 174 | } else { |
| 175 | s.Result.Target.Region = "foreign" |
| 176 | } |
| 177 | if err = f.p.ledger.write(servingLegacyKey("legacy-stop", f.current), s); err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | case "first-creator": |
| 181 | requested.Empty = true |
| 182 | case "marker": |
| 183 | if err := os.WriteFile(f.db+".gen", []byte("foreign"), 0600); err != nil { |
| 184 | t.Fatal(err) |
| 185 | } |
| 186 | case "tags": |
| 187 | f.tags(f.current.Stamp) |
| 188 | case "half-tags": |
| 189 | _, err := hostclaim.Update(t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { |
| 190 | rel, _ := filepath.Rel(f.f.guard.DataDir, f.db) |
| 191 | c.Inventory = map[string]hostclaim.InventoryEntry{f.actor.Box: {Layout: "nested", Paths: map[string]uint64{rel: f.current.Stamp}}} |
| 192 | return nil |
| 193 | }) |
| 194 | if err != nil { |
| 195 | t.Fatal(err) |
| 196 | } |
| 197 | case "peer": |
| 198 | f.p.root.Stop.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, ErrClaimStopIdentity } |
| 199 | case "census": |
| 200 | f.p.root.RequireNewWork = func(context.Context) error { return hostbox.ErrChanged } |
| 201 | case "enrollment": |
| 202 | f.actor.Enrollment = "changed" |
| 203 | case "binding": |
| 204 | f.actor.Binding = "8" |
| 205 | case "source": |
| 206 | if err := os.Rename(filepath.Dir(f.db), filepath.Dir(f.db)+".saved"); err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | if err := os.Mkdir(filepath.Dir(f.db), 0700); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | case "main-intention": |
| 213 | target := recoveryTuple(f.admitted, claimStopLocal{Actor: f.actor, BootID: "boot", ControllerID: "controller"}) |
| 214 | if err := f.f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}}); err != nil { |
| 215 | t.Fatal(err) |
| 216 | } |
| 217 | if err := f.f.journal.IntendUnit(t.Context(), hostbox.LaunchReservation{ID: "attempt:main", Role: "main", InvocationID: "attempt", Inputs: "digest", Target: target, Group: hostbox.LaunchGroup{RelativePath: "system.slice/carlos-app@shop.service"}, Executable: hostbox.LaunchFile{Device: 1, Inode: 3}}); err != nil { |
| 218 | t.Fatal(err) |
| 219 | } |
| 220 | case "empty-database", "empty-wal": |
| 221 | suffix := "" |
| 222 | if attack == "empty-wal" { |
| 223 | suffix = "-wal" |
| 224 | } |
| 225 | if err := os.WriteFile(f.db+suffix, []byte("unexpected"), 0600); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | case "empty-remote": |
| 229 | fixture := plan9D5Fixture(t) |
| 230 | fixture.bucket.Put("data", f.admitted.Claim.Holder.Location+"/unexpected", []byte("x")) |
| 231 | f.p.empty = fixture.owners |
| 232 | case "empty-unavailable": |
| 233 | f.p.empty = nil |
| 234 | case "selector": |
| 235 | requested.Session = 1 |
| 236 | requested.Generation = "forged" |
| 237 | } |
| 238 | before, _ := f.disk.Names() |
| 239 | if _, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, requested); err == nil { |
| 240 | t.Fatal("unsafe legacy authority accepted") |
| 241 | } |
| 242 | after, _ := f.disk.Names() |
| 243 | if !reflect.DeepEqual(before, after) { |
| 244 | t.Fatalf("refusal effects: %v -> %v", before, after) |
| 245 | } |
| 246 | if !empty { |
| 247 | db := f.db |
| 248 | if attack == "source" { |
| 249 | db = filepath.Join(filepath.Dir(db)+".saved", filepath.Base(db)) |
| 250 | } |
| 251 | b, err := os.ReadFile(db) |
| 252 | if err != nil || string(b) != "legacy bytes" { |
| 253 | t.Fatalf("source %q %v", b, err) |
| 254 | } |
| 255 | } |
| 256 | }) |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | func TestServingLegacyRootContinuation(t *testing.T) { |
| 261 | for _, point := range []string{"consumption", "intent", "rename", "parked", "retag", "lost-continuation"} { |
| 262 | t.Run(point, func(t *testing.T) { |
| 263 | f, _ := newServingLegacyRootFixture(t, false) |
| 264 | original := f.current |
| 265 | fs := f.p.ledger.fs.(*servingParkFS) |
| 266 | rename := fs.rename |
| 267 | if point == "rename" { |
| 268 | once := false |
| 269 | fs.rename = func(a int, b string, c int, d string, e uint) error { |
| 270 | err := rename(a, b, c, d, e) |
| 271 | if err == nil && !once { |
| 272 | once = true |
| 273 | return os.ErrPermission |
| 274 | } |
| 275 | return err |
| 276 | } |
| 277 | } |
| 278 | if point == "consumption" || point == "intent" { |
| 279 | n := 1 |
| 280 | if point == "intent" { |
| 281 | n = 2 |
| 282 | } |
| 283 | f.p.ledger.state = &servingParkFaultStore{Store: f.disk, failAt: n} |
| 284 | } |
| 285 | a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) |
| 286 | if point == "consumption" || point == "intent" || point == "rename" { |
| 287 | if err == nil { |
| 288 | t.Fatal("fault missed") |
| 289 | } |
| 290 | a.ID = servingRetirementID(servingArtifactIdentity{}, original, true) |
| 291 | } else if err != nil { |
| 292 | t.Fatal(err) |
| 293 | } |
| 294 | fs.rename = rename |
| 295 | f.p.ledger.state = f.disk |
| 296 | before, _ := f.p.ledger.read(a.ID) |
| 297 | if point == "retag" || point == "lost-continuation" { |
| 298 | f.tags(original.Stamp) |
| 299 | } |
| 300 | servingStop(t, f.f, f.f.read()) |
| 301 | o := f.f.read() |
| 302 | later, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G2"})) |
| 303 | if err != nil { |
| 304 | t.Fatal(err) |
| 305 | } |
| 306 | f.current, err = servingWriterIdentity(later, "box-1", "ie") |
| 307 | if err != nil { |
| 308 | t.Fatal(err) |
| 309 | } |
| 310 | f.admitted = later |
| 311 | f.install(later) |
| 312 | f.f.now = f.f.now.Add(time.Hour) |
| 313 | if point == "lost-continuation" { |
| 314 | f.p.ledger.state = &servingParkFaultStore{Store: f.disk, failAt: 1} |
| 315 | if _, err = f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err == nil { |
| 316 | t.Fatal("lost continuation fault missed") |
| 317 | } |
| 318 | f.p.ledger.state = f.disk |
| 319 | } |
| 320 | continued, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) |
| 321 | if err != nil { |
| 322 | t.Fatal(err) |
| 323 | } |
| 324 | if continued.ID != a.ID || continued.OriginalSuccessor != original || continued.Successor != f.current { |
| 325 | t.Fatalf("relabelled: %+v", continued) |
| 326 | } |
| 327 | f.tags(f.current.Stamp) |
| 328 | if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { |
| 329 | t.Fatal(err) |
| 330 | } |
| 331 | after, err := f.p.ledger.read(a.ID) |
| 332 | if err != nil || !after.Complete || len(after.Continuations) != 1 { |
| 333 | t.Fatalf("record %+v %v", after, err) |
| 334 | } |
| 335 | if !before.ParkedAt.IsZero() && !before.ReapAfter.Equal(after.ReapAfter) { |
| 336 | t.Fatal("deadline reset") |
| 337 | } |
| 338 | if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, original); err == nil { |
| 339 | t.Fatal("old successor completed") |
| 340 | } |
| 341 | }) |
| 342 | } |
| 343 | } |
| 344 | |
| 345 | func TestServingLegacyRootProductionRestoreSequence(t *testing.T) { |
| 346 | for _, failure := range []string{"", "enrolled-refusal", "missing-marker"} { |
| 347 | t.Run("first-restore-"+failure, func(t *testing.T) { runServingLegacyRestoreSequence(t, failure) }) |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | func runServingLegacyRestoreSequence(t *testing.T, failure string) { |
| 352 | f, in := newServingLegacyRootFixture(t, false) |
| 353 | fixture := plan12RestoreBaseline(t, f.f) |
| 354 | raw, err := f.f.data.GetVersion(t.Context(), f.f.bucket, fixture.Files[0].Key, fixture.Files[0].VersionID) |
| 355 | if err != nil { |
| 356 | t.Fatal(err) |
| 357 | } |
| 358 | bucket := f.f.data.(*leasetest.VersionedBucket) |
| 359 | rootKey := strings.Replace(fixture.Files[0].Key, "/g-src/", "/", 1) |
| 360 | bucket.Put(f.f.bucket, rootKey, []byte("older invalid snapshot")) |
| 361 | version := bucket.Put(f.f.bucket, rootKey, raw) |
| 362 | client := hostbox.ServingParkClient{Call: servingParkSocket(t, servingTransportRoot(t, f))} |
| 363 | var baseline *genselect.Plan |
| 364 | sentinels := map[string]string{filepath.Join(filepath.Dir(f.db), "files", "payload"): "live file", filepath.Join(filepath.Dir(f.db), ".env"): "environment"} |
| 365 | for path, body := range sentinels { |
| 366 | if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { |
| 367 | t.Fatal(err) |
| 368 | } |
| 369 | if err := os.WriteFile(path, []byte(body), 0600); err != nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | } |
| 373 | var legacy servingParkRecord |
| 374 | for _, generation := range []string{"G", "G2"} { |
| 375 | if generation == "G2" { |
| 376 | prior := f.current |
| 377 | target := recoveryTuple(f.f.read(), claimStopLocal{Actor: f.actor, BootID: "boot", ControllerID: "controller"}) |
| 378 | if err := f.f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 77, ProcessStart: "edge-start", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}}); err != nil { |
| 379 | t.Fatal(err) |
| 380 | } |
| 381 | servingStop(t, f.f, f.f.read()) |
| 382 | o := f.f.read() |
| 383 | admission := hostclaim.Admission{ObservedSession: prior.Session, ObservedGeneration: prior.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: generation} |
| 384 | var err error |
| 385 | f.admitted, err = hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(admission)) |
| 386 | if err != nil { |
| 387 | t.Fatal(err) |
| 388 | } |
| 389 | f.current, err = servingWriterIdentity(f.admitted, "box-1", "ie") |
| 390 | if err != nil { |
| 391 | t.Fatal(err) |
| 392 | } |
| 393 | in.Admission = admission |
| 394 | f.install(f.admitted) |
| 395 | } |
| 396 | plan, err := genselect.Select(t.Context(), f.f.data, genselect.Request{Bucket: f.f.bucket, Region: f.actor.Region, Host: f.current.Host, Holder: *f.f.read().Claim.Holder}) |
| 397 | if err != nil { |
| 398 | t.Fatal(err) |
| 399 | } |
| 400 | if plan.Generation != hostclaim.RootGeneration || len(plan.Files) != 1 || plan.Files[0].Key != rootKey || plan.Files[0].VersionID != version { |
| 401 | t.Fatalf("selected wrong baseline: %+v", plan) |
| 402 | } |
| 403 | if baseline == nil { |
| 404 | baseline = plan |
| 405 | } else if !reflect.DeepEqual(baseline, plan) { |
| 406 | t.Fatal("successor changed selected remote baseline") |
| 407 | } |
| 408 | r := &claimServingArtifactRetirement{Claims: f.f.claims, Guard: f.f.guard, DataDir: f.f.guard.DataDir, Box: f.actor.Box, Region: f.actor.Region, Root: client} |
| 409 | work := servingRootWork{f.f.guard, f.admitted} |
| 410 | restore := hostbox.ColdRestore{Reader: f.f.data, Bucket: f.f.bucket, Generation: generation, Work: work, BeforeLegacyInstall: func(ctx context.Context, _, _ string) error { return r.BeforeLegacyInstall(ctx, f.admitted, in, work) }, BeforeInstallLocked: func(ctx context.Context, _, _ string) error { return r.EnrollLocked(ctx, f.admitted, in) }, RetireArtifacts: func(ctx context.Context, g string) error { return r.Retire(ctx, f.admitted, in, g) }} |
| 411 | if err := r.Recover(t.Context(), f.admitted, in); err != nil { |
| 412 | t.Fatal(err) |
| 413 | } |
| 414 | if generation == "G" && failure == "enrolled-refusal" { |
| 415 | enroll := restore.BeforeInstallLocked |
| 416 | restore.BeforeInstallLocked = func(ctx context.Context, db, gen string) error { |
| 417 | if err := enroll(ctx, db, gen); err != nil { |
| 418 | return err |
| 419 | } |
| 420 | return os.ErrPermission |
| 421 | } |
| 422 | if err := restore.Install(t.Context(), f.db, plan); !errors.Is(err, os.ErrPermission) { |
| 423 | t.Fatalf("enrollment fault: %v", err) |
| 424 | } |
| 425 | if _, err := os.Stat(f.db); !os.IsNotExist(err) { |
| 426 | t.Fatalf("failed restore left database: %v", err) |
| 427 | } |
| 428 | } else { |
| 429 | if err := restore.Install(t.Context(), f.db, plan); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | if got := mwReadDB(t, f.db); got != "selected recovery baseline" { |
| 433 | t.Fatal(got) |
| 434 | } |
| 435 | marker, err := hostbox.ReadGenerationMarker(f.db + ".gen") |
| 436 | if err != nil || marker != generation { |
| 437 | t.Fatalf("marker %q %v", marker, err) |
| 438 | } |
| 439 | // Reproduce the durable state after database rename but before |
| 440 | // marker installation, with the actual restored SQLite database. |
| 441 | if generation == "G" && failure == "missing-marker" { |
| 442 | if err := os.Remove(f.db + ".gen"); err != nil { |
| 443 | t.Fatal(err) |
| 444 | } |
| 445 | } |
| 446 | } |
| 447 | slot, err := f.p.ledger.slot(servingParkAuthorization{Successor: f.current}) |
| 448 | if err != nil { |
| 449 | t.Fatal(err) |
| 450 | } |
| 451 | record, err := f.p.ledger.read(slot.ID) |
| 452 | if err != nil || !record.Authorization.Legacy { |
| 453 | t.Fatalf("legacy displaced %+v %v", record, err) |
| 454 | } |
| 455 | if generation == "G" { |
| 456 | legacy = record |
| 457 | } else if !reflect.DeepEqual(legacy, record) { |
| 458 | t.Fatal("never-launched restore replaced legacy") |
| 459 | } |
| 460 | } |
| 461 | for suffix, want := range map[string]string{"": "legacy bytes", "-wal": "legacy WAL", "-shm": "legacy SHM"} { |
| 462 | b, err := os.ReadFile(filepath.Join(f.f.guard.DataDir, "instances", ".serving-parked", legacy.Authorization.ID, filepath.Base(f.db)+suffix)) |
| 463 | if err != nil || string(b) != want { |
| 464 | t.Fatalf("legacy %q %v", b, err) |
| 465 | } |
| 466 | } |
| 467 | for path, want := range sentinels { |
| 468 | body, err := os.ReadFile(path) |
| 469 | if err != nil || string(body) != want { |
| 470 | t.Fatalf("unrelated artifact changed: %s %v", path, err) |
| 471 | } |
| 472 | } |
| 473 | after, err := f.f.data.GetVersion(t.Context(), f.f.bucket, rootKey, version) |
| 474 | if err != nil || string(after) != string(raw) { |
| 475 | t.Fatal("remote baseline changed", err) |
| 476 | } |
| 477 | |
| 478 | } |
| 479 | |
| 480 | func TestServingLegacyRootConsumedAfterExpiry(t *testing.T) { |
| 481 | f, _ := newServingLegacyRootFixture(t, false) |
| 482 | a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) |
| 483 | if err != nil { |
| 484 | t.Fatal(err) |
| 485 | } |
| 486 | f.tags(f.current.Stamp) |
| 487 | if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | f.f.now = f.f.now.Add(7 * 24 * time.Hour) |
| 491 | if err = f.p.ledger.withNamespace(t.Context(), func() error { return f.p.ledger.reapLocked(t.Context(), a.ID) }); err != nil { |
| 492 | t.Fatal(err) |
| 493 | } |
| 494 | if err = os.WriteFile(f.db, []byte("restored successor"), 0600); err != nil { |
| 495 | t.Fatal(err) |
| 496 | } |
| 497 | _, err = hostclaim.Update(t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { c.Inventory = nil; return nil }) |
| 498 | if err != nil { |
| 499 | t.Fatal(err) |
| 500 | } |
| 501 | if _, err = f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err == nil { |
| 502 | t.Fatal("reopened legacy source") |
| 503 | } |
| 504 | b, err := os.ReadFile(f.db) |
| 505 | if err != nil || string(b) != "restored successor" { |
| 506 | t.Fatalf("source %q %v", b, err) |
| 507 | } |
| 508 | } |
| 509 | |
| 510 | func TestServingLegacyRootAdmittedPredecessor(t *testing.T) { |
| 511 | f := newServingRootFixture(t, true) |
| 512 | if err := os.Remove(f.db + ".gen"); err != nil { |
| 513 | t.Fatal(err) |
| 514 | } |
| 515 | _, err := hostclaim.Update(t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { c.Inventory = nil; return nil }) |
| 516 | if err != nil { |
| 517 | t.Fatal(err) |
| 518 | } |
| 519 | a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{Session: f.prior.Session, Generation: f.prior.Generation}) |
| 520 | if err != nil { |
| 521 | t.Fatal(err) |
| 522 | } |
| 523 | f.tags(f.current.Stamp) |
| 524 | if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { |
| 525 | t.Fatal(err) |
| 526 | } |
| 527 | r, err := f.p.ledger.read(a.ID) |
| 528 | if err != nil || !r.Complete || r.LegacyEvidence.NoProcess != nil || len(r.Receipts) != 1 { |
| 529 | t.Fatalf("record %+v %v", r, err) |
| 530 | } |
| 531 | } |
| 532 | |
| 533 | func TestServingLegacyRootContinuationNeedsStop(t *testing.T) { |
| 534 | f, _ := newServingLegacyRootFixture(t, false) |
| 535 | a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) |
| 536 | if err != nil { |
| 537 | t.Fatal(err) |
| 538 | } |
| 539 | original := f.current |
| 540 | o := f.f.read() |
| 541 | later, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G2"})) |
| 542 | if err != nil { |
| 543 | t.Fatal(err) |
| 544 | } |
| 545 | f.current, err = servingWriterIdentity(later, "box-1", "ie") |
| 546 | if err != nil { |
| 547 | t.Fatal(err) |
| 548 | } |
| 549 | f.install(later) |
| 550 | if _, err = f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}); err == nil { |
| 551 | t.Fatal("continued without stop") |
| 552 | } |
| 553 | r, err := f.p.ledger.read(a.ID) |
| 554 | if err != nil || r.activeAuthorization().Successor != original || len(r.Continuations) != 0 { |
| 555 | t.Fatalf("changed record %+v %v", r, err) |
| 556 | } |
| 557 | } |
| 558 | |
| 559 | func TestServingLegacyRootEmptyContinuationAfterConsumption(t *testing.T) { |
| 560 | f, _ := newServingLegacyRootFixture(t, true) |
| 561 | original := f.current |
| 562 | f.p.ledger.state = &servingParkFaultStore{Store: f.disk, failAt: 1} |
| 563 | if _, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{Empty: true}); err == nil { |
| 564 | t.Fatal("consumption fault missed") |
| 565 | } |
| 566 | f.p.ledger.state = f.disk |
| 567 | servingStop(t, f.f, f.f.read()) |
| 568 | o := f.f.read() |
| 569 | later, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: "box-1", Region: "ie", PredecessorRegion: "ie", Mode: "cold", NewGeneration: "G2"})) |
| 570 | if err != nil { |
| 571 | t.Fatal(err) |
| 572 | } |
| 573 | f.current, err = servingWriterIdentity(later, "box-1", "ie") |
| 574 | if err != nil { |
| 575 | t.Fatal(err) |
| 576 | } |
| 577 | f.install(later) |
| 578 | a, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{}) |
| 579 | if err != nil { |
| 580 | t.Fatal(err) |
| 581 | } |
| 582 | if a.OriginalSuccessor != original { |
| 583 | t.Fatal("lost first-creator identity") |
| 584 | } |
| 585 | f.tags(f.current.Stamp) |
| 586 | if err = f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { |
| 587 | t.Fatal(err) |
| 588 | } |
| 589 | slot, err := f.p.ledger.slot(a) |
| 590 | if err != nil || slot.ID != "" { |
| 591 | t.Fatalf("empty bootstrap took retained slot: %+v %v", slot, err) |
| 592 | } |
| 593 | } |
| 594 | |
| 595 | type servingEmptyRaceReader struct { |
| 596 | genselect.Reader |
| 597 | during func() |
| 598 | } |
| 599 | |
| 600 | func (r servingEmptyRaceReader) ListVersions(ctx context.Context, bucket, prefix string) ([]lease.ObjectVersion, error) { |
| 601 | versions, err := r.Reader.ListVersions(ctx, bucket, prefix) |
| 602 | if r.during != nil { |
| 603 | r.during() |
| 604 | } |
| 605 | return versions, err |
| 606 | } |
| 607 | func TestServingLegacyRootEmptyProofClaimRace(t *testing.T) { |
| 608 | f, _ := newServingLegacyRootFixture(t, true) |
| 609 | f.p.empty.(*claimServingOwners).Data = servingEmptyRaceReader{Reader: f.p.empty.(*claimServingOwners).Data, during: func() { |
| 610 | _, err := hostclaim.Update(t.Context(), f.f.claims, f.f.read(), func(c *hostclaim.Claim) error { return nil }) |
| 611 | if err != nil { |
| 612 | t.Fatal(err) |
| 613 | } |
| 614 | }} |
| 615 | before, _ := f.disk.Names() |
| 616 | if _, err := f.p.parkLegacyServingDatabase(t.Context(), f.conn, f.current, servingLegacyPredecessor{Empty: true}); err == nil { |
| 617 | t.Fatal("stale first-creator proof authorized capture") |
| 618 | } |
| 619 | after, _ := f.disk.Names() |
| 620 | if !reflect.DeepEqual(before, after) { |
| 621 | t.Fatalf("stale proof wrote records: %v", after) |
| 622 | } |
| 623 | } |
| 624 | |
| 625 | func TestServingLegacyRootNoProcessRetryKeepsOriginalEvidence(t *testing.T) { |
| 626 | for _, changedIdentity := range []bool{false, true} { |
| 627 | t.Run(fmt.Sprint(changedIdentity), func(t *testing.T) { |
| 628 | newServingLegacyRootFixture(t, false, func(f *servingRootFixture, c *ClaimStopCoordinator, req StopRequest) { |
| 629 | o := f.f.read() |
| 630 | a := servingArtifactIdentity{Host: o.Host, Box: req.Box, Region: req.Region, Incarnation: req.Incarnation} |
| 631 | old, err := f.p.ledger.legacyStop(a) |
| 632 | if err != nil { |
| 633 | t.Fatal(err) |
| 634 | } |
| 635 | reserve := hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{HolderIncarnation: req.Incarnation, EpisodeID: "first-episode", Attempt: hostclaim.WakeRetryAttempt{ID: "first-attempt", Box: req.Box, Enrollment: req.Enrollment, BootID: req.BootID, ControllerID: req.ControllerID, InvocationID: "invocation", LaunchInputs: "digest"}, Now: f.f.now}) |
| 636 | next, err := hostclaim.Update(t.Context(), f.f.claims, o, func(claim *hostclaim.Claim) error { |
| 637 | if err := reserve(claim); err != nil { |
| 638 | return err |
| 639 | } |
| 640 | if changedIdentity { |
| 641 | claim.Holder.Location += "/" + claim.Holder.Incarnation |
| 642 | } |
| 643 | return nil |
| 644 | }) |
| 645 | if err != nil { |
| 646 | t.Fatal(err) |
| 647 | } |
| 648 | f.f.peer.BootID, f.f.peer.ControllerID = "next-boot", "next-controller" |
| 649 | req.ExpectedClaimETag, req.BootID, req.ControllerID = next.ETag, f.f.peer.BootID, f.f.peer.ControllerID |
| 650 | _, err = c.FenceAndStop(t.Context(), req) |
| 651 | if (err != nil) != changedIdentity { |
| 652 | t.Fatalf("repeated no-process stop: %v (changed identity %t)", err, changedIdentity) |
| 653 | } |
| 654 | after, err := f.p.ledger.legacyStop(a) |
| 655 | if err != nil || !reflect.DeepEqual(after, old) { |
| 656 | t.Fatal("replaced original no-process evidence") |
| 657 | } |
| 658 | }) |
| 659 | }) |
| 660 | } |
| 661 | } |
| 662 | |
amadan