| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostbox" |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostguard" |
| 15 | "github.com/carlosframework/platform/internal/hostops" |
| 16 | "github.com/carlosframework/platform/internal/provision" |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 19 | "github.com/carlosframework/platform/internal/rootstate" |
| 20 | ) |
| 21 | |
| 22 | var errRootInterrupted = errors.New("root interrupted before its commit") |
| 23 | |
| 24 | func (w *assemblyWorld) pending() int { |
| 25 | w.t.Helper() |
| 26 | pending, err := w.assembly.Guard.Root.PendingInstallations() |
| 27 | if err != nil { |
| 28 | w.t.Fatal(err) |
| 29 | } |
| 30 | return len(pending) |
| 31 | } |
| 32 | |
| 33 | // interruptedRegistration is the agent's state registration for rec through |
| 34 | // root's commitRootInventory, with root dying after its intent is durable |
| 35 | // and before it records the commit. |
| 36 | func (w *assemblyWorld) interruptedRegistration(agent *bindingAgent, rec provision.Record, route registry.Route, actor hostbox.Actor) { |
| 37 | w.t.Helper() |
| 38 | before := w.pending() |
| 39 | *w.interrupt = 1 |
| 40 | err := agent.registerState(rec.Host, route.DBPath, actor) |
| 41 | *w.interrupt = -1 |
| 42 | if err == nil || w.pending() != before+1 { |
| 43 | w.t.Fatalf("registration was not interrupted after its intent: %v, pending %d", err, w.pending()) |
| 44 | } |
| 45 | } |
| 46 | |
| 47 | // interruptedRebind leaves a recovery rebind intent for rec's installation |
| 48 | // stamped with stamp: the claim moves to a recovered successor stamp, and the |
| 49 | // guard's RebindRootInventory (the write rebindRecoveryInventory makes) |
| 50 | // persists the intent and is interrupted before its commit. |
| 51 | func (w *assemblyWorld) interruptedRebind(rec provision.Record, stamp string) recoveryRebindRequest { |
| 52 | w.t.Helper() |
| 53 | ctx := w.t.Context() |
| 54 | o := w.claim(rec.Host) |
| 55 | prior := hostguard.Captured{Incarnation: o.Claim.Holder.Incarnation, Stamp: o.Claim.Holder.Stamp} |
| 56 | _, revision := w.inventory(rec.Host) |
| 57 | next, err := hostclaim.Update(ctx, w.store, o, func(c *hostclaim.Claim) error { |
| 58 | c.Holder.Stamp, c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = prior.Stamp+1, prior.Stamp+1, "recovered", "box-1" |
| 59 | c.Holder.Admissions = []hostclaim.AdmissionRecord{{Region: "ie", Session: prior.Stamp + 1, Box: "box-1", Generation: "recovered", Mode: "cold"}} |
| 60 | return nil |
| 61 | }) |
| 62 | if err != nil { |
| 63 | w.t.Fatal(err) |
| 64 | } |
| 65 | h := next.Claim.Holder |
| 66 | _, recordETag, err := w.store.Get(ctx, h.Key) |
| 67 | if err != nil { |
| 68 | w.t.Fatal(err) |
| 69 | } |
| 70 | q := recoveryRebindRequest{V: 1, Prior: prior, Revision: revision, RecordKey: h.Key, RecordETag: recordETag, Next: hostbox.ColdSuccessor{Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Generation: h.Generation, Box: "box-1", Enrollment: w.roster.row().Enrollment, DirectoryRevision: stamp, ETag: next.ETag}} |
| 71 | raw, err := json.Marshal(recoveryRebindIntent{Recovery: &q}) |
| 72 | if err != nil { |
| 73 | w.t.Fatal(err) |
| 74 | } |
| 75 | installation := hostguard.RootInstallation{Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Account: h.Account, App: h.App, Record: h.Key, Box: "box-1", Enrollment: q.Next.Enrollment, Migration: 1, InventoryRevision: 1} |
| 76 | before := w.pending() |
| 77 | err = w.assembly.Guard.RebindRootInventory(ctx, prior, revision, installation, raw, func(*hostclaim.Claim) error { return nil }, func() error { return errRootInterrupted }) |
| 78 | if !errors.Is(err, errRootInterrupted) || w.pending() != before+1 { |
| 79 | w.t.Fatalf("rebind intent: %v, pending %d", err, w.pending()) |
| 80 | } |
| 81 | return q |
| 82 | } |
| 83 | |
| 84 | // migrationRequired gives root a completed Plan 14 migration for actor, |
| 85 | // seen while w.migrating is set, so OpenProductionClaimAssembly runs its |
| 86 | // startup inventory recovery and its migration census |
| 87 | // (verifyRootMigrationStartup) with participation required. The |
| 88 | // participation ledger and the migration record are root-owned directories |
| 89 | // on a box; here they are test directories. The census's last step, the |
| 90 | // installed floor binaries' check, needs root-owned binaries this host does |
| 91 | // not have and is the one step stood in for. |
| 92 | func (w *assemblyWorld) migrationRequired(actor hostbox.Actor) { |
| 93 | w.t.Helper() |
| 94 | census := rootparticipation.Census{Schema: 1, Box: actor.Box, Enrollment: actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true} |
| 95 | participation := &rollbackDisk{dir: w.t.TempDir()} |
| 96 | raw, _ := json.Marshal(census) |
| 97 | if err := participation.Write("census.json", raw); err != nil { |
| 98 | w.t.Fatal(err) |
| 99 | } |
| 100 | now := time.Now().UTC() |
| 101 | record := rootMigrationRecord{Schema: 1, Identity: rootMigrationIdentity{Deployment: actor.ControlBucket, Box: actor.Box, Enrollment: actor.Enrollment, Generation: 1}, Phase: "complete", Snapshot: rootMigrationSnapshot{Complete: true, DirectoryRevision: 1, Exclusion: "exclusion", Items: []rootMigrationItem{}, |
| 102 | Report: hostops.CensusReport{Box: actor.Box, Enrollment: actor.Enrollment, BootID: "boot", Reservation: "migration", DirectoryRevision: 1, CapturedAt: now, Complete: true, Installations: []hostops.CensusInstallation{}, Seeds: &hostops.CensusSeeds{Journaled: []string{}, Gated: []string{}}}}} |
| 103 | state := &rollbackDisk{dir: w.t.TempDir()} |
| 104 | raw, _ = json.Marshal(record) |
| 105 | if err := state.Write("migration.json", raw); err != nil { |
| 106 | w.t.Fatal(err) |
| 107 | } |
| 108 | original, originalState, originalFloors := openRootMigrationParticipation, openRootMigrationState, rootMigrationStartupFloors |
| 109 | w.t.Cleanup(func() { |
| 110 | openRootMigrationParticipation, openRootMigrationState, rootMigrationStartupFloors = original, originalState, originalFloors |
| 111 | }) |
| 112 | openRootMigrationParticipation = func() (rootparticipation.Ledger, bool, error) { |
| 113 | if !w.migrating { |
| 114 | return rootparticipation.Ledger{}, false, nil |
| 115 | } |
| 116 | return rootparticipation.Ledger{State: participation}, true, nil |
| 117 | } |
| 118 | openRootMigrationState = func() (rootstate.Store, error) { return state, nil } |
| 119 | rootMigrationStartupFloors = func(string, rootFloorIdentity) error { return nil } |
| 120 | } |
| 121 | |
| 122 | // plantInventoryIntent leaves q pending at root for installation, as a root |
| 123 | // that died after persisting the intent and before its commit leaves it: |
| 124 | // the guard's real CommitRootInventoryArtifacts persists the intent, and the |
| 125 | // commit fails before any inventory is written. |
| 126 | func (w *assemblyWorld) plantInventoryIntent(q hostbox.JournalRequest, installation hostguard.RootInstallation) { |
| 127 | w.t.Helper() |
| 128 | raw, err := json.Marshal(q) |
| 129 | if err != nil { |
| 130 | w.t.Fatal(err) |
| 131 | } |
| 132 | before := w.pending() |
| 133 | err = w.assembly.Guard.CommitRootInventoryArtifacts(w.t.Context(), installation, raw, func(*hostclaim.Claim) error { return nil }, nil, func(hostguard.RootInstallation) error { return errRootInterrupted }, nil) |
| 134 | if !errors.Is(err, errRootInterrupted) || w.pending() != before+1 { |
| 135 | w.t.Fatalf("planted intent: %v, pending %d", err, w.pending()) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // inventoryRequest is an inventory write for rec's current installation as |
| 140 | // the agent would send it: inventory in, at revision, against the claim and |
| 141 | // record as they are now. |
| 142 | func (w *assemblyWorld) inventoryRequest(rec provision.Record, in hostbox.StopInventory, revision uint64, id string) hostbox.JournalRequest { |
| 143 | w.t.Helper() |
| 144 | o := w.claim(rec.Host) |
| 145 | h := o.Claim.Holder |
| 146 | _, recordETag, err := w.store.Get(w.t.Context(), h.Key) |
| 147 | if err != nil { |
| 148 | w.t.Fatal(err) |
| 149 | } |
| 150 | return hostbox.JournalRequest{V: 3, ID: id, Op: "inventory-write", Revision: revision, RecordKey: h.Key, RecordETag: recordETag, Inventory: in, |
| 151 | Target: hostguard.FenceTarget{Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "box-1", Enrollment: w.roster.row().Enrollment, ClaimETag: o.ETag, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase}} |
| 152 | } |
| 153 | |
| 154 | // plantedFirstInstall leaves rec's first inventory write pending at root, |
| 155 | // stamped stamp, with no inventory written. |
| 156 | func (w *assemblyWorld) plantedFirstInstall(rec provision.Record, stamp string) { |
| 157 | w.t.Helper() |
| 158 | h := w.claim(rec.Host).Claim.Holder |
| 159 | enrollment := w.roster.row().Enrollment |
| 160 | q := w.inventoryRequest(rec, hostbox.StopInventory{Complete: true, Host: rec.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: enrollment, DirectoryRevision: stamp}, 0, "planted-"+rec.Host) |
| 161 | w.plantInventoryIntent(q, hostguard.RootInstallation{Empty: true, Host: rec.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Account: h.Account, App: h.App, Record: h.Key, Box: "box-1", Enrollment: enrollment, Migration: 1, InventoryRevision: 1}) |
| 162 | } |
| 163 | |
| 164 | // targetStampedWrite is agent's inventory write of rec's current inventory, |
| 165 | // unchanged, through the agent's own journal client, with its target's |
| 166 | // DirectoryRevision set to stamp: an agent-chosen request field that root |
| 167 | // must not take on trust. The request is prepared (nonce, issue time, ID) |
| 168 | // by the client exactly as any agent write is. |
| 169 | func (w *assemblyWorld) targetStampedWrite(agent *bindingAgent, rec provision.Record, stamp string) error { |
| 170 | w.t.Helper() |
| 171 | ctx := w.t.Context() |
| 172 | h := w.claim(rec.Host).Claim.Holder |
| 173 | _, etag, err := w.store.Get(ctx, h.Key) |
| 174 | if err != nil { |
| 175 | w.t.Fatal(err) |
| 176 | } |
| 177 | captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} |
| 178 | in, revision, err := agent.remote.Snapshot(ctx, rec.Host, captured) |
| 179 | if err != nil { |
| 180 | w.t.Fatal(err) |
| 181 | } |
| 182 | client := hostbox.ReconcileAdapter{Reconciler: hostbox.Reconciler{Guard: agent.guard, Claims: w.store}, Box: agent.pinned.Box, Enrollment: agent.pinned.Enrollment} |
| 183 | expected, err := client.CaptureInventory(ctx, hostbox.RecordSelection{Host: rec.Host, Key: h.Key, ETag: etag, Incarnation: h.Incarnation, Family: h.Family}, captured, revision) |
| 184 | if err != nil { |
| 185 | w.t.Fatal(err) |
| 186 | } |
| 187 | expected.Target.DirectoryRevision = stamp |
| 188 | _, err = agent.remote.WriteExpected(ctx, in, expected) |
| 189 | return err |
| 190 | } |
| 191 | |
| 192 | // N3 (roster binding task 5b, fix round 1 finding 1): interrupted intents |
| 193 | // are left pending at B0 (an inventory write whose commit landed before the |
| 194 | // record write was lost, a first install likewise, a first install whose |
| 195 | // journal never committed, and a recovery rebind); the box is quarantined |
| 196 | // and root restarts. Each intent is settled and logged, root comes up (never |
| 197 | // errRootMigrationBusy), and its stop service answers a stop of the B0 |
| 198 | // installation. After readmission and another restart root is still up and |
| 199 | // the B0 installation is refused at wake. The control: an interrupted intent |
| 200 | // whose binding did not change replays and completes. |
| 201 | // |
| 202 | // Two restarts settle pending intents in production: the startup inventory |
| 203 | // recovery inside OpenProductionClaimAssembly, followed there by the |
| 204 | // migration census (verifyRootMigrationStartup), either of which aborts the |
| 205 | // assembly; and the service's RecoverPending. "startup" restarts the real |
| 206 | // assembly with migration participation required through the whole |
| 207 | // construction, so the census inspects every settled installation; |
| 208 | // "service" restarts without participation and settles through the root |
| 209 | // handler's RecoverPending. |
| 210 | func TestRootStartupSettlesObsoleteIntents(t *testing.T) { |
| 211 | for _, path := range []string{"startup", "service"} { |
| 212 | t.Run(path, func(t *testing.T) { |
| 213 | ctx := t.Context() |
| 214 | w := newAssemblyWorld(t) |
| 215 | b0 := w.fresh() |
| 216 | if path == "startup" { |
| 217 | w.migrationRequired(b0) |
| 218 | } |
| 219 | w.mustStart() |
| 220 | agent := w.agent(b0) |
| 221 | recA, routeA := w.newHost("a") |
| 222 | recD, routeD := w.newHost("d") |
| 223 | recE, routeE := w.newHost("e") |
| 224 | for _, h := range []struct { |
| 225 | rec provision.Record |
| 226 | route registry.Route |
| 227 | }{{recA, routeA}, {recD, routeD}, {recE, routeE}} { |
| 228 | if err := agent.install(h.rec, h.route); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | } |
| 232 | recover := func() error { |
| 233 | if path == "startup" { |
| 234 | w.migrating = true |
| 235 | return w.restart() |
| 236 | } |
| 237 | if err := w.restart(); err != nil { |
| 238 | return err |
| 239 | } |
| 240 | return w.root.RecoverPending(ctx) |
| 241 | } |
| 242 | |
| 243 | // Control: same binding, interrupted, replays and completes. (The |
| 244 | // startup inventory recovery replays through root's creation |
| 245 | // history, a root-owned directory; the control runs through the |
| 246 | // service, before participation is required.) |
| 247 | w.roster.holdChurn() |
| 248 | _, before := w.inventory(recE.Host) |
| 249 | w.interruptedRegistration(agent, recE, routeE, w.fresh()) |
| 250 | if err := w.restart(); err != nil { |
| 251 | t.Fatal("restart with a same-binding intent:", err) |
| 252 | } |
| 253 | if err := w.root.RecoverPending(ctx); err != nil { |
| 254 | t.Fatal("recovery of a same-binding intent:", err) |
| 255 | } |
| 256 | if _, after := w.inventory(recE.Host); w.pending() != 0 || after != before+1 { |
| 257 | t.Fatalf("same-binding intent: pending %d, inventory revision %d -> %d", w.pending(), before, after) |
| 258 | } |
| 259 | |
| 260 | // Obsolete intents at B0, then quarantine and restart. |
| 261 | agent = w.agent(b0) |
| 262 | _, beforeA := w.inventory(recA.Host) |
| 263 | w.interruptedRegistration(agent, recA, routeA, w.fresh()) |
| 264 | if _, after := w.inventory(recA.Host); after != beforeA+1 { |
| 265 | t.Fatalf("interrupted registration did not land in the inventory: %d -> %d", beforeA, after) |
| 266 | } |
| 267 | recF, routeF := w.newHost("f") |
| 268 | *w.interrupt = 1 |
| 269 | err := agent.install(recF, routeF) |
| 270 | *w.interrupt = -1 |
| 271 | if err == nil || w.pending() != 2 { |
| 272 | t.Fatalf("first install was not interrupted after its intent: %v, pending %d", err, w.pending()) |
| 273 | } |
| 274 | recG, _ := w.newHost("g") |
| 275 | w.plantedFirstInstall(recG, b0.Binding) |
| 276 | w.interruptedRebind(recD, b0.Binding) |
| 277 | w.roster.quarantine("loss_1") |
| 278 | logs := captureLog(t) |
| 279 | if err := recover(); err != nil { |
| 280 | t.Fatal("quarantined restart with obsolete intents:", err) |
| 281 | } |
| 282 | if w.pending() != 0 { |
| 283 | t.Fatalf("obsolete intents left pending: %d", w.pending()) |
| 284 | } |
| 285 | for _, kind := range []string{"inventory", "rebind"} { |
| 286 | if !strings.Contains(logs(), "obsolete pending "+kind+" intent: stamped "+b0.Binding) { |
| 287 | t.Fatalf("no settlement log for the %s intent: %s", kind, logs()) |
| 288 | } |
| 289 | } |
| 290 | if strings.Contains(logs(), "not settled") { |
| 291 | t.Fatalf("a settlement failed: %s", logs()) |
| 292 | } |
| 293 | // Guard and inventory agree for every settled installation, and |
| 294 | // each still refuses execution. |
| 295 | views, err := w.assembly.Guard.Root.MigrationInstallations(ctx) |
| 296 | if err != nil { |
| 297 | t.Fatal("migration census after settlement:", err) |
| 298 | } |
| 299 | settled := map[string]hostguard.FenceView{} |
| 300 | for _, v := range views { |
| 301 | if v.Settled { |
| 302 | settled[v.Host] = v |
| 303 | } |
| 304 | } |
| 305 | for _, rec := range []provision.Record{recA, recD, recF, recG} { |
| 306 | v, ok := settled[rec.Host] |
| 307 | if !ok { |
| 308 | t.Fatalf("%s not settled: %+v", rec.Host, views) |
| 309 | } |
| 310 | _, revision, err := w.assembly.Journals.Inventory.Snapshot(ctx, rec.Host, v.Captured) |
| 311 | if err != nil || revision != v.Installation.InventoryRevision { |
| 312 | t.Fatalf("%s: guard revision %d, inventory %d %v", rec.Host, v.Installation.InventoryRevision, revision, err) |
| 313 | } |
| 314 | if _, err := w.assembly.Guard.Root.View(ctx, rec.Host, v.Captured); !errors.Is(err, hostguard.ErrFenced) { |
| 315 | t.Fatalf("%s: settled installation's execution view: %v", rec.Host, err) |
| 316 | } |
| 317 | } |
| 318 | if v := settled[recA.Host]; v.Installation.InventoryRevision != beforeA+1 { |
| 319 | t.Fatalf("landed registration not recorded: %d", v.Installation.InventoryRevision) |
| 320 | } |
| 321 | if v := settled[recF.Host]; v.Installation.InventoryRevision != 1 { |
| 322 | t.Fatalf("landed first install not recorded: %d", v.Installation.InventoryRevision) |
| 323 | } |
| 324 | if v := settled[recG.Host]; v.Installation.InventoryRevision != 0 { |
| 325 | t.Fatalf("unlanded first install: %d", v.Installation.InventoryRevision) |
| 326 | } |
| 327 | if _, err := w.root.InventoryActor(ctx); err == nil { |
| 328 | t.Fatal("quarantined root admitted new work") |
| 329 | } |
| 330 | if _, err := w.noProcessStop(recA.Host, b0.Binding, w.roster.read().Roster.Revision); err != nil { |
| 331 | t.Fatal("stop service after settlement:", err) |
| 332 | } |
| 333 | |
| 334 | // Readmission and another restart: up, the B0 installation refused. |
| 335 | w.roster.readmit() |
| 336 | if err := recover(); err != nil { |
| 337 | t.Fatal("readmitted restart:", err) |
| 338 | } |
| 339 | b1 := w.fresh() |
| 340 | if err := w.agent(b1).registerState(recE.Host, routeE.DBPath, b1); !errors.Is(err, hostbox.ErrTypedInventory) { |
| 341 | t.Fatalf("the B0 installation at wake after readmission: %v", err) |
| 342 | } |
| 343 | }) |
| 344 | } |
| 345 | } |
| 346 | |
| 347 | // Fix round 1, finding 2: root validates an agent-sent target stamp against |
| 348 | // the inventory's validated binding before it persists any intent, and |
| 349 | // startup decides obsolescence on validated binding evidence only. |
| 350 | // Contradictory: a request whose inventory names B0 (valid) and whose |
| 351 | // target names another binding is refused before any intent, and the |
| 352 | // installation keeps working. Control: the same request with the target |
| 353 | // naming B0, interrupted after its intent, replays and completes at the |
| 354 | // next start. Legacy: an intent persisted with a contradictory target (as |
| 355 | // the unchecked root accepted one) is not settled as obsolete at startup. |
| 356 | func TestRootInventoryRefusesContradictoryTarget(t *testing.T) { |
| 357 | ctx := t.Context() |
| 358 | w := newAssemblyWorld(t) |
| 359 | w.mustStart() |
| 360 | b0 := w.fresh() |
| 361 | recA, routeA := w.newHost("a") |
| 362 | recB, routeB := w.newHost("b") |
| 363 | for _, h := range []struct { |
| 364 | rec provision.Record |
| 365 | route registry.Route |
| 366 | }{{recA, routeA}, {recB, routeB}} { |
| 367 | if err := w.agent(b0).install(h.rec, h.route); err != nil { |
| 368 | t.Fatal(err) |
| 369 | } |
| 370 | } |
| 371 | other := strings.Repeat("ab", 32) |
| 372 | if other == b0.Binding { |
| 373 | t.Fatal("fixture binding collision") |
| 374 | } |
| 375 | restart := func(what string) string { |
| 376 | t.Helper() |
| 377 | logs := captureLog(t) |
| 378 | if err := w.restart(); err != nil { |
| 379 | t.Fatal(what, err) |
| 380 | } |
| 381 | if err := w.root.RecoverPending(ctx); err != nil { |
| 382 | t.Fatal(what, err) |
| 383 | } |
| 384 | return logs() |
| 385 | } |
| 386 | |
| 387 | // Contradictory target: refused before the intent. |
| 388 | _, before := w.inventory(recA.Host) |
| 389 | *w.interrupt = 1 |
| 390 | err := w.targetStampedWrite(w.agent(b0), recA, other) |
| 391 | consumed := *w.interrupt == 0 |
| 392 | *w.interrupt = -1 |
| 393 | if err == nil || w.pending() != 0 || consumed { |
| 394 | t.Fatalf("contradictory target: %v, pending %d, root wrote %v", err, w.pending(), consumed) |
| 395 | } |
| 396 | if _, after := w.inventory(recA.Host); after != before { |
| 397 | t.Fatalf("contradictory target changed the inventory: %d -> %d", before, after) |
| 398 | } |
| 399 | if logs := restart("restart after a contradictory target:"); strings.Contains(logs, "obsolete pending") { |
| 400 | t.Fatalf("an unchanged binding was settled as obsolete: %s", logs) |
| 401 | } |
| 402 | if err := w.agent(b0).registerState(recA.Host, routeA.DBPath, b0); err != nil { |
| 403 | t.Fatal("the installation after a refused contradictory target:", err) |
| 404 | } |
| 405 | |
| 406 | // Control: the target names the validated binding; interrupted, then |
| 407 | // replayed at the next start. |
| 408 | _, before = w.inventory(recB.Host) |
| 409 | *w.interrupt = 1 |
| 410 | err = w.targetStampedWrite(w.agent(b0), recB, b0.Binding) |
| 411 | *w.interrupt = -1 |
| 412 | if err == nil || w.pending() != 1 { |
| 413 | t.Fatalf("same-binding target was not interrupted after its intent: %v, pending %d", err, w.pending()) |
| 414 | } |
| 415 | if logs := restart("restart with a same-binding target:"); strings.Contains(logs, "obsolete pending") { |
| 416 | t.Fatalf("a same-binding intent was settled: %s", logs) |
| 417 | } |
| 418 | if _, after := w.inventory(recB.Host); w.pending() != 0 || after != before+1 { |
| 419 | t.Fatalf("same-binding target: pending %d, inventory revision %d -> %d", w.pending(), before, after) |
| 420 | } |
| 421 | |
| 422 | // Legacy: an intent persisted with a contradictory target. Startup does |
| 423 | // not take its target as obsolescence evidence; the replay is refused |
| 424 | // and the intent stays pending for the ordinary recovery. |
| 425 | in, revision := w.inventory(recA.Host) |
| 426 | q := w.inventoryRequest(recA, in, revision, "legacy-"+recA.Host) |
| 427 | q.Target.DirectoryRevision = other |
| 428 | h := w.claim(recA.Host).Claim.Holder |
| 429 | installed, err := w.assembly.Guard.Root.Installation(ctx, recA.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}) |
| 430 | if err != nil { |
| 431 | t.Fatal(err) |
| 432 | } |
| 433 | installed.InventoryRevision = revision + 1 |
| 434 | w.plantInventoryIntent(q, installed) |
| 435 | if logs := restart("restart with a legacy contradictory intent:"); strings.Contains(logs, "obsolete pending") { |
| 436 | t.Fatalf("a contradictory target was taken as obsolescence evidence: %s", logs) |
| 437 | } |
| 438 | if w.pending() != 1 { |
| 439 | t.Fatalf("legacy contradictory intent: pending %d", w.pending()) |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | // Fix round 2: an intent whose claim advanced before recovery is abandoned |
| 444 | // (the guard's exact-intent and fresh-claim checks), and the abandoned |
| 445 | // installation ends settled like an obsolete one, so it never blocks |
| 446 | // cleanup startup. The binding never changes here. A landed inventory write |
| 447 | // whose claim then moved to an operation session, and a rebind intent whose |
| 448 | // claim then advanced, are abandoned by the service's RecoverPending. Root |
| 449 | // then restarts twice with migration participation required through the |
| 450 | // whole OpenProductionClaimAssembly: its startup census inspects both |
| 451 | // abandoned installations, and the terminal state persists across restarts. |
| 452 | // (Abandonment runs in the service, as it can in production: the startup |
| 453 | // recovery's replays need root's creation history and the guard's lock |
| 454 | // directory, root-owned paths this test host does not have.) |
| 455 | func TestRootStartupInspectsSupersededIntents(t *testing.T) { |
| 456 | ctx := t.Context() |
| 457 | w := newAssemblyWorld(t) |
| 458 | b0 := w.fresh() |
| 459 | w.migrationRequired(b0) |
| 460 | w.mustStart() |
| 461 | agent := w.agent(b0) |
| 462 | recA, routeA := w.newHost("a") |
| 463 | recD, routeD := w.newHost("d") |
| 464 | if err := agent.install(recA, routeA); err != nil { |
| 465 | t.Fatal(err) |
| 466 | } |
| 467 | if err := agent.install(recD, routeD); err != nil { |
| 468 | t.Fatal(err) |
| 469 | } |
| 470 | |
| 471 | // A: a landed inventory write, then the claim moves on. |
| 472 | _, beforeA := w.inventory(recA.Host) |
| 473 | w.interruptedRegistration(agent, recA, routeA, b0) |
| 474 | moved := w.operationSession(recA.Host) |
| 475 | // D: a rebind intent, then the claim moves on. |
| 476 | w.interruptedRebind(recD, b0.Binding) |
| 477 | if _, err := hostclaim.Update(ctx, w.store, w.claim(recD.Host), func(c *hostclaim.Claim) error { |
| 478 | c.Holder.Generation = "advanced" |
| 479 | return nil |
| 480 | }); err != nil { |
| 481 | t.Fatal(err) |
| 482 | } |
| 483 | if err := w.restart(); err != nil { |
| 484 | t.Fatal(err) |
| 485 | } |
| 486 | if err := w.root.RecoverPending(ctx); err != nil { |
| 487 | t.Fatal("service recovery of superseded intents:", err) |
| 488 | } |
| 489 | if w.pending() != 0 { |
| 490 | t.Fatalf("superseded intents left pending: %d", w.pending()) |
| 491 | } |
| 492 | for _, round := range []string{"first restart", "second restart"} { |
| 493 | w.migrating = true |
| 494 | if err := w.restart(); err != nil { |
| 495 | t.Fatalf("%s with migration participation required: %v", round, err) |
| 496 | } |
| 497 | if w.fresh().Binding != b0.Binding { |
| 498 | t.Fatal("the binding changed") |
| 499 | } |
| 500 | } |
| 501 | views, err := w.assembly.Guard.Root.MigrationInstallations(ctx) |
| 502 | if err != nil { |
| 503 | t.Fatal("migration census:", err) |
| 504 | } |
| 505 | settled := map[string]hostguard.FenceView{} |
| 506 | for _, v := range views { |
| 507 | if v.Settled { |
| 508 | settled[v.Host] = v |
| 509 | } |
| 510 | } |
| 511 | for _, rec := range []provision.Record{recA, recD} { |
| 512 | v, ok := settled[rec.Host] |
| 513 | if !ok { |
| 514 | t.Fatalf("%s not settled: %+v", rec.Host, views) |
| 515 | } |
| 516 | _, revision, err := w.assembly.Journals.Inventory.Snapshot(ctx, rec.Host, v.Captured) |
| 517 | if err != nil || revision != v.Installation.InventoryRevision { |
| 518 | t.Fatalf("%s: guard revision %d, inventory %d %v", rec.Host, v.Installation.InventoryRevision, revision, err) |
| 519 | } |
| 520 | if _, err := w.assembly.Guard.Root.View(ctx, rec.Host, v.Captured); !errors.Is(err, hostguard.ErrFenced) { |
| 521 | t.Fatalf("%s: abandoned installation's execution view: %v", rec.Host, err) |
| 522 | } |
| 523 | } |
| 524 | if v := settled[recA.Host]; v.Installation.InventoryRevision != beforeA+1 { |
| 525 | t.Fatalf("landed write not recorded: %d", v.Installation.InventoryRevision) |
| 526 | } |
| 527 | // Startup proceeded over these views; journal-migrate's collector, the |
| 528 | // census's other consumer, still refuses them (census condition 1). |
| 529 | if err := refuseSettledInstallations(views); !errors.Is(err, hostguard.ErrFenced) { |
| 530 | t.Fatal("the collector would admit settled installations:", err) |
| 531 | } |
| 532 | // The stop service is up and stops A's moved-on operation session. |
| 533 | if _, err := w.root.Stop.Coordinator.FenceAndStop(ctx, w.priorStop(moved, b0.Binding)); err != nil { |
| 534 | t.Fatal("stop service after abandonment:", err) |
| 535 | } |
| 536 | } |
| 537 | |
amadan