| 1 | //go:build unix |
| 2 | |
| 3 | package hostbox |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "path/filepath" |
| 9 | "strconv" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostguard" |
| 15 | "github.com/carlosframework/platform/internal/hostwrite" |
| 16 | "github.com/carlosframework/platform/internal/park" |
| 17 | "github.com/carlosframework/platform/internal/provision" |
| 18 | "github.com/carlosframework/platform/internal/registry" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | // sourceReconcile is the box "box" as a relocation's source, built as the |
| 23 | // claim assembly builds it: the production ReconcileAdapter with the real |
| 24 | // census and park.Local, provision's pass, and Task 10b's production |
| 25 | // hooks. Its row is minted by a serving pass, then one cold writer session |
| 26 | // is admitted at the same stamp. |
| 27 | type sourceReconcile struct { |
| 28 | f admissionFixture |
| 29 | reg *registry.Registry |
| 30 | inv *StopInventoryDB |
| 31 | census *EmptyInstallationCensus |
| 32 | guard hostguard.Guard |
| 33 | adapter ReconcileAdapter |
| 34 | cfg provision.ReconcileConfig |
| 35 | } |
| 36 | |
| 37 | func newSourceReconcile(t *testing.T) *sourceReconcile { |
| 38 | t.Helper() |
| 39 | s := &sourceReconcile{f: newAdmissionFixture(t, false)} |
| 40 | var err error |
| 41 | if s.reg, err = registry.Open(filepath.Join(t.TempDir(), "registry.db")); err != nil { |
| 42 | t.Fatal(err) |
| 43 | } |
| 44 | t.Cleanup(func() { s.reg.Close() }) |
| 45 | if s.inv, err = OpenStopInventoryDB(filepath.Join(t.TempDir(), "inventory.db")); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | t.Cleanup(func() { s.inv.Close() }) |
| 49 | launches, err := OpenLaunchJournal(filepath.Join(t.TempDir(), "launches.db")) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | t.Cleanup(func() { launches.Close() }) |
| 54 | s.census = &EmptyInstallationCensus{Inventory: s.inv, Registry: s.reg, Launches: launches, Box: "box", Enrollment: "en", DirectoryRevision: "1", |
| 55 | Roots: map[string]string{"data": s.f.data, "opt": t.TempDir(), "units": t.TempDir()}, SharedPaths: map[string][]string{"data": {"hostguard"}}} |
| 56 | s.guard = hostguard.Guard{DataDir: s.f.data, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: s.f.store}} |
| 57 | source := WorkSource{Guard: s.guard, Box: "box", Region: "ie", Now: time.Now, |
| 58 | Policy: func(context.Context, *hostclaim.Claim, Selection, hostwrite.Identity) error { return nil }, |
| 59 | Local: func(context.Context, *hostclaim.Claim, Selection) error { return nil }} |
| 60 | s.adapter = ReconcileAdapter{Registry: s.reg, Inventory: s.inv, Census: s.census, Box: "box", Enrollment: "en", DirectoryRevision: "1", |
| 61 | Reconciler: Reconciler{Claims: s.f.store, Box: "box", Region: "ie", Guard: s.guard, Writes: hostwrite.Config{Enabled: true, Selector: source}, |
| 62 | Local: park.Local{Guard: s.guard, Claims: s.f.store, DataDir: s.f.data, Box: "box"}, |
| 63 | Now: func() time.Time { return time.Unix(100, 0) }, CheckRecord: func(context.Context, RecordSelection) error { return nil }, CheckLocal: s.census.CheckLocal, |
| 64 | FenceClosed: AdmissionFenceClosed(s.guard), HoldRow: RegistryHoldRow(s.reg)}} |
| 65 | s.cfg = provision.ReconcileConfig{DataDir: s.f.data, Region: "ie"} |
| 66 | s.cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil } |
| 67 | s.adapter.Configure(&s.cfg) |
| 68 | s.pass(t) |
| 69 | if row := s.row(t); row == nil || row.Status != "ready" || row.HoldOp != "" || !row.ClaimOwnership.Valid() { |
| 70 | t.Fatalf("the serving pass minted %+v", row) |
| 71 | } |
| 72 | s.update(t, func(c *hostclaim.Claim) error { |
| 73 | return hostclaim.AdmitWriter(hostclaim.Admission{Box: "box", Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})(c) |
| 74 | }) |
| 75 | return s |
| 76 | } |
| 77 | |
| 78 | func (s *sourceReconcile) pass(t *testing.T) { |
| 79 | t.Helper() |
| 80 | if err := s.f.pass(t.Context(), s.f.store, s.reg, s.cfg); err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | func (s *sourceReconcile) row(t *testing.T) *registry.Route { |
| 86 | t.Helper() |
| 87 | row, err := s.reg.Get(s.f.host) |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | return row |
| 92 | } |
| 93 | |
| 94 | func (s *sourceReconcile) update(t *testing.T, change func(*hostclaim.Claim) error) hostclaim.Observed { |
| 95 | t.Helper() |
| 96 | o, err := hostclaim.Read(t.Context(), s.f.store, s.f.host) |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | if o, err = hostclaim.Update(t.Context(), s.f.store, o, change); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | return o |
| 104 | } |
| 105 | |
| 106 | // releasingThenInstalling moves the claim through a drain from "box" to |
| 107 | // "dst": releasing, root's release stop fencing the source's admission |
| 108 | // (BeginFence), then the installing CAS, which advances holder.stamp to |
| 109 | // the target's install stamp. No reconcile runs on the source in between. |
| 110 | func (s *sourceReconcile) releasingThenInstalling(t *testing.T) hostclaim.Observed { |
| 111 | t.Helper() |
| 112 | ctx := t.Context() |
| 113 | o := s.update(t, func(c *hostclaim.Claim) error { |
| 114 | c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseReleasing, Exec: "box", Participants: []string{"box", "dst"}, |
| 115 | Intent: map[string]string{"region": "ie", "reason": "drain", "source_box": "box", "target_box": "dst"}, Journal: map[string]string{"exec_ack.box": "{}", "exec_ack.dst": "{}"}} |
| 116 | return nil |
| 117 | }) |
| 118 | captured, err := hostguard.Capture(s.f.data, s.f.host) |
| 119 | if err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | target := hostguard.FenceTarget{FenceID: "fence-1", Host: s.f.host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Session: o.Claim.Holder.Session, |
| 123 | Op: "rel-1", Phase: hostclaim.PhaseReleasing, Box: "box", Enrollment: "en", BootID: "boot", ControllerID: "ctl", ClaimETag: o.ETag, DirectoryRevision: "1"} |
| 124 | if _, err := s.guard.BeginFence(ctx, target); err != nil { |
| 125 | t.Fatal(err) |
| 126 | } |
| 127 | return s.update(t, func(c *hostclaim.Claim) error { |
| 128 | c.Holder.Stamp = c.Gen + 1 |
| 129 | c.Op.Phase, c.Op.Exec = hostclaim.PhaseInstalling, "dst" |
| 130 | c.Op.Journal["install_stamp"] = strconv.FormatUint(c.Holder.Stamp, 10) |
| 131 | return nil |
| 132 | }) |
| 133 | } |
| 134 | |
| 135 | // Astra Task 10b round 1 (P2; rev 5.4.1): the source's fence closes in |
| 136 | // releasing, but its first reconcile after that runs only once the claim |
| 137 | // is at installing, with holder.stamp at the target's install stamp, which |
| 138 | // this box never had. Through the production adapter, census and |
| 139 | // provision pass, the source's row is held all the same, its ownership |
| 140 | // unchanged: the hold-only path checks the source's own installation (the |
| 141 | // row's stamp and its inventory), never the holder's stamp, and grants |
| 142 | // nothing, so the guard record and the claim are unchanged. A replay |
| 143 | // changes nothing. |
| 144 | func TestReconcileHoldsTheSourceWhenItsFirstPassAfterTheFenceIsAtInstalling(t *testing.T) { |
| 145 | s := newSourceReconcile(t) |
| 146 | minted := *s.row(t) |
| 147 | o := s.releasingThenInstalling(t) |
| 148 | if o.Claim.Holder.Stamp == minted.ClaimStamp { |
| 149 | t.Fatal("the installing CAS kept the source's stamp") |
| 150 | } |
| 151 | guard := targetGuardFiles(t, s.f.data) |
| 152 | for i := 0; i < 2; i++ { |
| 153 | s.pass(t) |
| 154 | row := s.row(t) |
| 155 | if row == nil || row.Status != "relocating" || row.HoldOp != "rel-1" || row.ClaimOwnership != minted.ClaimOwnership { |
| 156 | t.Fatalf("pass %d at installing: the source's row is %+v (minted %+v)", i, row, minted) |
| 157 | } |
| 158 | held := *row |
| 159 | held.Status, held.HoldOp = minted.Status, minted.HoldOp |
| 160 | if held != minted { |
| 161 | t.Fatalf("pass %d: the hold changed more than status and hold_op: %+v", i, row) |
| 162 | } |
| 163 | if got := targetGuardFiles(t, s.f.data); got[s.f.host+".json"] != guard[s.f.host+".json"] { |
| 164 | t.Fatalf("pass %d: the guard record changed", i) |
| 165 | } |
| 166 | after, err := hostclaim.Read(t.Context(), s.f.store, s.f.host) |
| 167 | if err != nil || after.ETag != o.ETag { |
| 168 | t.Fatalf("pass %d: the claim was written: %v", i, err) |
| 169 | } |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | // The hold-only check (EmptyInstallationCensus.CheckHeld) accepts the |
| 174 | // source's own existing installation at installing, where main's |
| 175 | // CheckLocal refuses (it asks for the target's stamp). Absence is never |
| 176 | // evidence: an inventory of another box or enrollment, no inventory, a |
| 177 | // row owned by another row identity, another incarnation, no row and an |
| 178 | // unreadable registry all refuse. Through the pass, a contradicting |
| 179 | // inventory leaves the row exactly as it was. |
| 180 | func TestCensusCheckHeldRefusesAContradictingOrMissingSourceInventory(t *testing.T) { |
| 181 | ctx := t.Context() |
| 182 | s := newSourceReconcile(t) |
| 183 | minted := *s.row(t) |
| 184 | o := s.releasingThenInstalling(t) |
| 185 | selected := RecordSelection{Host: s.f.host, Incarnation: o.Claim.Holder.Incarnation} |
| 186 | if err := s.census.CheckHeld(ctx, o, selected); err != nil { |
| 187 | t.Fatalf("the source's own installation: %v", err) |
| 188 | } |
| 189 | if err := s.census.CheckLocal(ctx, o, selected); !errors.Is(err, ErrTypedInventory) { |
| 190 | t.Fatalf("main's CheckLocal at installing: %v (it asks for the target's stamp)", err) |
| 191 | } |
| 192 | empty, err := OpenStopInventoryDB(filepath.Join(t.TempDir(), "inventory.db")) |
| 193 | if err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | t.Cleanup(func() { empty.Close() }) |
| 197 | otherRows, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 198 | if err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | t.Cleanup(func() { otherRows.Close() }) |
| 202 | stranger := minted |
| 203 | stranger.ClaimRowID = "another-row" |
| 204 | if err := otherRows.Upsert(stranger); err != nil { |
| 205 | t.Fatal(err) |
| 206 | } |
| 207 | for name, tc := range map[string]struct { |
| 208 | change func(*EmptyInstallationCensus, *RecordSelection) |
| 209 | want error |
| 210 | }{ |
| 211 | "another box": {func(c *EmptyInstallationCensus, _ *RecordSelection) { c.Box = "dst" }, ErrTypedInventory}, |
| 212 | "another enrollment": {func(c *EmptyInstallationCensus, _ *RecordSelection) { c.Enrollment = "en-2" }, ErrTypedInventory}, |
| 213 | "no inventory": {func(c *EmptyInstallationCensus, _ *RecordSelection) { c.Inventory = empty }, ErrTypedInventory}, |
| 214 | "another row id": {func(c *EmptyInstallationCensus, _ *RecordSelection) { c.Registry = otherRows }, ErrTypedInventory}, |
| 215 | "another incarnation": {func(_ *EmptyInstallationCensus, sel *RecordSelection) { sel.Incarnation = "inc-other" }, ErrChanged}, |
| 216 | } { |
| 217 | c, sel := *s.census, selected |
| 218 | tc.change(&c, &sel) |
| 219 | if err := c.CheckHeld(ctx, o, sel); !errors.Is(err, tc.want) { |
| 220 | t.Fatalf("%s: %v, want %v", name, err, tc.want) |
| 221 | } |
| 222 | } |
| 223 | noRow, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 224 | if err != nil { |
| 225 | t.Fatal(err) |
| 226 | } |
| 227 | t.Cleanup(func() { noRow.Close() }) |
| 228 | c := *s.census |
| 229 | c.Registry = noRow |
| 230 | if err := c.CheckHeld(ctx, o, selected); !errors.Is(err, ErrChanged) { |
| 231 | t.Fatalf("no row: %v", err) |
| 232 | } |
| 233 | closed, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 234 | if err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | closed.Close() |
| 238 | c.Registry = closed |
| 239 | if err := c.CheckHeld(ctx, o, selected); err == nil || errors.Is(err, ErrChanged) || errors.Is(err, ErrTypedInventory) { |
| 240 | t.Fatalf("an unreadable registry: %v (want the read's error)", err) |
| 241 | } |
| 242 | // Through the pass: a contradicting inventory refuses the hold. |
| 243 | contradicting := *s.census |
| 244 | contradicting.Enrollment = "en-2" |
| 245 | s.adapter.Census = &contradicting |
| 246 | s.adapter.Configure(&s.cfg) |
| 247 | s.pass(t) |
| 248 | if row := s.row(t); *row != minted { |
| 249 | t.Fatalf("a contradicting inventory: the row is %+v (was %+v)", row, minted) |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | // CheckHeld accepts the source's installation only from a complete |
| 254 | // inventory with exactly one materialized route entry for the host, owned |
| 255 | // by a row for the selected incarnation. The inventory as recorded is |
| 256 | // accepted from a fresh inventory database; the same inventory marked |
| 257 | // incomplete, with its route entry removed, with that entry recorded |
| 258 | // twice, or with that entry only intended is ErrTypedInventory. So is a |
| 259 | // row owned by another incarnation while the claim and the selection |
| 260 | // still match, even with a complete inventory of that incarnation that |
| 261 | // passes every later check. |
| 262 | func TestCensusCheckHeldRefusesAnInventoryThatDoesNotProveTheRowsInstallation(t *testing.T) { |
| 263 | ctx := t.Context() |
| 264 | s := newSourceReconcile(t) |
| 265 | minted := *s.row(t) |
| 266 | o := s.releasingThenInstalling(t) |
| 267 | selected := RecordSelection{Host: s.f.host, Incarnation: o.Claim.Holder.Incarnation} |
| 268 | recorded, err := s.inv.ReadInstallation(ctx, s.f.host, minted.ClaimIncarnation, minted.ClaimStamp) |
| 269 | if err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | isRoute := func(e StopInventoryEntry) bool { return e.Kind == "route" && e.RelativePath == s.f.host } |
| 273 | routes := 0 |
| 274 | for _, e := range recorded.Entries { |
| 275 | if isRoute(e) { |
| 276 | routes++ |
| 277 | } |
| 278 | } |
| 279 | if !recorded.Complete || routes != 1 { |
| 280 | t.Fatalf("the recorded inventory: complete %v, %d route entries", recorded.Complete, routes) |
| 281 | } |
| 282 | for _, tc := range []struct { |
| 283 | name string |
| 284 | change func(*StopInventory) |
| 285 | want error |
| 286 | row func(*registry.Route) |
| 287 | }{ |
| 288 | {"as recorded", func(*StopInventory) {}, nil, nil}, |
| 289 | {"incomplete", func(in *StopInventory) { in.Complete = false }, ErrTypedInventory, nil}, |
| 290 | {"no route entry", func(in *StopInventory) { |
| 291 | kept := []StopInventoryEntry{} |
| 292 | for _, e := range in.Entries { |
| 293 | if !isRoute(e) { |
| 294 | kept = append(kept, e) |
| 295 | } |
| 296 | } |
| 297 | in.Entries = kept |
| 298 | }, ErrTypedInventory, nil}, |
| 299 | {"two route entries", func(in *StopInventory) { |
| 300 | for _, e := range in.Entries { |
| 301 | if isRoute(e) { |
| 302 | e.ArtifactID, e.Inode = e.ArtifactID+"-2", e.Inode+1 |
| 303 | in.Entries = append(in.Entries, e) |
| 304 | break |
| 305 | } |
| 306 | } |
| 307 | }, ErrTypedInventory, nil}, |
| 308 | {"route entry only intended", func(in *StopInventory) { |
| 309 | for i, e := range in.Entries { |
| 310 | if isRoute(e) { |
| 311 | in.Entries[i].Materialization = "intended" |
| 312 | } |
| 313 | } |
| 314 | }, ErrTypedInventory, nil}, |
| 315 | // The row's own incarnation check is the only guard here: the |
| 316 | // inventory is complete and recorded at the row's incarnation and |
| 317 | // stamp, so every later check would accept it. |
| 318 | {"row owned by another incarnation", func(in *StopInventory) { in.Incarnation = "inc-other" }, ErrTypedInventory, |
| 319 | func(r *registry.Route) { r.ClaimIncarnation = "inc-other" }}, |
| 320 | } { |
| 321 | in := recorded |
| 322 | in.Entries = append([]StopInventoryEntry(nil), recorded.Entries...) |
| 323 | tc.change(&in) |
| 324 | db, err := OpenStopInventoryDB(filepath.Join(t.TempDir(), "inventory.db")) |
| 325 | if err != nil { |
| 326 | t.Fatal(err) |
| 327 | } |
| 328 | t.Cleanup(func() { db.Close() }) |
| 329 | if err := db.WriteStopInventory(ctx, in); err != nil { |
| 330 | t.Fatalf("%s: writing the inventory: %v", tc.name, err) |
| 331 | } |
| 332 | c := *s.census |
| 333 | c.Inventory = db |
| 334 | if tc.row != nil { |
| 335 | row := minted |
| 336 | tc.row(&row) |
| 337 | if row.ClaimIncarnation == selected.Incarnation { |
| 338 | t.Fatalf("%s: the row's incarnation still matches the selection", tc.name) |
| 339 | } |
| 340 | rows, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 341 | if err != nil { |
| 342 | t.Fatal(err) |
| 343 | } |
| 344 | t.Cleanup(func() { rows.Close() }) |
| 345 | if err := rows.Upsert(row); err != nil { |
| 346 | t.Fatal(err) |
| 347 | } |
| 348 | c.Registry = rows |
| 349 | } |
| 350 | if err := c.CheckHeld(ctx, o, selected); !errors.Is(err, tc.want) || (tc.want == nil && err != nil) { |
| 351 | t.Fatalf("%s: %v, want %v", tc.name, err, tc.want) |
| 352 | } |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | func TestPlan14RemoteSourceHoldReadsInventoryOutsideHostLock(t *testing.T) { |
| 357 | for _, stale := range []bool{false, true} { |
| 358 | t.Run(strconv.FormatBool(stale), func(t *testing.T) { |
| 359 | s := newSourceReconcile(t) |
| 360 | before := *s.row(t) |
| 361 | s.releasingThenInstalling(t) |
| 362 | captured := hostguard.Captured{Incarnation: before.ClaimIncarnation, Stamp: before.ClaimStamp} |
| 363 | calls := 0 |
| 364 | _, remote := RemoteJournals(func(ctx context.Context, q JournalRequest) (JournalResponse, error) { |
| 365 | if q.Op != "inventory-version" { |
| 366 | t.Fatalf("hold requested %s", q.Op) |
| 367 | } |
| 368 | calls++ |
| 369 | var out JournalResponse |
| 370 | err := s.guard.WithLocal(ctx, s.f.host, func() error { |
| 371 | var err error |
| 372 | out, err = s.inv.inventoryVersion(ctx, q.Target) |
| 373 | out.ID, out.V = q.ID, 1 |
| 374 | return err |
| 375 | }) |
| 376 | return out, err |
| 377 | }) |
| 378 | s.adapter.Inventory = remote |
| 379 | census := *s.census |
| 380 | census.Inventory = remote |
| 381 | s.adapter.Census = &census |
| 382 | s.adapter.Reconciler.Guard.FenceView = func(ctx context.Context, host string, c hostguard.Captured) (hostguard.FenceView, error) { |
| 383 | if c != captured { |
| 384 | return hostguard.FenceView{}, ErrChanged |
| 385 | } |
| 386 | _, revision, err := s.inv.Snapshot(ctx, host, c) |
| 387 | if stale { |
| 388 | revision++ |
| 389 | } |
| 390 | return hostguard.FenceView{Schema: 1, Host: host, Captured: c, Installation: hostguard.RootInstallation{InventoryRevision: revision}}, err |
| 391 | } |
| 392 | s.adapter.Configure(&s.cfg) |
| 393 | ctx, cancel := context.WithTimeout(t.Context(), time.Second) |
| 394 | defer cancel() |
| 395 | if err := s.f.pass(ctx, s.f.store, s.reg, s.cfg); err != nil { |
| 396 | t.Fatal(err) |
| 397 | } |
| 398 | row := s.row(t) |
| 399 | if calls != 1 { |
| 400 | t.Fatalf("inventory RPCs=%d", calls) |
| 401 | } |
| 402 | if ctx.Err() != nil { |
| 403 | t.Fatal("inventory read deadlocked", ctx.Err()) |
| 404 | } |
| 405 | if stale { |
| 406 | if *row != before { |
| 407 | t.Fatal("stale inventory changed row") |
| 408 | } |
| 409 | return |
| 410 | } |
| 411 | if row.Status != "relocating" || row.HoldOp != "rel-1" || row.ClaimOwnership != before.ClaimOwnership { |
| 412 | t.Fatalf("hold: %+v", row) |
| 413 | } |
| 414 | }) |
| 415 | } |
| 416 | } |
| 417 | |
amadan