| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "net/http" |
| 10 | "net/http/httptest" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostbox" |
| 17 | "github.com/carlosframework/platform/internal/hostclaim" |
| 18 | "github.com/carlosframework/platform/internal/hostguard" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | ) |
| 21 | |
| 22 | // operationStop moves host's claim into an operation session this box |
| 23 | // executes (a moving op, with a writer session numbered as an admission at |
| 24 | // this write would number it) by a TEST-ONLY claim transition, then stops |
| 25 | // it through root's real stop coordinator: the prior's completed fence |
| 26 | // receipt, stamped with stamp. |
| 27 | func (w *assemblyWorld) operationStop(host, stamp string) hostclaim.Observed { |
| 28 | w.t.Helper() |
| 29 | o := w.operationSession(host) |
| 30 | if _, err := w.root.Stop.Coordinator.FenceAndStop(w.t.Context(), w.priorStop(o, stamp)); err != nil { |
| 31 | w.t.Fatal("operation stop of the prior:", err) |
| 32 | } |
| 33 | return o |
| 34 | } |
| 35 | |
| 36 | // operationSession is operationStop's TEST-ONLY claim transition alone. |
| 37 | func (w *assemblyWorld) operationSession(host string) hostclaim.Observed { |
| 38 | w.t.Helper() |
| 39 | o, err := hostclaim.Update(w.t.Context(), w.store, w.claim(host), func(c *hostclaim.Claim) error { |
| 40 | h := c.Holder |
| 41 | h.Session, h.WriterBox, h.Generation = c.Gen+1, "box-1", "g5" |
| 42 | h.Admissions = []hostclaim.AdmissionRecord{{Region: "ie", Session: h.Session, Box: "box-1", Generation: "g5", Mode: "fresh"}} |
| 43 | c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "purge-1", Phase: "stopping", Exec: "box-1"} |
| 44 | return nil |
| 45 | }) |
| 46 | if err != nil { |
| 47 | w.t.Fatal(err) |
| 48 | } |
| 49 | return o |
| 50 | } |
| 51 | |
| 52 | // priorStop is the agent's stop request for o's writer session. |
| 53 | func (w *assemblyWorld) priorStop(o hostclaim.Observed, stamp string) StopRequest { |
| 54 | h := o.Claim.Holder |
| 55 | return StopRequest{Region: "ie", Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, Box: "box-1", Enrollment: w.roster.row().Enrollment, |
| 56 | BootID: rootPeer.BootID, ControllerID: rootPeer.ControllerID, ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: stamp, RosterRevision: w.roster.read().Roster.Revision, |
| 57 | OperationID: "purge-1", Phase: "stopping", ObligationID: "pre-teardown-stop"} |
| 58 | } |
| 59 | |
| 60 | // coldAdmit is the executor's real cold writer admission on host's claim. |
| 61 | func (w *assemblyWorld) coldAdmit(host, generation string) hostclaim.Observed { |
| 62 | w.t.Helper() |
| 63 | o := w.claim(host) |
| 64 | h := o.Claim.Holder |
| 65 | next, err := hostclaim.Update(w.t.Context(), w.store, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "box-1", Mode: "cold", NewGeneration: generation, Region: "ie", PredecessorRegion: "ie"})) |
| 66 | if err != nil { |
| 67 | w.t.Fatal(err) |
| 68 | } |
| 69 | return next |
| 70 | } |
| 71 | |
| 72 | // agentGuard is the agent's guard: root's fence view over the socket. |
| 73 | func (w *assemblyWorld) agentGuard(socket string) hostguard.Guard { |
| 74 | return hostguard.Guard{DataDir: w.r.DataDir, LockDir: w.locks, Claims: hostguard.StoreReader{Store: w.store}, FenceView: rootFenceViewClient(rootJournalClient(socket))} |
| 75 | } |
| 76 | |
| 77 | // predecessorReceipt is the prior's completed receipt as the agent's |
| 78 | // production cold admission hook finds it. |
| 79 | func (w *assemblyWorld) predecessorReceipt(socket string, prior hostclaim.Observed) hostguard.FenceReceipt { |
| 80 | w.t.Helper() |
| 81 | h := prior.Claim.Holder |
| 82 | receipt, err := w.agentGuard(socket).CompletedColdPredecessor(w.t.Context(), prior.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, h.Session, prior.Claim.Op.ID, prior.Claim.Op.Phase) |
| 83 | if err != nil { |
| 84 | w.t.Fatal("prior receipt:", err) |
| 85 | } |
| 86 | return receipt |
| 87 | } |
| 88 | |
| 89 | func coldRequestFor(prior hostguard.FenceTarget, admitted hostclaim.Observed, enrollment, stamp string) coldHandoverRequest { |
| 90 | h := admitted.Claim.Holder |
| 91 | return coldHandoverRequest{Prior: prior, Next: hostbox.ColdSuccessor{Host: admitted.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Generation: h.Generation, Box: "box-1", Enrollment: enrollment, DirectoryRevision: stamp, ETag: admitted.ETag, Op: admitted.Claim.Op.ID, Phase: admitted.Claim.Op.Phase}} |
| 92 | } |
| 93 | |
| 94 | // coldJournals is root's cold handover journals by file name and phase. |
| 95 | func (w *assemblyWorld) coldJournals() map[string]string { |
| 96 | w.t.Helper() |
| 97 | out := map[string]string{} |
| 98 | entries, err := os.ReadDir(filepath.Join(w.retire, "cold")) |
| 99 | if errors.Is(err, os.ErrNotExist) { |
| 100 | return out |
| 101 | } |
| 102 | if err != nil { |
| 103 | w.t.Fatal(err) |
| 104 | } |
| 105 | for _, e := range entries { |
| 106 | if filepath.Ext(e.Name()) != ".json" { |
| 107 | continue |
| 108 | } |
| 109 | raw, err := os.ReadFile(filepath.Join(w.retire, "cold", e.Name())) |
| 110 | if err != nil { |
| 111 | w.t.Fatal(err) |
| 112 | } |
| 113 | var j struct{ Phase string } |
| 114 | if err := json.Unmarshal(raw, &j); err != nil { |
| 115 | w.t.Fatal(err) |
| 116 | } |
| 117 | out[e.Name()] = j.Phase |
| 118 | } |
| 119 | return out |
| 120 | } |
| 121 | |
| 122 | // priorIntact asserts nothing of the prior moved: its route row is still |
| 123 | // owned by its installation and root's active stamp is still its own. |
| 124 | func (w *assemblyWorld) priorIntact(what string, prior hostguard.FenceTarget, route registry.Route) { |
| 125 | w.t.Helper() |
| 126 | row, err := w.reg.Get(prior.Host) |
| 127 | if err != nil || row == nil || row.ClaimStamp != prior.Stamp || row.ClaimOwnership != route.ClaimOwnership { |
| 128 | w.t.Fatalf("%s: the prior's row moved: %+v %v", what, row, err) |
| 129 | } |
| 130 | if v := w.view(prior.Host, hostguard.Captured{Incarnation: prior.Incarnation, Stamp: prior.Stamp}); v.Installation.Stamp != prior.Stamp { |
| 131 | w.t.Fatalf("%s: root's active stamp moved: %+v", what, v.Installation) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | // installedRoute is host's registry row as the agent's install left it. |
| 136 | func (w *assemblyWorld) installedRoute(host string) registry.Route { |
| 137 | w.t.Helper() |
| 138 | row, err := w.reg.Get(host) |
| 139 | if err != nil || row == nil { |
| 140 | w.t.Fatalf("no route for %s: %v", host, err) |
| 141 | } |
| 142 | return *row |
| 143 | } |
| 144 | |
| 145 | // N5 (roster binding task 6b): root's operation cold handover admits only a |
| 146 | // successor stamped with today's binding, checked before any successor |
| 147 | // state. Through root's handler over the socket: a B0 prior (installed and |
| 148 | // stopped before a quarantine) hands over to a B1 successor after |
| 149 | // readmission (parked and started); while quarantined the handover is |
| 150 | // refused; a successor at the old binding is refused with no journal file, |
| 151 | // no successor inventory and the prior not parked. |
| 152 | func TestColdHandoverVerifiesSuccessorAtRoot(t *testing.T) { |
| 153 | ctx := t.Context() |
| 154 | w := newAssemblyWorld(t) |
| 155 | w.mustStart() |
| 156 | b0 := w.fresh() |
| 157 | rec, route := w.newHost("a") |
| 158 | if err := w.agent(b0).install(rec, route); err != nil { |
| 159 | t.Fatal(err) |
| 160 | } |
| 161 | route = w.installedRoute(rec.Host) |
| 162 | w.roster.holdChurn() |
| 163 | prior := w.operationStop(rec.Host, b0.Binding) |
| 164 | socket := w.serve() |
| 165 | receipt := w.predecessorReceipt(socket, prior) |
| 166 | if receipt.Target.DirectoryRevision != b0.Binding { |
| 167 | t.Fatalf("prior receipt target stamp %q", receipt.Target.DirectoryRevision) |
| 168 | } |
| 169 | admitted := w.coldAdmit(rec.Host, "g6") |
| 170 | enrollment := w.roster.row().Enrollment |
| 171 | |
| 172 | // Quarantined: refused before any successor state. Through the socket |
| 173 | // ServeHTTP's RequireNewWork answers 403; the handler's own admission |
| 174 | // answers 409. |
| 175 | w.roster.quarantine("loss_1") |
| 176 | quarantined := coldRequestFor(receipt.Target, admitted, enrollment, b0.Binding) |
| 177 | if status := postRecoveryHandover(t, socket, quarantined); status != http.StatusForbidden { |
| 178 | t.Fatalf("quarantined handover through the socket: HTTP %d", status) |
| 179 | } |
| 180 | body, _ := json.Marshal(quarantined) |
| 181 | direct := httptest.NewRecorder() |
| 182 | w.root.serveColdHandover(direct, httptest.NewRequest(http.MethodPost, "http://instance-control/claim/handover", bytes.NewReader(body)).WithContext(ctx), rootPeer) |
| 183 | if direct.Code != http.StatusConflict { |
| 184 | t.Fatalf("quarantined handover at the handler: HTTP %d", direct.Code) |
| 185 | } |
| 186 | |
| 187 | // Readmitted: a successor at the old binding is refused before any state. |
| 188 | w.roster.readmit() |
| 189 | b1 := w.fresh() |
| 190 | if b1.Binding == b0.Binding { |
| 191 | t.Fatal("readmission did not move the binding") |
| 192 | } |
| 193 | for _, stamp := range []string{b0.Binding, strings.Repeat("e", 64)} { |
| 194 | if status := postRecoveryHandover(t, socket, coldRequestFor(receipt.Target, admitted, enrollment, stamp)); status != http.StatusConflict { |
| 195 | t.Fatalf("successor stamped %q: HTTP %d", stamp, status) |
| 196 | } |
| 197 | } |
| 198 | if j := w.coldJournals(); len(j) != 0 { |
| 199 | t.Fatalf("refused handover wrote a journal: %v", j) |
| 200 | } |
| 201 | if _, revision := w.successorInventory(admitted); revision != 0 { |
| 202 | t.Fatal("refused handover wrote a successor inventory") |
| 203 | } |
| 204 | w.priorIntact("refused successor", receipt.Target, route) |
| 205 | |
| 206 | // B0 prior with its receipt, B1 successor through the agent's production |
| 207 | // cold admission hook: parked and started. |
| 208 | if err := coldAdmissionHook(w.agentGuard(socket), w.owners.base.Identity, socket)(ctx, prior, admitted); err != nil { |
| 209 | t.Fatal("B0 prior to B1 successor:", err) |
| 210 | } |
| 211 | successor, revision := w.successorInventory(admitted) |
| 212 | if revision == 0 || successor.DirectoryRevision != b1.Binding { |
| 213 | t.Fatalf("successor inventory: revision %d stamp %q", revision, successor.DirectoryRevision) |
| 214 | } |
| 215 | if row, err := w.reg.Get(rec.Host); err != nil || row == nil || row.ClaimStamp != admitted.Claim.Holder.Stamp { |
| 216 | t.Fatalf("successor not started: %+v %v", row, err) |
| 217 | } |
| 218 | if v := w.view(rec.Host, hostguard.Captured{Incarnation: successor.Incarnation, Stamp: successor.InstallationStamp}); v.Installation.Stamp != admitted.Claim.Holder.Stamp || v.Barrier != nil { |
| 219 | t.Fatalf("root after the handover: %+v barrier %v", v.Installation, v.Barrier) |
| 220 | } |
| 221 | for name, phase := range w.coldJournals() { |
| 222 | if phase != "complete" { |
| 223 | t.Fatalf("journal %s: %s", name, phase) |
| 224 | } |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // N5 extended (B1) and B3: a cold handover interrupted after |
| 229 | // BeginColdHandover (its barrier durable, nothing of the prior moved); the |
| 230 | // box is quarantined and readmitted. The agent's replay of the journaled |
| 231 | // handover is refused (not resumed). Root restarts: startup comes up, its |
| 232 | // RecoverPending does not resume the handover and settles it instead (the |
| 233 | // barrier cleared, the successor stamp retired, the journal marked settled, |
| 234 | // logged). The prior keeps its historical authority: its cleanup stop |
| 235 | // returns its receipt, and a clean successor admitted under B1 takes over |
| 236 | // from it through root's handler, parking the prior. Another restart leaves |
| 237 | // the settled journal alone. |
| 238 | func TestColdHandoverObsoleteBarrierSettledAtStartup(t *testing.T) { |
| 239 | ctx := t.Context() |
| 240 | w := newAssemblyWorld(t) |
| 241 | w.mustStart() |
| 242 | b0 := w.fresh() |
| 243 | rec, route := w.newHost("a") |
| 244 | if err := w.agent(b0).install(rec, route); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | route = w.installedRoute(rec.Host) |
| 248 | prior := w.operationStop(rec.Host, b0.Binding) |
| 249 | socket := w.serve() |
| 250 | receipt := w.predecessorReceipt(socket, prior) |
| 251 | admitted := w.coldAdmit(rec.Host, "g6") |
| 252 | enrollment := w.roster.row().Enrollment |
| 253 | request := coldRequestFor(receipt.Target, admitted, enrollment, b0.Binding) |
| 254 | successorStamp := hostguard.Captured{Incarnation: admitted.Claim.Holder.Incarnation, Stamp: admitted.Claim.Holder.Stamp} |
| 255 | captured := hostguard.Captured{Incarnation: receipt.Target.Incarnation, Stamp: receipt.Target.Stamp} |
| 256 | |
| 257 | // Root dies right after BeginColdHandover's write: the journal's prepare |
| 258 | // write and the barrier write succeed, the prior's retirement write fails. |
| 259 | *w.interrupt = 2 |
| 260 | status := postRecoveryHandover(t, socket, request) |
| 261 | *w.interrupt = -1 |
| 262 | if status != http.StatusConflict { |
| 263 | t.Fatalf("interrupted handover: HTTP %d", status) |
| 264 | } |
| 265 | if v := w.view(rec.Host, captured); v.Barrier == nil || v.Barrier.Stamp != successorStamp.Stamp { |
| 266 | t.Fatalf("no durable barrier after the interruption: %+v", v.Barrier) |
| 267 | } |
| 268 | journals := w.coldJournals() |
| 269 | if len(journals) != 1 { |
| 270 | t.Fatalf("journals %v", journals) |
| 271 | } |
| 272 | for _, phase := range journals { |
| 273 | if phase != "prepared" { |
| 274 | t.Fatalf("interrupted journal phase %q", phase) |
| 275 | } |
| 276 | } |
| 277 | w.priorIntact("interrupted", receipt.Target, route) |
| 278 | |
| 279 | // Binding change, then the agent's replay: refused, not resumed. |
| 280 | w.roster.quarantine("loss_1") |
| 281 | if status := postRecoveryHandover(t, socket, request); status == http.StatusNoContent { |
| 282 | t.Fatal("quarantined replay resumed") |
| 283 | } |
| 284 | w.roster.readmit() |
| 285 | b1 := w.fresh() |
| 286 | if status := postRecoveryHandover(t, socket, request); status != http.StatusConflict { |
| 287 | t.Fatalf("replay with an obsolete successor: HTTP %d", status) |
| 288 | } |
| 289 | if v := w.view(rec.Host, captured); v.Barrier == nil { |
| 290 | t.Fatal("replay refusal cleared the barrier") |
| 291 | } |
| 292 | w.priorIntact("refused replay", receipt.Target, route) |
| 293 | |
| 294 | // Startup: up, not resumed, settled. |
| 295 | logs := captureLog(t) |
| 296 | if err := w.restart(); err != nil { |
| 297 | t.Fatal("startup with an obsolete handover barrier:", err) |
| 298 | } |
| 299 | if err := w.root.RecoverPending(ctx); err != nil { |
| 300 | t.Fatal("startup recovery with an obsolete handover barrier:", err) |
| 301 | } |
| 302 | if !strings.Contains(logs(), "obsolete cold handover: stamped "+b0.Binding+" current "+b1.Binding) { |
| 303 | t.Fatalf("no settlement log: %s", logs()) |
| 304 | } |
| 305 | v := w.view(rec.Host, captured) |
| 306 | if v.Barrier != nil || v.Pending == nil || *v.Pending != receipt.Target || v.Installation.Stamp != receipt.Target.Stamp { |
| 307 | t.Fatalf("after settlement: barrier %v pending %v installation %+v", v.Barrier, v.Pending, v.Installation) |
| 308 | } |
| 309 | retired := false |
| 310 | for _, old := range v.Retired { |
| 311 | retired = retired || old == successorStamp |
| 312 | } |
| 313 | if !retired { |
| 314 | t.Fatalf("obsolete successor stamp not retired: %v", v.Retired) |
| 315 | } |
| 316 | for name, phase := range w.coldJournals() { |
| 317 | if phase != "settled" { |
| 318 | t.Fatalf("journal %s after settlement: %s", name, phase) |
| 319 | } |
| 320 | } |
| 321 | if _, revision := w.successorInventory(admitted); revision != 0 { |
| 322 | t.Fatal("the obsolete successor got an inventory") |
| 323 | } |
| 324 | w.priorIntact("settled", receipt.Target, route) |
| 325 | |
| 326 | // The prior's historical authority: its cleanup stop completes with its |
| 327 | // original receipt. |
| 328 | socket = w.serve() |
| 329 | cleanup := w.priorStop(prior, b0.Binding) |
| 330 | cleanup.Cleanup = true |
| 331 | stopped, err := w.root.Stop.Coordinator.FenceAndStop(ctx, cleanup) |
| 332 | if err != nil || !bytes.Equal(stopped.Bytes, receipt.Bytes) { |
| 333 | t.Fatalf("prior's cleanup stop after settlement: %v (same receipt %v)", err, bytes.Equal(stopped.Bytes, receipt.Bytes)) |
| 334 | } |
| 335 | // The obsolete successor is never reused, at any binding. |
| 336 | if status := postRecoveryHandover(t, socket, coldRequestFor(receipt.Target, admitted, enrollment, b1.Binding)); status != http.StatusConflict { |
| 337 | t.Fatalf("settled successor reused: HTTP %d", status) |
| 338 | } |
| 339 | |
| 340 | // Clean reinstall at B1: a new successor admitted after the settlement, |
| 341 | // handed the prior through root's handler. |
| 342 | clean := w.coldAdmit(rec.Host, "g7") |
| 343 | if status := postRecoveryHandover(t, socket, coldRequestFor(receipt.Target, clean, enrollment, b1.Binding)); status != http.StatusNoContent { |
| 344 | t.Fatalf("clean reinstall at B1: HTTP %d", status) |
| 345 | } |
| 346 | successor, revision := w.successorInventory(clean) |
| 347 | if revision == 0 || successor.DirectoryRevision != b1.Binding { |
| 348 | t.Fatalf("clean successor inventory: revision %d stamp %q", revision, successor.DirectoryRevision) |
| 349 | } |
| 350 | if row, err := w.reg.Get(rec.Host); err != nil || row == nil || row.ClaimStamp != clean.Claim.Holder.Stamp { |
| 351 | t.Fatalf("clean successor not started: %+v %v", row, err) |
| 352 | } |
| 353 | |
| 354 | // Another restart: the settled journal is left alone. |
| 355 | if err := w.restart(); err != nil { |
| 356 | t.Fatal(err) |
| 357 | } |
| 358 | if err := w.root.RecoverPending(ctx); err != nil { |
| 359 | t.Fatal("restart after the clean reinstall:", err) |
| 360 | } |
| 361 | settled := 0 |
| 362 | for _, phase := range w.coldJournals() { |
| 363 | if phase == "settled" { |
| 364 | settled++ |
| 365 | } |
| 366 | } |
| 367 | if settled != 1 { |
| 368 | t.Fatalf("journals after the reinstall: %v", w.coldJournals()) |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | // B1, independently (fix round 1, finding 4): a journaled cold handover |
| 373 | // whose successor matches root's startup binding is not obsolete, so only |
| 374 | // startup's fresh admission stands between it and a resume. Root restarts |
| 375 | // with the roster unchanged (startup binding B0), then the fresh authority |
| 376 | // changes before RecoverPending: a quarantine, then a readmission (B1). |
| 377 | // Each time startup refuses the resume, and nothing of the successor |
| 378 | // exists: the barrier stays, the journal stays prepared, the successor has |
| 379 | // no inventory, the prior's row and active stamp are its own. Removing only |
| 380 | // startup's admission (the Admit of RecoverPending's cold handover) resumes |
| 381 | // the handover here. |
| 382 | func TestColdHandoverStartupAdmitsAgainstFreshAuthority(t *testing.T) { |
| 383 | ctx := t.Context() |
| 384 | w := newAssemblyWorld(t) |
| 385 | w.mustStart() |
| 386 | b0 := w.fresh() |
| 387 | rec, route := w.newHost("a") |
| 388 | if err := w.agent(b0).install(rec, route); err != nil { |
| 389 | t.Fatal(err) |
| 390 | } |
| 391 | route = w.installedRoute(rec.Host) |
| 392 | prior := w.operationStop(rec.Host, b0.Binding) |
| 393 | socket := w.serve() |
| 394 | receipt := w.predecessorReceipt(socket, prior) |
| 395 | admitted := w.coldAdmit(rec.Host, "g6") |
| 396 | request := coldRequestFor(receipt.Target, admitted, w.roster.row().Enrollment, b0.Binding) |
| 397 | successorStamp := hostguard.Captured{Incarnation: admitted.Claim.Holder.Incarnation, Stamp: admitted.Claim.Holder.Stamp} |
| 398 | captured := hostguard.Captured{Incarnation: receipt.Target.Incarnation, Stamp: receipt.Target.Stamp} |
| 399 | |
| 400 | // Root dies right after BeginColdHandover's write, under B0. |
| 401 | *w.interrupt = 2 |
| 402 | status := postRecoveryHandover(t, socket, request) |
| 403 | *w.interrupt = -1 |
| 404 | if status != http.StatusConflict { |
| 405 | t.Fatalf("interrupted handover: HTTP %d", status) |
| 406 | } |
| 407 | // Restart with the roster unchanged: the journal matches the startup |
| 408 | // binding, so it is not obsolete. |
| 409 | if err := w.restart(); err != nil { |
| 410 | t.Fatal(err) |
| 411 | } |
| 412 | if w.root.startupBinding != b0.Binding { |
| 413 | t.Fatalf("startup binding %q, want B0 %q", w.root.startupBinding, b0.Binding) |
| 414 | } |
| 415 | untouched := func(what string) { |
| 416 | t.Helper() |
| 417 | if v := w.view(rec.Host, captured); v.Barrier == nil || v.Barrier.Stamp != successorStamp.Stamp { |
| 418 | t.Fatalf("%s: barrier %+v", what, v.Barrier) |
| 419 | } |
| 420 | journals := w.coldJournals() |
| 421 | if len(journals) != 1 { |
| 422 | t.Fatalf("%s: journals %v", what, journals) |
| 423 | } |
| 424 | for _, phase := range journals { |
| 425 | if phase != "prepared" { |
| 426 | t.Fatalf("%s: journal phase %q", what, phase) |
| 427 | } |
| 428 | } |
| 429 | if _, revision := w.successorInventory(admitted); revision != 0 { |
| 430 | t.Fatalf("%s: the successor got an inventory", what) |
| 431 | } |
| 432 | w.priorIntact(what, receipt.Target, route) |
| 433 | } |
| 434 | for _, change := range []struct { |
| 435 | what string |
| 436 | apply func() |
| 437 | }{{"quarantined", func() { w.roster.quarantine("loss_1") }}, {"readmitted", func() { w.roster.readmit() }}} { |
| 438 | change.apply() |
| 439 | logs := captureLog(t) |
| 440 | err := w.root.RecoverPending(ctx) |
| 441 | if strings.Contains(logs(), "obsolete cold handover") { |
| 442 | t.Fatalf("%s: settled as obsolete, so admission was not what refused: %s", change.what, logs()) |
| 443 | } |
| 444 | untouched(change.what) |
| 445 | if err == nil { |
| 446 | t.Fatalf("%s: startup recovery reported no refusal", change.what) |
| 447 | } |
| 448 | } |
| 449 | if b1 := w.fresh(); b1.Binding == b0.Binding { |
| 450 | t.Fatal("readmission did not move the binding") |
| 451 | } |
| 452 | } |
| 453 | |
amadan