| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostclaim" |
| 13 | "github.com/carlosframework/platform/internal/lease" |
| 14 | "github.com/carlosframework/platform/internal/moves" |
| 15 | "github.com/carlosframework/platform/internal/park" |
| 16 | "github.com/carlosframework/platform/internal/provision" |
| 17 | ) |
| 18 | |
| 19 | // The abort (moves spec §7.11; Plan 5 Task 1). The fixture is the Phase 1 |
| 20 | // harness after one preparing round, so the lane holds staged objects in the |
| 21 | // target bucket (data-uk) and the source's own bucket (data-ie) holds its |
| 22 | // receipts, its ledger and a snapshot. The target region's box (uk-box) runs |
| 23 | // the lane park through the real park engine over the same buckets with a |
| 24 | // fake lease table. Everything else is real: the source's abort, hostclaim's |
| 25 | // Abort and ReleaseAbort, and provision's guarded transitions. |
| 26 | |
| 27 | type abortHarness struct { |
| 28 | *phase1Harness |
| 29 | tdir string |
| 30 | tleases *fakeParkLeases |
| 31 | tw *movesWorker |
| 32 | abAt string |
| 33 | abDone bool |
| 34 | abHits []string |
| 35 | } |
| 36 | |
| 37 | func newAbortHarness(t *testing.T) *abortHarness { |
| 38 | t.Helper() |
| 39 | h := &abortHarness{phase1Harness: newPhase1Harness(t), tdir: t.TempDir(), tleases: &fakeParkLeases{}} |
| 40 | h.start(t, "manual") |
| 41 | h.pass(t) |
| 42 | if len(h.laneKeys()) == 0 { |
| 43 | t.Fatal("the fixture staged nothing") |
| 44 | } |
| 45 | h.f.put("data-ie", moves.SnapshotsPrefix(p1Host, p1ID)+"files-x.sealed", "s") |
| 46 | h.cfg.abFault = func(point string) error { |
| 47 | h.abHits = append(h.abHits, point) |
| 48 | if point == h.abAt && !h.abDone { |
| 49 | h.abDone = true |
| 50 | return errCrash |
| 51 | } |
| 52 | return nil |
| 53 | } |
| 54 | return h |
| 55 | } |
| 56 | |
| 57 | // abort is the console's abort door: the guarded record CAS. |
| 58 | func (h *abortHarness) abort(t *testing.T) { |
| 59 | t.Helper() |
| 60 | g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc} |
| 61 | if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | // laneCfg is the target region's box with its lane park armed. |
| 67 | func (h *abortHarness) laneCfg() moveCfg { |
| 68 | c := h.targetCfg() |
| 69 | c.DataDir = h.tdir |
| 70 | armParkFakes(&c, h.s, h.tleases, c.Region, roleTarget) |
| 71 | return c |
| 72 | } |
| 73 | |
| 74 | // tPass runs one pass as the target region's box. |
| 75 | func (h *abortHarness) tPass(t *testing.T) { |
| 76 | t.Helper() |
| 77 | if h.tw == nil { |
| 78 | h.tw = &movesWorker{mirrors: map[string]lease.MirrorStore{}} |
| 79 | } |
| 80 | h.tw.cfg = h.laneCfg() |
| 81 | if err := movesPass(context.Background(), h.s, h.tw); err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | // laneChore is the lane's park the abort queued in the target region. |
| 87 | func (h *abortHarness) laneChore(t *testing.T) *hostclaim.Obligation { |
| 88 | t.Helper() |
| 89 | for _, o := range h.claim(t).Obligations { |
| 90 | if o.Op == p1ID && o.Kind == "park-replica" && o.Region == "uk" && o.Park != nil && o.Park.Location == moves.ScopeLocation(p1Host, p1ID) { |
| 91 | return &o |
| 92 | } |
| 93 | } |
| 94 | return nil |
| 95 | } |
| 96 | |
| 97 | // laneKeys are the objects still in the move's lane in the target bucket. |
| 98 | func (h *abortHarness) laneKeys() []string { |
| 99 | var out []string |
| 100 | for k := range h.f.buckets["data-uk"] { |
| 101 | if strings.HasPrefix(k, moves.ScopeLocation(p1Host, p1ID)+"/") { |
| 102 | out = append(out, k) |
| 103 | } |
| 104 | } |
| 105 | return out |
| 106 | } |
| 107 | |
| 108 | // archived counts the objects under one retirement's archive of the lane. |
| 109 | func (h *abortHarness) archived(inc, ret string) int { |
| 110 | prefix := "parked/" + inc + "/" + p1Host + "/" + ret + "/moves/" |
| 111 | n := 0 |
| 112 | for k := range h.f.buckets["data-uk"] { |
| 113 | if strings.HasPrefix(k, prefix) { |
| 114 | n++ |
| 115 | } |
| 116 | } |
| 117 | return n |
| 118 | } |
| 119 | |
| 120 | // targetDeletes are every delete request anyone sent to the target bucket: |
| 121 | // a plain delete, a guarded staging delete, or a delete by version. |
| 122 | func (h *abortHarness) targetDeletes() []string { |
| 123 | var out []string |
| 124 | for _, op := range h.f.ops { |
| 125 | if strings.HasPrefix(op, "delete data-uk/") { |
| 126 | out = append(out, op) |
| 127 | } |
| 128 | } |
| 129 | for _, c := range h.f.stagingCalls() { |
| 130 | if c.Op == "delete" && c.Bucket == "data-uk" { |
| 131 | out = append(out, "staging delete "+c.Key) |
| 132 | } |
| 133 | } |
| 134 | out = append(out, h.vs.callsOn("delete-version data-uk/")...) |
| 135 | return out |
| 136 | } |
| 137 | |
| 138 | func (h *abortHarness) pastHorizon(t *testing.T, o *hostclaim.Obligation) { |
| 139 | t.Helper() |
| 140 | until := o.Park.EvidenceAt.Add(park.SignatureAge + o.Park.RequestLimit) |
| 141 | if d := until.Sub(h.clock.now); d > 0 { |
| 142 | h.clock.Advance(d + time.Second) |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | // waiting asserts the abort is still waiting for the lane park: the record |
| 147 | // aborting, the claim moving/aborting under the source, the chore owed. |
| 148 | func (h *abortHarness) waiting(t *testing.T) { |
| 149 | t.Helper() |
| 150 | if m := h.read(t, p1Host).Move; m == nil || m.State != provision.MoveAborting { |
| 151 | t.Fatalf("the record is not aborting: %+v", m) |
| 152 | } |
| 153 | c := h.claim(t) |
| 154 | if c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseAborting || c.Op.ID != p1ID || c.Op.Exec != "ie-box" { |
| 155 | t.Fatalf("the claim while the abort waits: %+v", c.Op) |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | // closed asserts the end: the record at rest in ie with LastMove aborted, |
| 160 | // the claim serving under the source with this move recorded as released. |
| 161 | func (h *abortHarness) closed(t *testing.T) { |
| 162 | t.Helper() |
| 163 | rec := h.read(t, p1Host) |
| 164 | if rec.Move != nil || rec.Region != "ie" || rec.LastMove == nil || rec.LastMove.ID != p1ID || rec.LastMove.Outcome != "aborted" { |
| 165 | t.Fatalf("the record after the abort: region %s move %+v last %+v", rec.Region, rec.Move, rec.LastMove) |
| 166 | } |
| 167 | c := h.claim(t) |
| 168 | if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(p1ID)(c) != nil || len(c.Obligations) != 0 { |
| 169 | t.Fatalf("the claim after the abort: %+v obligations %+v", c.Op, c.Obligations) |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | // ownStateGone asserts the source retired its own state for the move: the |
| 174 | // snapshots, receipts and ledger in its bucket, and its journal directory. |
| 175 | func (h *abortHarness) ownStateGone(t *testing.T) { |
| 176 | t.Helper() |
| 177 | for k := range h.f.buckets["data-ie"] { |
| 178 | if strings.HasPrefix(k, moves.SnapshotsPrefix(p1Host, p1ID)) || k == moves.ReceiptsKey(p1Host, p1ID) || k == moves.BuildStateReceiptsKey(p1Host, p1ID) || k == moves.LedgerKey(p1Host, p1ID) { |
| 179 | t.Fatalf("%s survived the abort", k) |
| 180 | } |
| 181 | } |
| 182 | if _, err := os.Stat(moves.JournalDir(h.cfg.DataDir, p1Host, p1ID)); !os.IsNotExist(err) { |
| 183 | t.Fatalf("the source's journal directory is still there: %v", err) |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // TestAbortParksFromTheTargetAndWaits (Review Focus 2, §7.11, joint N13): the |
| 188 | // source's passes never send a delete to the target bucket; the source takes |
| 189 | // the claim to moving/aborting with the lane's park queued for the target |
| 190 | // region, and retires its own snapshots (by version), receipts, ledger and |
| 191 | // journal. The target region's box archives moves/<host>/<id>/... under |
| 192 | // parked/<inc>/<host>/<ret>/moves/... and acknowledges only after the |
| 193 | // horizon. Only then does the source release the claim to serving under |
| 194 | // itself, and the record clear. |
| 195 | func TestAbortParksFromTheTargetAndWaits(t *testing.T) { |
| 196 | h := newAbortHarness(t) |
| 197 | lane := h.laneKeys() |
| 198 | h.abort(t) |
| 199 | deletes := len(h.targetDeletes()) |
| 200 | acquired := h.lease.acquired |
| 201 | decided := h.clock.now |
| 202 | h.pass(t) |
| 203 | h.waiting(t) |
| 204 | o := h.laneChore(t) |
| 205 | if o == nil { |
| 206 | t.Fatal("no lane park queued") |
| 207 | } |
| 208 | if o.Owner != "region:uk" || o.Park.Incarnation != h.inc || o.Park.RequestLimit != moves.ParkRequestLimit { |
| 209 | t.Fatalf("the lane park: %+v %+v", o, o.Park) |
| 210 | } |
| 211 | if min := decided.Add(moves.SignatureLife + moves.DefaultSkew); o.Park.EvidenceAt.Before(min) || o.Park.EvidenceAt.After(min.Add(time.Second)) { |
| 212 | t.Fatalf("the park's horizon starts at %v, want the decision's time plus a signature's life and the skew, %v", o.Park.EvidenceAt, min) |
| 213 | } |
| 214 | if h.lease.acquired != acquired+1 || h.lease.released != h.lease.acquired { |
| 215 | t.Fatalf("the staging lease: acquired %d (was %d), released %d", h.lease.acquired, acquired, h.lease.released) |
| 216 | } |
| 217 | h.ownStateGone(t) |
| 218 | if got := h.vs.callsOn("delete-version data-ie/" + moves.SnapshotsPrefix(p1Host, p1ID)); len(got) == 0 { |
| 219 | t.Fatal("the snapshots were not retired by version") |
| 220 | } |
| 221 | if got := h.laneKeys(); len(got) != len(lane) { |
| 222 | t.Fatalf("the source touched the lane: %d keys, had %d", len(got), len(lane)) |
| 223 | } |
| 224 | // The source waits: another pass changes nothing and deletes nothing there. |
| 225 | gen := h.claim(t).Gen |
| 226 | h.pass(t) |
| 227 | h.waiting(t) |
| 228 | if h.claim(t).Gen != gen { |
| 229 | t.Fatal("a waiting source wrote the claim") |
| 230 | } |
| 231 | if got := h.targetDeletes(); len(got) != deletes { |
| 232 | t.Fatalf("the source sent deletes to the target bucket: %v", got[deletes:]) |
| 233 | } |
| 234 | // The target region's box archives the lane; the horizon has not passed. |
| 235 | h.tPass(t) |
| 236 | if left := h.laneKeys(); len(left) != 0 { |
| 237 | t.Fatalf("the lane still holds %v", left) |
| 238 | } |
| 239 | if n := h.archived(o.Park.Incarnation, o.Retirement); n < len(lane) { |
| 240 | t.Fatalf("%d archived, the lane had %d", n, len(lane)) |
| 241 | } |
| 242 | if h.laneChore(t) == nil { |
| 243 | t.Fatal("the lane park was acknowledged before its horizon") |
| 244 | } |
| 245 | h.pass(t) |
| 246 | h.waiting(t) |
| 247 | // Past the horizon the target acknowledges; the source then releases. |
| 248 | h.pastHorizon(t, o) |
| 249 | h.tPass(t) |
| 250 | if h.laneChore(t) != nil { |
| 251 | t.Fatal("the lane park was not acknowledged after its horizon") |
| 252 | } |
| 253 | h.waiting(t) |
| 254 | h.pass(t) |
| 255 | h.closed(t) |
| 256 | before := h.claim(t) |
| 257 | h.pass(t) |
| 258 | h.tPass(t) |
| 259 | if after := h.claim(t); after.Gen != before.Gen { |
| 260 | t.Fatalf("a pass after the abort wrote the claim (%d to %d)", before.Gen, after.Gen) |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | // TestAbortScopeIsTheWritersHorizon (witness wStagingQuiet; ruling K5): the |
| 265 | // abort's horizon covers every staging request the source signed before it, |
| 266 | // and a staging PUT signed before the abort that lands after the park's |
| 267 | // first pass is found by the verification listing and parked as a further |
| 268 | // retirement event. The park acknowledges only after abort_bound + H. |
| 269 | func TestAbortScopeIsTheWritersHorizon(t *testing.T) { |
| 270 | h := newAbortHarness(t) |
| 271 | signed := h.f.stagingCalls() |
| 272 | if len(signed) == 0 { |
| 273 | t.Fatal("the fixture signed no staging request") |
| 274 | } |
| 275 | h.abort(t) |
| 276 | h.pass(t) |
| 277 | o := h.laneChore(t) |
| 278 | if o == nil { |
| 279 | t.Fatal("no lane park queued") |
| 280 | } |
| 281 | bound := o.Park.EvidenceAt |
| 282 | for _, c := range signed { |
| 283 | if c.SignAt.Add(moves.SignatureLife + moves.DefaultSkew).After(bound) { |
| 284 | t.Fatalf("a staging request signed at %v can land after the abort's horizon %v", c.SignAt, bound) |
| 285 | } |
| 286 | } |
| 287 | // The park's first pass archives the lane. |
| 288 | h.tPass(t) |
| 289 | first := o.Retirement |
| 290 | if len(h.laneKeys()) != 0 || h.laneChore(t) == nil { |
| 291 | t.Fatalf("after the first pass: lane %v", h.laneKeys()) |
| 292 | } |
| 293 | // A request signed before the abort lands now, in the lane. |
| 294 | late := moves.ScopeLocation(p1Host, p1ID) + "/replica/0000/0000000000000009-0000000000000009.ltx" |
| 295 | h.f.store("data-uk", late, fakeMoveObject{lease.MirrorObject{ETag: "late", Size: 4}, "late"}) |
| 296 | h.f.setVersion("data-uk", late, h.f.nextVersion()) |
| 297 | // One second before abort_bound + H nothing is acknowledged, whatever |
| 298 | // the listing says. |
| 299 | deadline := bound.Add(park.SignatureAge + o.Park.RequestLimit) |
| 300 | if d := deadline.Add(-time.Second).Sub(h.clock.now); d > 0 { |
| 301 | h.clock.Advance(d) |
| 302 | } |
| 303 | h.tPass(t) |
| 304 | if h.laneChore(t) == nil { |
| 305 | t.Fatal("the lane park was acknowledged before abort_bound + H") |
| 306 | } |
| 307 | h.pass(t) |
| 308 | h.waiting(t) |
| 309 | // Past the horizon the listing finds the late write: a further retirement. |
| 310 | h.pastHorizon(t, o) |
| 311 | h.tPass(t) |
| 312 | o2 := h.laneChore(t) |
| 313 | if o2 == nil { |
| 314 | t.Fatal("the park acknowledged over the late write") |
| 315 | } |
| 316 | if o2.Retirement == first || !o2.Park.EvidenceAt.Equal(bound) { |
| 317 | t.Fatalf("the further event: retirement %q (first %q), horizon %v", o2.Retirement, first, o2.Park.EvidenceAt) |
| 318 | } |
| 319 | h.tPass(t) |
| 320 | if h.laneChore(t) != nil { |
| 321 | t.Fatal("the further retirement was not acknowledged") |
| 322 | } |
| 323 | if _, ok := h.f.buckets["data-uk"][late]; ok { |
| 324 | t.Fatal("the late write is still in the lane") |
| 325 | } |
| 326 | if n := h.archived(o.Park.Incarnation, o2.Retirement); n != 1 { |
| 327 | t.Fatalf("the further retirement archived %d objects, want the late write", n) |
| 328 | } |
| 329 | if q := h.claim(t).QuietAfter["uk"]; !q.Equal(deadline) { |
| 330 | t.Fatalf("quiet_after %v, want %v", q, deadline) |
| 331 | } |
| 332 | h.pass(t) |
| 333 | h.closed(t) |
| 334 | } |
| 335 | |
| 336 | // TestAbortWaitsForAnUnreachableTargetRegion (ruling K21): with no box of |
| 337 | // the target region running the lane park, the claim stays moving/aborting |
| 338 | // and nothing releases, however long the source waits. A delete cannot |
| 339 | // begin over it, because the claim is not serving. |
| 340 | func TestAbortWaitsForAnUnreachableTargetRegion(t *testing.T) { |
| 341 | h := newAbortHarness(t) |
| 342 | h.abort(t) |
| 343 | h.pass(t) |
| 344 | o := h.laneChore(t) |
| 345 | if o == nil { |
| 346 | t.Fatal("no lane park queued") |
| 347 | } |
| 348 | for i := 0; i < 4; i++ { |
| 349 | h.clock.Advance(12 * time.Hour) |
| 350 | h.pass(t) |
| 351 | h.waiting(t) |
| 352 | if h.laneChore(t) == nil { |
| 353 | t.Fatal("the lane park vanished with no target pass") |
| 354 | } |
| 355 | } |
| 356 | c := h.claim(t) |
| 357 | if err := hostclaim.StartGuard(c, []string{"ie"}, []string{"ie-box"}, ""); err == nil { |
| 358 | t.Fatal("a new operation could start over the waiting abort") |
| 359 | } |
| 360 | // An unarmed target box parks nothing either: the abort's lane park goes |
| 361 | // through laneParkTarget (movesfinal.go's dispatch for an aborting |
| 362 | // target), whose arming is the shared lane predicate, revertLaneArmed, |
| 363 | // which requires all three of the claim's park hooks (Plan 5 Task 9a, |
| 364 | // M17). Each one missing leaves the arm unarmed, with its standing |
| 365 | // notice, and the lane untouched. |
| 366 | for name, unarm := range map[string]func(*moveCfg){ |
| 367 | "no park leases": func(c *moveCfg) { c.ParkLeases = nil }, |
| 368 | "no park view": func(c *moveCfg) { c.ParkView = nil }, |
| 369 | "no park authorizer": func(c *moveCfg) { c.ParkAuthorize = nil }, |
| 370 | } { |
| 371 | unarmed := h.laneCfg() |
| 372 | unarm(&unarmed) |
| 373 | logs := captureLog(t) |
| 374 | w := &movesWorker{cfg: unarmed, mirrors: map[string]lease.MirrorStore{}} |
| 375 | if err := movesPass(context.Background(), h.s, w); err != nil { |
| 376 | t.Fatal(err) |
| 377 | } |
| 378 | if h.laneChore(t) == nil || len(h.laneKeys()) == 0 { |
| 379 | t.Fatalf("%s: an unarmed target box parked the lane", name) |
| 380 | } |
| 381 | if !strings.Contains(logs(), "this box's park engine is not armed") { |
| 382 | t.Fatalf("%s: the abort's lane park was tried by an unarmed box:\n%s", name, logs()) |
| 383 | } |
| 384 | } |
| 385 | // Once the region returns, the abort completes. |
| 386 | h.tPass(t) |
| 387 | h.pastHorizon(t, o) |
| 388 | h.tPass(t) |
| 389 | h.pass(t) |
| 390 | h.closed(t) |
| 391 | } |
| 392 | |
| 393 | // TestAbortReplays: a crash after any of the abort's side effects, then a |
| 394 | // restarted agent, still reaches the close, with the journalled bound named |
| 395 | // once (the lane park's scope never changes), one lane park queued, and the |
| 396 | // source's own state retired. |
| 397 | func TestAbortReplays(t *testing.T) { |
| 398 | t.Parallel() |
| 399 | for _, point := range []string{"after-abort-bound", "after-abort-claim", "after-abort-own-state", "after-abort-journal", "after-release-abort"} { |
| 400 | t.Run(point, func(t *testing.T) { |
| 401 | h := newAbortHarness(t) |
| 402 | h.abort(t) |
| 403 | h.abAt = point |
| 404 | if err := movesPass(context.Background(), h.s, h.w()); err != nil { |
| 405 | t.Fatal(err) |
| 406 | } |
| 407 | for i := 0; i < 6 && !h.abDone; i++ { |
| 408 | h.tPass(t) |
| 409 | if o := h.laneChore(t); o != nil { |
| 410 | h.pastHorizon(t, o) |
| 411 | } |
| 412 | h.pass(t) |
| 413 | } |
| 414 | if !h.abDone { |
| 415 | t.Fatalf("never reached %s (%v)", point, h.abHits) |
| 416 | } |
| 417 | var bound time.Time |
| 418 | if j, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID); err == nil { |
| 419 | if raw := j.Steps[abortStepBound]; raw != "" { |
| 420 | bound, _ = time.Parse(time.RFC3339Nano, raw) |
| 421 | } |
| 422 | } |
| 423 | if o := h.laneChore(t); o != nil { |
| 424 | if !bound.IsZero() && !o.Park.EvidenceAt.Equal(bound) { |
| 425 | t.Fatalf("the park's horizon %v is not the journalled bound %v", o.Park.EvidenceAt, bound) |
| 426 | } |
| 427 | bound = o.Park.EvidenceAt |
| 428 | } |
| 429 | h.clock.Advance(time.Minute) |
| 430 | h.worker1 = nil |
| 431 | for i := 0; i < 8 && h.read(t, p1Host).Move != nil; i++ { |
| 432 | h.pass(t) |
| 433 | if o := h.laneChore(t); o != nil { |
| 434 | if !bound.IsZero() && !o.Park.EvidenceAt.Equal(bound) { |
| 435 | t.Fatalf("the replay named another horizon: %v, journalled %v", o.Park.EvidenceAt, bound) |
| 436 | } |
| 437 | h.tPass(t) |
| 438 | h.pastHorizon(t, o) |
| 439 | h.tPass(t) |
| 440 | } |
| 441 | } |
| 442 | h.closed(t) |
| 443 | h.ownStateGone(t) |
| 444 | }) |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | // An aborting record never flipped: its region is the source's, so the |
| 449 | // target region's box has neither the row nor a journal of the move. The |
| 450 | // participant rule answers it as the target on the state alone, as for a |
| 451 | // revert: its one piece of work is the lane park (ruling K6). The source's |
| 452 | // region answers nothing without the row (its abort is phase1MoveRound's), |
| 453 | // and a third region nothing. |
| 454 | func TestAbortingTargetIsAParticipant(t *testing.T) { |
| 455 | cfg := moveCfg{Region: "uk", DataDir: t.TempDir(), OwnsRoute: func(string) bool { return false }} |
| 456 | rec := provision.Record{Host: p1Host, Region: "ie", Move: &provision.MoveState{ID: p1ID, State: provision.MoveAborting, From: "ie", To: "uk"}} |
| 457 | if got := movesParticipantRecord(cfg, rec); got != roleTarget { |
| 458 | t.Fatalf("an aborting record's target: %q", got) |
| 459 | } |
| 460 | cfg.Region = "ie" |
| 461 | if got := movesParticipantRecord(cfg, rec); got != roleNone { |
| 462 | t.Fatalf("the source's region without the row: %q", got) |
| 463 | } |
| 464 | cfg.Region = "fr" |
| 465 | if got := movesParticipantRecord(cfg, rec); got != roleNone { |
| 466 | t.Fatalf("a third region: %q", got) |
| 467 | } |
| 468 | } |
| 469 | |
| 470 | // The abort's crash boundary between the claim's release and the record's |
| 471 | // clear: a round that finds the record aborting and the claim serving |
| 472 | // completes the record only when the serving op records this move as the |
| 473 | // one it released (RequireReleasedMove). A serving claim without that |
| 474 | // evidence was released by someone else, and the record is left alone. |
| 475 | func TestAbortCompletesOnlyWithItsReleaseEvidence(t *testing.T) { |
| 476 | ctx := context.Background() |
| 477 | h := newAbortHarness(t) |
| 478 | h.abort(t) |
| 479 | obs, err := hostclaim.Read(ctx, h.s, p1Host) |
| 480 | if err != nil { |
| 481 | t.Fatal(err) |
| 482 | } |
| 483 | scope := moves.AbortScope(p1Host, p1ID, h.inc, h.clock.now.Add(time.Hour)) |
| 484 | if obs, err = hostclaim.Update(ctx, h.s, obs, hostclaim.Abort(p1ID, "uk", scope, obligationID)); err != nil { |
| 485 | t.Fatal(err) |
| 486 | } |
| 487 | if obs, err = hostclaim.Update(ctx, h.s, obs, hostclaim.CompleteObligation(p1ID, obs.Claim.Obligations[0].ID)); err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | if _, err = hostclaim.Update(ctx, h.s, obs, hostclaim.ReleaseAbort(p1ID, "ie-box")); err != nil { |
| 491 | t.Fatal(err) |
| 492 | } |
| 493 | h.pass(t) |
| 494 | h.closed(t) |
| 495 | h.ownStateGone(t) |
| 496 | |
| 497 | // A serving claim with no evidence of this release: left alone. |
| 498 | h2 := newAbortHarness(t) |
| 499 | h2.abort(t) |
| 500 | rewriteClaimOp(t, h2.s, func(op *hostclaim.Op) { *op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} }) |
| 501 | h2.pass(t) |
| 502 | if got := h2.read(t, p1Host); got.Move == nil || got.Move.State != provision.MoveAborting { |
| 503 | t.Fatalf("a serving claim without evidence completed the record: %+v", got.Move) |
| 504 | } |
| 505 | } |
| 506 | |
| 507 | // Fix round 1 (task-1-review.md, Important): release never follows from an |
| 508 | // inference that the source's cleanup finished. After the claim's CAS, the |
| 509 | // source retires its own state on every pass before ReleaseAbort, and any |
| 510 | // failure to do so blocks the release until it succeeds. |
| 511 | func TestAbortReleaseWaitsForTheSourcesCleanup(t *testing.T) { |
| 512 | // crashAfterClaim stops the source right after the Abort CAS, then lets |
| 513 | // the target region park and acknowledge the lane, so only the source's |
| 514 | // own cleanup stands between the claim and its release. |
| 515 | crashAfterClaim := func(t *testing.T) *abortHarness { |
| 516 | t.Helper() |
| 517 | h := newAbortHarness(t) |
| 518 | h.abort(t) |
| 519 | h.abAt = "after-abort-claim" |
| 520 | if err := movesPass(context.Background(), h.s, h.w()); err != nil { |
| 521 | t.Fatal(err) |
| 522 | } |
| 523 | if !h.abDone { |
| 524 | t.Fatalf("never reached after-abort-claim (%v)", h.abHits) |
| 525 | } |
| 526 | o := h.laneChore(t) |
| 527 | if o == nil { |
| 528 | t.Fatal("no lane park queued") |
| 529 | } |
| 530 | h.tPass(t) |
| 531 | h.pastHorizon(t, o) |
| 532 | h.tPass(t) |
| 533 | if h.laneChore(t) != nil { |
| 534 | t.Fatal("the lane park was not acknowledged") |
| 535 | } |
| 536 | h.worker1 = nil |
| 537 | return h |
| 538 | } |
| 539 | t.Run("the journal is gone but the state is not", func(t *testing.T) { |
| 540 | h := crashAfterClaim(t) |
| 541 | if err := os.RemoveAll(moves.JournalDir(h.cfg.DataDir, p1Host, p1ID)); err != nil { |
| 542 | t.Fatal(err) |
| 543 | } |
| 544 | h.pass(t) |
| 545 | h.closed(t) |
| 546 | h.ownStateGone(t) |
| 547 | }) |
| 548 | t.Run("the journal cannot be inspected", func(t *testing.T) { |
| 549 | if os.Geteuid() == 0 { |
| 550 | t.Skip("root reads and removes past any mode") |
| 551 | } |
| 552 | h := crashAfterClaim(t) |
| 553 | hostDir := filepath.Dir(moves.JournalDir(h.cfg.DataDir, p1Host, p1ID)) |
| 554 | if err := os.Chmod(hostDir, 0); err != nil { |
| 555 | t.Fatal(err) |
| 556 | } |
| 557 | t.Cleanup(func() { os.Chmod(hostDir, 0o755) }) |
| 558 | for i := 0; i < 3; i++ { |
| 559 | movesPass(context.Background(), h.s, h.w()) |
| 560 | h.waiting(t) |
| 561 | } |
| 562 | if err := os.Chmod(hostDir, 0o755); err != nil { |
| 563 | t.Fatal(err) |
| 564 | } |
| 565 | h.pass(t) |
| 566 | h.closed(t) |
| 567 | h.ownStateGone(t) |
| 568 | }) |
| 569 | t.Run("the snapshots cannot be listed", func(t *testing.T) { |
| 570 | h := crashAfterClaim(t) |
| 571 | h.vs.deny = func(op, bucket, key string) error { |
| 572 | if op == "list-versions" && bucket == "data-ie" { |
| 573 | return errors.New("injected 503") |
| 574 | } |
| 575 | return nil |
| 576 | } |
| 577 | for i := 0; i < 3; i++ { |
| 578 | movesPass(context.Background(), h.s, h.w()) |
| 579 | h.waiting(t) |
| 580 | } |
| 581 | h.vs.deny = nil |
| 582 | h.pass(t) |
| 583 | h.closed(t) |
| 584 | h.ownStateGone(t) |
| 585 | }) |
| 586 | } |
| 587 | |
amadan