| new file mode 100644 |
| index 000000000..331436394 |
| --- /dev/null |
| +++ b/internal/host/movesoffline_lifetime_test.go |
| @@ -0,0 +1,455 @@ |
| +//go:build linux |
| + |
| +package host |
| + |
| +import ( |
| + "context" |
| + "database/sql" |
| + "encoding/json" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/edge" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Lifetimes end to end on the boxes (self-serve moves plan 1, Task 9): the |
| +// host incarnation and the per-host move sequence carried through a forward |
| +// move, an abort, a reconstructed row and a recreated host, from the |
| +// record's writes to the rows they produce and the publications an edge |
| +// folds. |
| +// |
| +// The world is TestOfflineMoveE2E's (newE2EWorld with landOpts.lifetime): |
| +// the console's real doors, two boxes ticked in the agent's order, every |
| +// box judging lifetimes as a claims-off agent does, and each box's edges |
| +// file written by the production writer with the box's lifetimes switch. |
| +// Publications are folded by the real edge fold (edge.FoldedHome) as a |
| +// third edge sees them, in both file orders. |
| +// |
| +// Two inputs are not the doors' own: |
| +// - a fold input this plan's writer never produces (a held row's |
| +// publication: plan 3's holding entry is what such a box publishes) is |
| +// built by the real writer over a scratch registry row, and its helper |
| +// says so in its name (syntheticReadyPublication); |
| +// - the recreation scenes need a retirement and a create of the hostname, |
| +// which are the console's lifetime doors; the console in this world runs |
| +// before the cutover (no marker, no box floor), so the bucket states |
| +// those doors produce are written in their order through the provision |
| +// writers they use (retireThroughTombstone, recreateLifetime), as Task 7's |
| +// createLifetime writes the first create. Moves the scene needs a box to |
| +// have slept through are the record's transitions through the record |
| +// writer the doors call (moveOnTheRecord). |
| + |
| +// edgeSelf is the third edge whose view the folds take: neither box. |
| +const edgeSelf = "edge-3" |
| + |
| +// published is b's edges file as an edge fetching it now reads it. |
| +func (w *e2eWorld) published(b *offBox) []byte { |
| + w.t.Helper() |
| + body, _, err := w.wd.s.Get(context.Background(), edgesFileKey(b.name)) |
| + if err != nil { |
| + w.t.Fatal(err) |
| + } |
| + return body |
| +} |
| + |
| +// publishedPair is the host's entry in an edges-file body: its pair, and |
| +// whether the body lists the host at all. |
| +func publishedPair(t *testing.T, body []byte) (registry.Pair, bool) { |
| + t.Helper() |
| + var f edgesFile |
| + if err := json.Unmarshal(body, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + h, ok := f.Hosts[offHost] |
| + return registry.Pair{Incarnation: h.Incarnation, ServingSeq: h.ServingSeq}, ok |
| +} |
| + |
| +// requirePublished: body lists the host at want. |
| +func requirePublished(t *testing.T, why string, body []byte, want registry.Pair) { |
| + t.Helper() |
| + got, ok := publishedPair(t, body) |
| + if !ok || got != want { |
| + t.Fatalf("%s: the publication lists the host %v at %+v, want %+v", why, ok, got, want) |
| + } |
| +} |
| + |
| +// requireFoldWinner folds two edges-file bodies through the real edge fold |
| +// as edgeSelf sees them, once with each in the first file position, and |
| +// requires want's box to win the host's dial both times. |
| +func requireFoldWinner(t *testing.T, why string, a, b []byte, want string) { |
| + t.Helper() |
| + for _, order := range [][2][]byte{{a, b}, {b, a}} { |
| + files := map[string][]byte{edgesFileKey("annie"): order[0], edgesFileKey("betty"): order[1]} |
| + box, ok, err := edge.FoldedHome(files, edgeSelf, offHost) |
| + if err != nil || !ok || box != want { |
| + t.Fatalf("%s: the fold chose %q (dial %v, %v), want %s", why, box, ok, err, want) |
| + } |
| + } |
| +} |
| + |
| +// syntheticReadyPublication is a SYNTHETIC fold input: box's edges file as |
| +// the real writer publishes a ready row for the host at pair, built over a |
| +// scratch registry. This plan's writer never publishes the row it stands |
| +// for (a held row is not published; plan 3's holding entry is), so no box |
| +// of the world produces it. |
| +func syntheticReadyPublication(t *testing.T, box string, pair registry.Pair) []byte { |
| + t.Helper() |
| + reg := edgesTestRegistry(t) |
| + lifetimeRow(t, reg, offHost, pair.Incarnation, pair.ServingSeq) |
| + body, _ := publishLifetimes(t, reg, box, nil, true) |
| + requirePublished(t, "the synthetic publication", body, pair) |
| + return body |
| +} |
| + |
| +// forwardMove drives the doors through a move from ie to uk, done, and |
| +// returns ie-box's publication from before the start (what an edge that |
| +// has not refetched ie-box's file since still folds). |
| +func (w *e2eWorld) forwardMove() (retainedSource []byte) { |
| + w.t.Helper() |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 0}, "", "after the create") |
| + retainedSource = w.published(w.ie) |
| + requirePublished(w.t, "ie-box before the move", retainedSource, registry.Pair{Incarnation: 1, ServingSeq: 0}) |
| + w.begin() |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 1}, w.id, "after start") |
| + w.seal() |
| + w.flipAndLand() |
| + w.requirePair(w.uk, registry.Pair{Incarnation: 1, ServingSeq: 1}, w.id, "after the landing") |
| + w.open() |
| + w.requirePair(w.uk, registry.Pair{Incarnation: 1, ServingSeq: 2}, "", "after open") |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 1}, w.id, "after open") |
| + w.finish("done") |
| + w.tickAll() |
| + w.requirePair(w.uk, registry.Pair{Incarnation: 1, ServingSeq: 2}, "", "after finish") |
| + if rt := w.ie.row(offHost); rt != nil { |
| + w.t.Fatalf("ie-box keeps a row after finish: %+v", rt) |
| + } |
| + if rec := w.wd.record(); rec.Incarnation != 1 || rec.MoveSeq != 3 || rec.Move != nil { |
| + w.t.Fatalf("after finish the record is at (%d, %d) with move %+v, want (1, 3) and none", rec.Incarnation, rec.MoveSeq, rec.Move) |
| + } |
| + return retainedSource |
| +} |
| + |
| +// abortFromLanded drives the doors through a move from ie to uk aborted |
| +// from landed and finished: the source reopens at the abort's sequence. |
| +func (w *e2eWorld) abortFromLanded() { |
| + w.t.Helper() |
| + w.begin() |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 1}, w.id, "after start") |
| + w.seal() |
| + w.flipAndLand() |
| + if st := w.wd.record().Move.State; st != provision.OfflineLanded { |
| + w.t.Fatalf("fixture: state %s, want landed", st) |
| + } |
| + w.requirePair(w.uk, registry.Pair{Incarnation: 1, ServingSeq: 1}, w.id, "after the landing") |
| + if w.uk.lists(offHost) { |
| + w.t.Fatal("uk-box publishes its held landing row") |
| + } |
| + w.accept("abort", "", provision.OfflineReverting) |
| + w.until("the revert", nextIs("finish")) |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 2}, "", "after the revert") |
| + w.finish("aborted") |
| + w.tickAll() |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 2}, "", "after finish") |
| + if rt := w.uk.row(offHost); rt != nil { |
| + w.t.Fatalf("uk-box keeps a row after the abort: %+v", rt) |
| + } |
| + if rec := w.wd.record(); rec.Incarnation != 1 || rec.MoveSeq != 3 || rec.Move != nil { |
| + w.t.Fatalf("after finish the record is at (%d, %d) with move %+v, want (1, 3) and none", rec.Incarnation, rec.MoveSeq, rec.Move) |
| + } |
| +} |
| + |
| +// reconstruct removes b's row for the host and runs b's next tick: the row |
| +// a rebuilt registry, a box back from an outage or a row removed and |
| +// restored gets from ordinary reconciliation. |
| +func (w *e2eWorld) reconstruct(b *offBox) { |
| + w.t.Helper() |
| + if err := b.reg.Delete(offHost); err != nil { |
| + w.t.Fatal(err) |
| + } |
| + if b.row(offHost) != nil { |
| + w.t.Fatal("fixture: the row survived its removal") |
| + } |
| + w.tick(b) |
| +} |
| + |
| +func TestLifetimeAcrossAForwardMove(t *testing.T) { |
| + w := newE2EWorld(t, landOpts{lifetime: true}) |
| + retained := w.forwardMove() |
| + // The target's publication at open's sequence beats the source's |
| + // pair-less-in-sequence (1, 0) an edge still folds from before the |
| + // start. |
| + fresh := w.published(w.uk) |
| + requirePublished(t, "uk-box after finish", fresh, registry.Pair{Incarnation: 1, ServingSeq: 2}) |
| + if _, ok := publishedPair(t, w.published(w.ie)); ok { |
| + t.Fatal("ie-box still publishes the host after finish") |
| + } |
| + requireFoldWinner(t, "the target's (1, 2) against the retained source (1, 0)", fresh, retained, "uk-box") |
| +} |
| + |
| +func TestLifetimeAcrossAnAbort(t *testing.T) { |
| + w := newE2EWorld(t, landOpts{lifetime: true}) |
| + w.abortFromLanded() |
| + fresh := w.published(w.ie) |
| + requirePublished(t, "ie-box after the abort", fresh, registry.Pair{Incarnation: 1, ServingSeq: 2}) |
| + if _, ok := publishedPair(t, w.published(w.uk)); ok { |
| + t.Fatal("uk-box publishes the host after the abort") |
| + } |
| + // The target's landing row was held at (1, 1) and never published; a |
| + // ready publication of it is synthetic. |
| + requireFoldWinner(t, "the reopened source's (1, 2) against a synthetic target (1, 1)", |
| + fresh, syntheticReadyPublication(t, "uk-box", registry.Pair{Incarnation: 1, ServingSeq: 1}), "ie-box") |
| +} |
| + |
| +func TestReconstructedRowCarriesTheRecordsPair(t *testing.T) { |
| + t.Run("the target after a finished move", func(t *testing.T) { |
| + w := newE2EWorld(t, landOpts{lifetime: true}) |
| + w.forwardMove() |
| + // The row that stayed through the finish keeps open's (1, 2); |
| + // minted again from the record it takes the record's (1, 3). |
| + w.reconstruct(w.uk) |
| + w.requirePair(w.uk, registry.Pair{Incarnation: 1, ServingSeq: 3}, "", "the reconstructed target row") |
| + requirePublished(t, "uk-box's reconstructed row", w.published(w.uk), registry.Pair{Incarnation: 1, ServingSeq: 3}) |
| + w.tick(w.uk) |
| + w.requirePair(w.uk, registry.Pair{Incarnation: 1, ServingSeq: 3}, "", "a pass later") |
| + }) |
| + t.Run("the source after an abort from landed", func(t *testing.T) { |
| + w := newE2EWorld(t, landOpts{lifetime: true}) |
| + w.abortFromLanded() |
| + w.reconstruct(w.ie) |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 3}, "", "the reconstructed source row") |
| + requirePublished(t, "ie-box's reconstructed row", w.published(w.ie), registry.Pair{Incarnation: 1, ServingSeq: 3}) |
| + w.tick(w.ie) |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 3}, "", "a pass later") |
| + }) |
| +} |
| + |
| +// §11's form of this witness folds the reconstructed route against the |
| +// target's retained HOLDING publication; holding entries are plan 3's, so |
| +// here the target's landing pair stands as a synthetic ready publication. |
| +func TestReconstructedSourceAfterAbortWins(t *testing.T) { |
| + w := newE2EWorld(t, landOpts{lifetime: true}) |
| + w.abortFromLanded() |
| + w.reconstruct(w.ie) |
| + w.requirePair(w.ie, registry.Pair{Incarnation: 1, ServingSeq: 3}, "", "the reconstructed source row") |
| + fresh := w.published(w.ie) |
| + requirePublished(t, "ie-box's reconstructed row", fresh, registry.Pair{Incarnation: 1, ServingSeq: 3}) |
| + requireFoldWinner(t, "the reconstructed source's (1, 3) against a synthetic target (1, 1)", |
| + fresh, syntheticReadyPublication(t, "uk-box", registry.Pair{Incarnation: 1, ServingSeq: 1}), "ie-box") |
| +} |
| + |
| +// ---- recreation -------------------------------------------------------------- |
| + |
| +// moveOnTheRecord runs one whole offline move of the host, start to finish, |
| +// through the record writer alone (offWorld.move: SetRecordMove with the |
| +// transitions the doors apply), with no box taking part: what a box that |
| +// was down throughout finds afterwards. Each move advances move_seq by 3. |
| +func (wd *offWorld) moveOnTheRecord(id, from, to string) { |
| + wd.t.Helper() |
| + wd.move(provision.OfflineStart(id, from, to, e2eAddr, wd.clock())) |
| + wd.move(provision.OfflineSeal(id, "sealer", []string{"sealer"})) |
| + wd.move(provision.OfflineSealedBy(id, "sealer", provision.MoveSource{})) |
| + wd.move(provision.OfflineFlip(id)) |
| + wd.move(provision.OfflineLandedBy(id, to)) |
| + wd.move(provision.OfflineOpenTransition(id, wd.clock())) |
| + wd.move(provision.OfflineFinishingTransition(id)) |
| + wd.move(provision.OfflineFinish(id, wd.clock())) |
| +} |
| + |
| +// moveID is a well-formed offline move id, one per letter. |
| +func moveID(letter string) string { return "mv_" + strings.Repeat(letter, 26) } |
| + |
| +// retireThroughTombstone is the bucket the console's retirement leaves |
| +// (lifetimes_retire.go), step by step through the writers it uses: the |
| +// record replaced by its tombstone on the ETag read (provision.WriteRecord), |
| +// the slot retired on its ETag, the tombstone deleted on its own ETag. |
| +func (wd *offWorld) retireThroughTombstone(host string) { |
| + wd.t.Helper() |
| + ctx := context.Background() |
| + key := provision.RecordKey(wd.acct, "app", host) |
| + raw, etag, err := wd.s.Get(ctx, key) |
| + if err != nil { |
| + wd.t.Fatal(err) |
| + } |
| + rec, err := provision.DecodeRecord(raw) |
| + if err != nil { |
| + wd.t.Fatal(err) |
| + } |
| + slot, slotETag, err := provision.ReadHostIncarnation(ctx, wd.s, host) |
| + if err != nil || slot == nil || slot.State != provision.SlotLive || slot.Incarnation != rec.Incarnation { |
| + wd.t.Fatalf("fixture: slot %+v (%v) for a record at %d", slot, err, rec.Incarnation) |
| + } |
| + tomb, err := provision.TombstoneBody(raw) |
| + if err != nil { |
| + wd.t.Fatal(err) |
| + } |
| + tombETag, _, err := provision.WriteRecord(ctx, wd.s, key, raw, tomb, etag) |
| + if err != nil { |
| + wd.t.Fatal(err) |
| + } |
| + retired := provision.HostIncarnation{Account: wd.acct, App: "app", Incarnation: slot.Incarnation, State: provision.SlotRetired, Intent: "retire e2e"} |
| + if _, err := provision.WriteHostIncarnation(ctx, wd.s, host, retired, slotETag); err != nil { |
| + wd.t.Fatal(err) |
| + } |
| + if err := release.DeleteIfMatch(ctx, wd.s, key, tombETag); err != nil { |
| + wd.t.Fatal(err) |
| + } |
| +} |
| + |
| +// recreateLifetime is the console's create over the retired slot |
| +// (lifetimes_create.go): the slot advanced to {inc, live} on the retired |
| +// slot's ETag, then the record at inc, created in region. |
| +func (wd *offWorld) recreateLifetime(host string, inc uint64, region string) { |
| + wd.t.Helper() |
| + ctx := context.Background() |
| + slot, slotETag, err := provision.ReadHostIncarnation(ctx, wd.s, host) |
| + if err != nil || slot == nil || slot.State != provision.SlotRetired || slot.Incarnation+1 != inc { |
| + wd.t.Fatalf("fixture: slot %+v (%v) before the create of %d", slot, err, inc) |
| + } |
| + live := provision.HostIncarnation{Account: wd.acct, App: "app", Incarnation: inc, State: provision.SlotLive, Intent: "create e2e-recreated"} |
| + if _, err := provision.WriteHostIncarnation(ctx, wd.s, host, live, slotETag); err != nil { |
| + wd.t.Fatal(err) |
| + } |
| + rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], Account: wd.acct, App: "app", |
| + Region: region, Channel: "stable", CreatedAt: wd.clock(), Incarnation: inc} |
| + if err := provision.CreateRecord(ctx, wd.s, rec); err != nil { |
| + wd.t.Fatal(err) |
| + } |
| +} |
| + |
| +// lateBoxesAtNine is the recreation scenes' start: lifetime 1 of the host |
| +// moved three times while uk-a and uk-b (two more boxes in uk) were down, |
| +// then both back, each minting its row from the record at (1, 9) and |
| +// publishing it. |
| +func lateBoxesAtNine(t *testing.T) (w *e2eWorld, a, b *offBox) { |
| + t.Helper() |
| + w = newE2EWorld(t, landOpts{lifetime: true}) |
| + a, b = w.wd.box("uk-a", "uk", nil), w.wd.box("uk-b", "uk", nil) |
| + w.wd.moveOnTheRecord(moveID("b"), "", "uk") |
| + w.wd.moveOnTheRecord(moveID("c"), "uk", "") |
| + w.wd.moveOnTheRecord(moveID("d"), "", "uk") |
| + if rec := w.wd.record(); rec.Incarnation != 1 || rec.MoveSeq != 9 || rec.Region != "uk" { |
| + t.Fatalf("fixture: the record is at (%d, %d) in %q", rec.Incarnation, rec.MoveSeq, rec.Region) |
| + } |
| + for _, bx := range []*offBox{a, b} { |
| + w.tick(bx) |
| + w.requirePair(bx, registry.Pair{Incarnation: 1, ServingSeq: 9}, "", bx.name+" back from its outage") |
| + requirePublished(t, bx.name, w.published(bx), registry.Pair{Incarnation: 1, ServingSeq: 9}) |
| + } |
| + return w, a, b |
| +} |
| + |
| +func TestRecreatedHostOutranksItsPreviousLifetime(t *testing.T) { |
| + w, a, b := lateBoxesAtNine(t) |
| + retained := w.published(b) // uk-b's last publication of lifetime 1 |
| + w.wd.retireThroughTombstone(offHost) |
| + w.wd.recreateLifetime(offHost, 2, "uk") |
| + |
| + // uk-a reconciles into the recreation: its ready row of lifetime 1 |
| + // takes lifetime 2 at the new record's (2, 0). |
| + w.tick(a) |
| + w.requirePair(a, registry.Pair{Incarnation: 2, ServingSeq: 0}, "", "uk-a after the recreation") |
| + fresh := w.published(a) |
| + requirePublished(t, "uk-a after the recreation", fresh, registry.Pair{Incarnation: 2, ServingSeq: 0}) |
| + requireFoldWinner(t, "the recreated host's (2, 0) against uk-b's retained (1, 9)", fresh, retained, "uk-a") |
| + // The brief's synthetic retained (1, 9), from a box this world does not |
| + // run, loses the same way. |
| + requireFoldWinner(t, "the recreated host's (2, 0) against a synthetic retained (1, 9)", |
| + fresh, syntheticReadyPublication(t, "ie-box", registry.Pair{Incarnation: 1, ServingSeq: 9}), "uk-a") |
| + |
| + // uk-b, back in touch, converges on the same pair. |
| + w.tick(b) |
| + w.requirePair(b, registry.Pair{Incarnation: 2, ServingSeq: 0}, "", "uk-b after the recreation") |
| +} |
| + |
| +// rowWrite is one write of the host's registry row, as a trigger on the |
| +// box's own registry database saw it. |
| +type rowWrite struct { |
| + Status, Op string |
| + Pair registry.Pair |
| +} |
| + |
| +// rowWrites installs a trigger on b's registry database that logs every |
| +// insert and update of a routes row, and returns a reader of the host's |
| +// writes since the last read. |
| +func rowWrites(t *testing.T, b *offBox) func() []rowWrite { |
| + t.Helper() |
| + db, err := sql.Open("sqlite", b.dd+"/registry.db") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { db.Close() }) |
| + for _, stmt := range []string{ |
| + `CREATE TABLE IF NOT EXISTS t9_writes (n INTEGER PRIMARY KEY AUTOINCREMENT, host TEXT, status TEXT, hold_op TEXT, incarnation INTEGER, serving_seq INTEGER)`, |
| + `CREATE TRIGGER IF NOT EXISTS t9_u AFTER UPDATE ON routes BEGIN INSERT INTO t9_writes (host, status, hold_op, incarnation, serving_seq) VALUES (new.host, new.status, new.hold_op, new.incarnation, new.serving_seq); END`, |
| + `CREATE TRIGGER IF NOT EXISTS t9_i AFTER INSERT ON routes BEGIN INSERT INTO t9_writes (host, status, hold_op, incarnation, serving_seq) VALUES (new.host, new.status, new.hold_op, new.incarnation, new.serving_seq); END`, |
| + } { |
| + if _, err := db.Exec(stmt); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + return func() []rowWrite { |
| + t.Helper() |
| + rows, err := db.Query(`SELECT status, hold_op, incarnation, serving_seq FROM t9_writes WHERE host = ? ORDER BY n`, offHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer rows.Close() |
| + var out []rowWrite |
| + for rows.Next() { |
| + var rw rowWrite |
| + if err := rows.Scan(&rw.Status, &rw.Op, &rw.Pair.Incarnation, &rw.Pair.ServingSeq); err != nil { |
| + t.Fatal(err) |
| + } |
| + out = append(out, rw) |
| + } |
| + if err := rows.Err(); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := db.Exec(`DELETE FROM t9_writes`); err != nil { |
| + t.Fatal(err) |
| + } |
| + return out |
| + } |
| +} |
| + |
| +func TestRecreatedHostWithAStartedMoveIsHeldAtOnce(t *testing.T) { |
| + w, a, _ := lateBoxesAtNine(t) |
| + w.wd.retireThroughTombstone(offHost) |
| + w.wd.recreateLifetime(offHost, 2, "uk") |
| + id := moveID("e") |
| + w.wd.move(provision.OfflineStart(id, "uk", "", e2eAddr, w.wd.clock())) |
| + if rec := w.wd.record(); rec.Incarnation != 2 || rec.MoveSeq != 1 || rec.Move == nil || rec.Move.Seq != 1 { |
| + t.Fatalf("fixture: the record is at (%d, %d), move %+v", rec.Incarnation, rec.MoveSeq, rec.Move) |
| + } |
| + |
| + // uk-a slept through the retirement, the recreation and the start: its |
| + // row is still lifetime 1's, ready at (1, 9). Its next reconcile holds |
| + // it for the new lifetime's move in one write, never a ready row at |
| + // the hold's pair. |
| + w.requirePair(a, registry.Pair{Incarnation: 1, ServingSeq: 9}, "", "uk-a before its reconcile") |
| + writes := rowWrites(t, a) |
| + a.reconcile() |
| + got := writes() |
| + if len(got) != 1 || got[0].Op != id || !registry.IsHold(got[0].Status) || got[0].Pair != (registry.Pair{Incarnation: 2, ServingSeq: 1}) { |
| + t.Fatalf("uk-a's row writes = %+v, want one: held by %s at (2, 1)", got, id) |
| + } |
| + w.requirePair(a, registry.Pair{Incarnation: 2, ServingSeq: 1}, id, "uk-a after its reconcile") |
| + |
| + // The rest of the tick: the box stops publishing the held host, and a |
| + // second pass leaves the row as it is. |
| + w.publish(a) |
| + a.pass() |
| + if _, ok := publishedPair(t, w.published(a)); ok { |
| + t.Fatal("uk-a publishes its held row") |
| + } |
| + w.tick(a) |
| + for _, rw := range writes() { |
| + if rw.Status == "ready" || rw.Pair != (registry.Pair{Incarnation: 2, ServingSeq: 1}) { |
| + t.Fatalf("a later pass wrote the row %+v", rw) |
| + } |
| + } |
| + w.requirePair(a, registry.Pair{Incarnation: 2, ServingSeq: 1}, id, "uk-a a pass later") |
| +} |
amadan