carlos / platform Public

provision: reconcile installs a lifetime on a row with its hold in one write and never moves another lifetime's row

RouteFor mints a row at the record's (incarnation, move_seq). Under a
judge, an existing provisioned row of the record's own app and account
follows the existing-row rules, each a separate write under the host's
hold lock: a predecessor's row held by an op the record does not carry
is deleted through DeleteIfHold first and the record minted on the same
pass (nil callback, refusal, error or dry run: logged, skipped); a hold
writes through SetHoldIfSeq with ReconcileConfig.PairOf, and mintHeld
mints at that pair; an unheld legacy row takes the lifetime and keeps
its sequence, an unheld predecessor's row takes the lifetime and the
record's move_seq (InstallLifetimeIf); the same lifetime changes
nothing. PairOf nil keeps SetHoldIf, the pair untouched.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude agent, operated by paul@keymail.dev 7eb497f9b79789c5b5c866778e7bd441a4ed7b71 parent 2c82428
2 files changed, +648 −5
  • internal/provision/reconcile.go +158 −4
  • internal/provision/reconcile_lifetime_test.go +490 −1
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index fc69eb11d..fb84e5aef 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -147,8 +147,18 @@ type ReconcileConfig struct {
// timers (registry.DeleteIfHoldWithTimers). The removal phase uses it,
// under a judge, for a held row whose own record is established gone
// or stale: the record carries no op, so the hold cannot be its own.
- // Nil: such a row is removed as any other (claimRetire).
+ // Nil: such a row is removed as any other (claimRetire). The existing-
+ // row rules (Task 7, existingRowLifetime) use it for a predecessor's row
+ // held by an op the current record does not carry; nil there logs and
+ // leaves the row for a later pass.
DeleteIfHold func(host, op string) (bool, error)
+ // PairOf is the lifetime pair (registry.Pair) a hold for rec writes in
+ // the same statement as its status and op (self-serve moves plan 1 Task
+ // 7): holdExisting writes through SetHoldIfSeq and mintHeld mints the
+ // row at it. The host sets it on a claims-off box's offline hooks, as
+ // (rec.Incarnation, rec.Move.Seq). Nil: SetHoldIf, the pair untouched
+ // (Phase 1 exactly).
+ PairOf func(rec Record) registry.Pair
// beforeHoldWrite is the tests' pause under the host's hold lock, just
// before holdExisting's conditional write (TestReconcileHoldWrite...).
@@ -462,6 +472,13 @@ func RouteFor(rec Record, dataDir, seed string, nested bool, policy *UpdatePolic
// row it always did.
UpdateSettings: EffectiveUpdates(rec, policy).Column(),
SeedSource: SeedColumn(rec),
+ // The lifetime pair (design §6.4, "rows minted outside a move"): a
+ // row minted from the record takes its lifetime and its move_seq,
+ // the last finished or aborted move's; (0, 0) for a fieldless
+ // record with no move history. A held mint takes the hold's pair
+ // instead (mintHeld, ReconcileConfig.PairOf).
+ Incarnation: rec.Incarnation,
+ ServingSeq: rec.MoveSeq,
}
if rec.DerivesPaths() {
rt.Socket = DefaultSocket(rec.Host)
@@ -1253,15 +1270,29 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
}
seed, seedIsKnown = "", true
}
+ // Existing rows across lifetimes (self-serve moves plan 1 Task 7),
+ // rule (0): a predecessor's row held by an op this record does not
+ // carry is retired by that op first, whatever the record asks; the
+ // record is then minted on this pass as a missing row is.
+ if ok && heldByVanishedPredecessor(cfg, rec, existing) {
+ if !retireHeldPredecessor(cfg, rec, existing) {
+ continue
+ }
+ ok, existing = false, registry.Route{}
+ }
// A move's hold is judged first (moves spec §9.6). A missing row is
// minted held by mintHeld, which has a ready mint's refusals and
// the same claim hook (below) around its write, so the hold and the
// claim both have to allow the mint. An existing row is held in
// place with a narrow status write, outside the claim hook as the
// other status writers are, and then converged as usual: the owned
- // branch's narrow setters never touch status or hold_op.
+ // branch's narrow setters never touch status or hold_op. With
+ // cfg.PairOf the hold writes the lifetime pair in the same statement
+ // (rule (i)).
+ holdAsked := false
if cfg.Hold != nil {
if status, op, held := cfg.Hold(rec); held {
+ holdAsked = true
if !ok {
p, known := policyFor(rec.Account, rec.App)
mintHeld(ctx, reg, cfg, rec, seed, status, op, p, known)
@@ -1270,6 +1301,11 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
holdExisting(ctx, s, reg, cfg, rec, existing, status, op)
}
}
+ // Rules (ii) and (iii): an unheld row of an older lifetime that no
+ // hold is asked of takes this record's lifetime.
+ if ok && !holdAsked {
+ installRowLifetime(reg, cfg, rec, existing)
+ }
// Refusals that need no claim are decided BEFORE cfg.claimInstall: the
// hook activates the host guard and writes inventory, effects that a
@@ -1717,6 +1753,11 @@ func mintHeld(ctx context.Context, reg *registry.Registry, cfg ReconcileConfig,
}
nr := RouteFor(rec, cfg.DataDir, seed, false, policy)
nr.Status, nr.HoldOp = status, op
+ // The hold's lifetime pair lands with the hold, in the INSERT.
+ if cfg.PairOf != nil {
+ pair := cfg.PairOf(rec)
+ nr.Incarnation, nr.ServingSeq = pair.Incarnation, pair.ServingSeq
+ }
if cfg.ConfigGate != nil {
if err := cfg.ConfigGate(ctx, nr); err != nil {
return
@@ -1809,6 +1850,12 @@ func WithHostHoldLock(host string, fn func()) {
// never overwritten; the row is left to it and the next pass judges
// it afresh.
//
+// With cfg.PairOf (self-serve moves plan 1 Task 7, rule (i)) the write is
+// SetHoldIfSeq with the fresh record's pair: a legacy row at 0, a row of the
+// same lifetime and an unheld predecessor's row take the lifetime, the
+// status, the op and the pair together, so no ready row ever carries a
+// hold's pair, and a row of a newer lifetime is left alone.
+//
// SetHoldIf can wait out root's activation guard and fail (G30): logged,
// and the next pass tries again.
func holdExisting(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig, rec Record, existing registry.Route, status, op string) {
@@ -1852,13 +1899,18 @@ func holdExisting(ctx context.Context, s release.Store, reg *registry.Registry,
if cfg.beforeHoldWrite != nil {
cfg.beforeHoldWrite(rec.Host)
}
- held, err := reg.SetHoldIf(rec.Host, status, op, "")
+ var held bool
+ if cfg.PairOf != nil {
+ held, err = reg.SetHoldIfSeq(rec.Host, status, op, "", cfg.PairOf(*fresh))
+ } else {
+ held, err = reg.SetHoldIf(rec.Host, status, op, "")
+ }
if err != nil {
log.Printf("provision: hold %s %s by %s: %v", rec.Host, status, op, err)
return
}
if !held {
- log.Printf("provision: %s: not held %s by %s: the row's hold changed under this pass; the next pass judges it afresh", rec.Host, status, op)
+ log.Printf("provision: %s: not held %s by %s: the row's hold changed under this pass, or the row belongs to a later lifetime; the next pass judges it afresh", rec.Host, status, op)
return
}
log.Printf("provision: held %s %s by %s (was %s)", rec.Host, status, op, existing.Status)
@@ -1868,6 +1920,108 @@ func holdExisting(ctx context.Context, s release.Store, reg *registry.Registry,
})
}
+// The existing-row rules across lifetimes (self-serve moves plan 1 Task 7).
+// They apply under a judge (claims off) to a provisioned row of the record's
+// own app and account, in this order, each a separate write under the
+// host's hold lock:
+//
+// (0)/(iv) a predecessor's row held by an op the record does not carry:
+// DeleteIfHold under that op, then the record is minted on the
+// same pass (held if it asks for a hold, ready otherwise);
+// (i) the record asks for a hold: holdExisting, with the pair;
+// (ii) migration, unheld, no hold asked: the row takes the record's
+// lifetime and keeps its own sequence;
+// (iii) recreation, unheld, no hold asked: the row takes the record's
+// lifetime and move_seq;
+// (v) the same lifetime: nothing. A row held by the record's own
+// move is the move's.
+//
+// A row of another record key is not this pass's: the removal phase retires
+// it before the successor is minted.
+
+// ownsRowLifetime reports whether the existing-row rules apply to existing
+// for rec: a judged pass, and a row this pass minted for the record's own
+// app and account.
+func ownsRowLifetime(cfg ReconcileConfig, rec Record, existing registry.Route) bool {
+ return cfg.Judge != nil && existing.Provisioned && existing.App == rec.App && existing.Account == rec.Account
+}
+
+// heldByVanishedPredecessor is rule (iv)'s condition: the row belongs to an
+// older lifetime and is held by an op the record does not carry.
+func heldByVanishedPredecessor(cfg ReconcileConfig, rec Record, existing registry.Route) bool {
+ return ownsRowLifetime(cfg, rec, existing) && existing.Incarnation < rec.Incarnation && existing.HoldOp != "" &&
+ (rec.Move == nil || rec.Move.ID != existing.HoldOp)
+}
+
+// retireHeldPredecessor is rule (iv): the predecessor's row is deleted
+// only while the vanished op still holds it, with the move's timers. true
+// means the row is gone and the record is minted on this pass; anything
+// else (no callback, a dry run, a refusal, an error) is logged and the
+// record is skipped until a later pass.
+func retireHeldPredecessor(cfg ReconcileConfig, rec Record, existing registry.Route) bool {
+ switch {
+ case cfg.DryRun:
+ log.Printf("provision: dry-run: would remove %s's row of lifetime %d held by %s, then mint lifetime %d",
+ rec.Host, existing.Incarnation, existing.HoldOp, rec.Incarnation)
+ return false
+ case cfg.DeleteIfHold == nil:
+ log.Printf("provision: %s: the row of lifetime %d is held by %s, which lifetime %d does not carry; nothing here can remove it, leaving it",
+ rec.Host, existing.Incarnation, existing.HoldOp, rec.Incarnation)
+ return false
+ }
+ var deleted bool
+ var err error
+ WithHostHoldLock(rec.Host, func() {
+ deleted, err = cfg.DeleteIfHold(rec.Host, existing.HoldOp)
+ })
+ switch {
+ case err != nil:
+ log.Printf("provision: remove %s's row of lifetime %d held by %s: %v", rec.Host, existing.Incarnation, existing.HoldOp, err)
+ return false
+ case !deleted:
+ log.Printf("provision: %s: not removing the row of lifetime %d: its hold by %s changed since this pass listed it; the next pass judges it afresh",
+ rec.Host, existing.Incarnation, existing.HoldOp)
+ return false
+ }
+ log.Printf("provision: removed %s's row of lifetime %d held by %s (lifetime %d carries no such move)",
+ rec.Host, existing.Incarnation, existing.HoldOp, rec.Incarnation)
+ return true
+}
+
+// installRowLifetime is rules (ii) and (iii): an unheld row of an older
+// lifetime takes the record's, by InstallLifetimeIf from the pair this pass
+// listed, so a row a hold or another install changed since is left alone.
+// A migrated legacy row (at 0) keeps its own sequence; a predecessor's row
+// takes the record's move_seq.
+func installRowLifetime(reg *registry.Registry, cfg ReconcileConfig, rec Record, existing registry.Route) {
+ if !ownsRowLifetime(cfg, rec, existing) || existing.HoldOp != "" || existing.Incarnation >= rec.Incarnation {
+ return
+ }
+ expect := registry.Pair{Incarnation: existing.Incarnation, ServingSeq: existing.ServingSeq}
+ next := registry.Pair{Incarnation: rec.Incarnation, ServingSeq: rec.MoveSeq}
+ why := "recreation"
+ if existing.Incarnation == 0 {
+ next.ServingSeq, why = existing.ServingSeq, "migration"
+ }
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would install lifetime (%d, %d) on %s (%s, was (%d, %d))",
+ next.Incarnation, next.ServingSeq, rec.Host, why, expect.Incarnation, expect.ServingSeq)
+ return
+ }
+ WithHostHoldLock(rec.Host, func() {
+ installed, err := reg.InstallLifetimeIf(rec.Host, expect, next)
+ switch {
+ case err != nil:
+ log.Printf("provision: install lifetime %d on %s: %v", next.Incarnation, rec.Host, err)
+ case !installed:
+ log.Printf("provision: %s: lifetime %d not installed: the row changed since this pass listed it; the next pass judges it afresh", rec.Host, next.Incarnation)
+ default:
+ log.Printf("provision: installed lifetime (%d, %d) on %s (%s, was (%d, %d))",
+ next.Incarnation, next.ServingSeq, rec.Host, why, expect.Incarnation, expect.ServingSeq)
+ }
+ })
+}
+
// gateWord renders a gate column for a log line: "open" or "set", never the
// hash — it has no business in a journal.
func gateWord(hash string) string {
diff --git a/internal/provision/reconcile_lifetime_test.go b/internal/provision/reconcile_lifetime_test.go
index 63afdcef0..2dce21778 100644
--- a/internal/provision/reconcile_lifetime_test.go
+++ b/internal/provision/reconcile_lifetime_test.go
@@ -2,9 +2,11 @@ package provision
import (
"context"
+ "database/sql"
"errors"
"fmt"
"os"
+ "path/filepath"
"strings"
"sync"
"testing"
@@ -30,7 +32,10 @@ type lifeScene struct {
s *release.DirStore
sqid string
reg *registry.Registry
+ path string // the registry's file
cfg ReconcileConfig
+ // dryDeleted is set by a test's DeleteIfHold that a dry run must not call.
+ dryDeleted bool
}
func newLifeScene(t *testing.T) *lifeScene {
@@ -39,7 +44,12 @@ func newLifeScene(t *testing.T) *lifeScene {
if err := PutManifest(ctx, s, sqid, "cart", Manifest{Schema: 1, App: "cart", Domain: "shop.example.com"}); err != nil {
t.Fatal(err)
}
- return &lifeScene{s: s, sqid: sqid, reg: regWith(t),
+ path := filepath.Join(t.TempDir(), "registry.db")
+ reg, err := registry.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &lifeScene{s: s, sqid: sqid, reg: reg, path: path,
cfg: ReconcileConfig{Region: "ie", DataDir: "/data", Judge: NewLifetimeJudge()}}
}
@@ -652,3 +662,482 @@ func TestDeleteIfHoldRefusalLeavesTheRowForTheNextPass(t *testing.T) {
t.Fatalf("DeleteIfHold calls %q, want mv_1 then mv_2", calls)
}
}
+
+// --- the lifetime pair on rows (self-serve moves plan 1 Task 7) -------------
+
+const (
+ pairMove = "mv_abcdefghijklmnopqrstuvwxyz" // X: the predecessor's move
+ pairMove2 = "mv_zyxwvutsrqponmlkjihgfedcba" // Y: the successor's own move
+)
+
+// offlineHoldAt is the offline hold as a box in region here asks it
+// (internal/host's offlineHold, over plain region codes): the source holds
+// moving from stopping through sealed, the target holds landing once the
+// record names it.
+func offlineHoldAt(here string) func(Record) (string, string, bool) {
+ return func(r Record) (string, string, bool) {
+ m := r.Move
+ if !m.Offline() {
+ return "", "", false
+ }
+ switch m.State {
+ case OfflineStopping, OfflineSealing, OfflineSealed:
+ if m.From == here {
+ return "moving", m.ID, true
+ }
+ case OfflineLanding, OfflineLanded:
+ if r.Region == m.To && m.To == here {
+ return "landing", m.ID, true
+ }
+ }
+ return "", "", false
+ }
+}
+
+// movePair is the pair a hold or a release names: the record's lifetime and
+// the move's sequence (internal/host's PairOf).
+func movePair(r Record) registry.Pair {
+ p := registry.Pair{Incarnation: r.Incarnation}
+ if r.Move != nil {
+ p.ServingSeq = r.Move.Seq
+ }
+ return p
+}
+
+// offline arms the scene's pass with the offline hold and the pair, as a
+// claims-off box's reconcileSerializedWith does.
+func (x *lifeScene) offline() {
+ x.cfg.Hold, x.cfg.PairOf = offlineHoldAt(x.cfg.Region), movePair
+ x.cfg.DeleteIfHold = x.reg.DeleteIfHoldWithTimers
+}
+
+// lifetime writes host's slot at inc, live, and the shop record at inc with
+// move_seq seq (inc 0: a fieldless record and no slot).
+func (x *lifeScene) lifetime(t *testing.T, inc, seq uint64) {
+ t.Helper()
+ rec := x.record("shop", inc)
+ rec.MoveSeq = seq
+ if inc > 0 {
+ x.putSlot(t, "shop", inc, SlotLive)
+ }
+ x.putRecord(t, rec)
+}
+
+// migrate is the lazy migration's end state for the record as it stands,
+// its move included: the slot at 1, live, and the record at incarnation 1.
+func (x *lifeScene) migrate(t *testing.T) {
+ t.Helper()
+ rec := x.current(t)
+ rec.Incarnation = 1
+ x.putSlot(t, "shop", 1, SlotLive)
+ x.putRecord(t, rec)
+}
+
+func (x *lifeScene) current(t *testing.T) Record {
+ t.Helper()
+ rec, err := GetRecord(ctx, x.s, x.sqid, "shop", lifeHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return *rec
+}
+
+// move applies offline transitions to the shop record through the writer.
+func (x *lifeScene) move(t *testing.T, muts ...func(*Record) error) Record {
+ t.Helper()
+ var rec *Record
+ for _, mut := range muts {
+ var err error
+ if rec, err = SetRecordMove(ctx, x.s, x.sqid, "shop", lifeHost, mut); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return *rec
+}
+
+// The move's steps from rest, for a host in ie moving to uk.
+var pairNow = time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+
+func startAs(id string) func(*Record) error {
+ return OfflineStart(id, "ie", "uk", "ops@example.com", pairNow)
+}
+func throughLanded(id string) []func(*Record) error {
+ return []func(*Record) error{OfflineSeal(id, "box-a", []string{"box-a"}),
+ OfflineSealedBy(id, "box-a", MoveSource{Release: "v1"}), OfflineFlip(id), OfflineLandedBy(id, "uk")}
+}
+
+func (x *lifeScene) pairIs(t *testing.T, want registry.Pair, status, op string) {
+ t.Helper()
+ rt := x.row(t)
+ if rt == nil {
+ t.Fatalf("no row, want %s %q at %+v", status, op, want)
+ }
+ if got := (registry.Pair{Incarnation: rt.Incarnation, ServingSeq: rt.ServingSeq}); got != want || rt.Status != status || rt.HoldOp != op {
+ t.Fatalf("row %s %q at %+v, want %s %q at %+v", rt.Status, rt.HoldOp, got, status, op, want)
+ }
+}
+
+// rowWrite is one insert or update of the host's row, as a trigger on the
+// registry's own table saw it.
+type rowWrite struct {
+ Status, Op string
+ Pair registry.Pair
+}
+
+// writes arms a hook after every registry write of a routes row (a trigger
+// in the registry's file) and returns the reader of the writes since the
+// last read.
+func (x *lifeScene) writes(t *testing.T) func() []rowWrite {
+ t.Helper()
+ db, err := sql.Open("sqlite", x.path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { db.Close() })
+ for _, stmt := range []string{
+ `CREATE TABLE IF NOT EXISTS t7_writes (n INTEGER PRIMARY KEY AUTOINCREMENT, host TEXT, status TEXT, hold_op TEXT, incarnation INTEGER, serving_seq INTEGER)`,
+ `CREATE TRIGGER IF NOT EXISTS t7_u AFTER UPDATE ON routes BEGIN INSERT INTO t7_writes (host, status, hold_op, incarnation, serving_seq) VALUES (new.host, new.status, new.hold_op, new.incarnation, new.serving_seq); END`,
+ `CREATE TRIGGER IF NOT EXISTS t7_i AFTER INSERT ON routes BEGIN INSERT INTO t7_writes (host, status, hold_op, incarnation, serving_seq) VALUES (new.host, new.status, new.hold_op, new.incarnation, new.serving_seq); END`,
+ } {
+ if _, err := db.Exec(stmt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return func() []rowWrite {
+ t.Helper()
+ rows, err := db.Query(`SELECT status, hold_op, incarnation, serving_seq FROM t7_writes WHERE host = ? ORDER BY n`, lifeHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer rows.Close()
+ var out []rowWrite
+ for rows.Next() {
+ var w rowWrite
+ if err := rows.Scan(&w.Status, &w.Op, &w.Pair.Incarnation, &w.Pair.ServingSeq); err != nil {
+ t.Fatal(err)
+ }
+ out = append(out, w)
+ }
+ if _, err := db.Exec(`DELETE FROM t7_writes`); err != nil {
+ t.Fatal(err)
+ }
+ return out
+ }
+}
+
+// pairChanges is the writes that moved the row's pair, from before.
+func pairChanges(before registry.Pair, ws []rowWrite) []rowWrite {
+ var out []rowWrite
+ for _, w := range ws {
+ if w.Pair != before {
+ out = append(out, w)
+ before = w.Pair
+ }
+ }
+ return out
+}
+
+func TestRouteForCopiesThePair(t *testing.T) {
+ rec := shopRecord("bab")
+ if rt := RouteFor(rec, "/data", "", false, nil); rt.Incarnation != 0 || rt.ServingSeq != 0 {
+ t.Fatalf("a fieldless record minted (%d, %d)", rt.Incarnation, rt.ServingSeq)
+ }
+ rec.Incarnation, rec.MoveSeq = 2, 3
+ rec.Move = &MoveState{Kind: MoveKindOffline, ID: pairMove, To: "uk", State: OfflineOpen, Seq: 7}
+ if rt := RouteFor(rec, "/data", "", true, nil); rt.Incarnation != 2 || rt.ServingSeq != 3 {
+ t.Fatalf("RouteFor minted (%d, %d), want the record's (2, 3)", rt.Incarnation, rt.ServingSeq)
+ }
+}
+
+// A row minted outside a move (a rebuilt registry) takes the record's pair:
+// the lifetime and the last finished or aborted move's move_seq.
+func TestReconcileInstallsTheRecordsPairOnAMintedRow(t *testing.T) {
+ for _, c := range []struct {
+ name, region string
+ muts []func(*Record) error
+ }{
+ {"after finish", "uk", append(append([]func(*Record) error{startAs(pairMove)}, throughLanded(pairMove)...),
+ OfflineOpenTransition(pairMove, pairNow.Add(time.Hour)), OfflineFinishingTransition(pairMove), OfflineFinish(pairMove, pairNow.Add(2*time.Hour)))},
+ {"after abort then finish", "ie", []func(*Record) error{startAs(pairMove), OfflineAbort(pairMove), OfflineFinish(pairMove, pairNow.Add(time.Hour))}},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ x := newLifeScene(t)
+ x.cfg.Region = c.region
+ x.offline()
+ x.lifetime(t, 1, 0)
+ if rec := x.move(t, c.muts...); rec.Move != nil || rec.MoveSeq != 3 || rec.Region != c.region {
+ t.Fatalf("fixture: record %+v move_seq %d region %q", rec.Move, rec.MoveSeq, rec.Region)
+ }
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 3}, "ready", "")
+ })
+ }
+}
+
+// The target's row is held at the hold's pair, opened at open's, and keeps
+// open's through finishing and finish: observing a newer record does not move
+// a row's sequence.
+func TestAnExistingRowKeepsOpenSequenceThroughFinish(t *testing.T) {
+ x := newLifeScene(t)
+ x.cfg.Region = "uk"
+ x.offline()
+ x.lifetime(t, 1, 0)
+ x.move(t, startAs(pairMove))
+ x.move(t, throughLanded(pairMove)[:3]...) // landing: the target holds
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 1}, "landing", pairMove)
+ x.move(t, OfflineLandedBy(pairMove, "uk"))
+ rec := x.move(t, OfflineOpenTransition(pairMove, pairNow.Add(time.Hour)))
+ // The target's open releases with the pair of the record it observed.
+ if ok, err := x.reg.ReleaseHoldSeq(lifeHost, pairMove, movePair(rec)); err != nil || !ok {
+ t.Fatalf("open's release: %v %v", ok, err)
+ }
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 2}, "ready", "")
+ for _, mut := range []func(*Record) error{OfflineFinishingTransition(pairMove), OfflineFinish(pairMove, pairNow.Add(2*time.Hour))} {
+ x.pass(t)
+ x.move(t, mut)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 2}, "ready", "")
+ }
+ if rec := x.current(t); rec.MoveSeq != 3 {
+ t.Fatalf("fixture: move_seq %d after finish", rec.MoveSeq)
+ }
+}
+
+func TestExistingRowsAcrossLifetimes(t *testing.T) {
+ t.Run("ready legacy row, migrated record", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.lifetime(t, 0, 0)
+ x.minted(t, "shop")
+ x.pairIs(t, registry.Pair{}, "ready", "")
+ log := x.writes(t)
+ x.lifetime(t, 1, 0)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 0}, "ready", "")
+ if ch := pairChanges(registry.Pair{}, log()); len(ch) != 1 {
+ t.Fatalf("pair writes %+v, want one install", ch)
+ }
+ // Same lifetime: a later move_seq does not move the row.
+ x.lifetime(t, 1, 5)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 0}, "ready", "")
+ if ch := pairChanges(registry.Pair{Incarnation: 1}, log()); len(ch) != 0 {
+ t.Fatalf("same lifetime wrote %+v", ch)
+ }
+ })
+
+ t.Run("ready legacy row keeps its own sequence through the migration", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.lifetime(t, 0, 4)
+ x.minted(t, "shop")
+ x.pairIs(t, registry.Pair{Incarnation: 0, ServingSeq: 4}, "ready", "")
+ x.lifetime(t, 1, 6)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 4}, "ready", "")
+ })
+
+ t.Run("ready row, recreation, no move", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.lifetime(t, 1, 9)
+ x.minted(t, "shop")
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 9}, "ready", "")
+ x.lifetime(t, 2, 0)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 0}, "ready", "")
+ })
+
+ t.Run("ready predecessor row, recreation with a started move", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.offline()
+ x.lifetime(t, 1, 9)
+ x.minted(t, "shop")
+ log := x.writes(t)
+ x.lifetime(t, 2, 0)
+ if rec := x.move(t, startAs(pairMove2)); movePair(rec) != (registry.Pair{Incarnation: 2, ServingSeq: 1}) {
+ t.Fatalf("fixture: the successor's hold names %+v", movePair(rec))
+ }
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 1}, "moving", pairMove2)
+ ws := log()
+ ch := pairChanges(registry.Pair{Incarnation: 1, ServingSeq: 9}, ws)
+ if len(ch) != 1 || ch[0].Status != "moving" || ch[0].Op != pairMove2 {
+ t.Fatalf("pair writes %+v, want one held write", ch)
+ }
+ for _, w := range ws {
+ if w.Op == "" && w.Pair.Incarnation == 2 {
+ t.Fatalf("a ready row carried the successor's pair: %+v (writes %+v)", w, ws)
+ }
+ }
+ })
+
+ t.Run("held legacy row, migration during a move", func(t *testing.T) {
+ for _, end := range []string{"open", "abort"} {
+ t.Run(end, func(t *testing.T) {
+ x := newLifeScene(t)
+ x.offline()
+ x.lifetime(t, 0, 0)
+ x.minted(t, "shop")
+ x.move(t, startAs(pairMove))
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 0, ServingSeq: 1}, "moving", pairMove)
+ log := x.writes(t)
+ x.migrate(t)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 0, ServingSeq: 1}, "moving", pairMove)
+ if ws := log(); len(ws) != 0 {
+ t.Fatalf("the installer wrote a row held by the record's own move: %+v", ws)
+ }
+ var rec Record
+ if end == "open" {
+ x.move(t, throughLanded(pairMove)...)
+ rec = x.move(t, OfflineOpenTransition(pairMove, pairNow.Add(time.Hour)))
+ } else {
+ rec = x.move(t, OfflineAbort(pairMove))
+ }
+ if ok, err := x.reg.ReleaseHoldSeq(lifeHost, pairMove, movePair(rec)); err != nil || !ok {
+ t.Fatalf("%s's release: %v %v", end, ok, err)
+ }
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 2}, "ready", "")
+ })
+ }
+ })
+
+ t.Run("hold across migration", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.offline()
+ x.lifetime(t, 0, 0)
+ x.minted(t, "shop")
+ log := x.writes(t)
+ x.lifetime(t, 1, 0)
+ x.move(t, startAs(pairMove))
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 1}, "moving", pairMove)
+ if ch := pairChanges(registry.Pair{}, log()); len(ch) != 1 || ch[0].Op != pairMove {
+ t.Fatalf("pair writes %+v, want one held write", ch)
+ }
+ })
+
+ // The predecessor's row held by a move its record no longer carries.
+ held := func(t *testing.T) *lifeScene {
+ x := newLifeScene(t)
+ x.offline()
+ x.lifetime(t, 1, 0)
+ x.minted(t, "shop")
+ x.move(t, startAs(pairMove))
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 1}, "moving", pairMove)
+ return x
+ }
+
+ t.Run("held predecessor by a vanished op", func(t *testing.T) {
+ x := held(t)
+ var calls []string
+ x.cfg.DeleteIfHold = func(host, op string) (bool, error) {
+ calls = append(calls, op)
+ return x.reg.DeleteIfHoldWithTimers(host, op)
+ }
+ x.lifetime(t, 2, 0)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 0}, "ready", "")
+ if len(calls) != 1 || calls[0] != pairMove {
+ t.Fatalf("DeleteIfHold calls %q, want the vanished op's", calls)
+ }
+ })
+
+ for _, c := range []struct {
+ name string
+ delete func(x *lifeScene) func(string, string) (bool, error)
+ dry bool
+ }{
+ {"held predecessor by a vanished op, no callback", func(*lifeScene) func(string, string) (bool, error) { return nil }, false},
+ {"held predecessor by a vanished op, refused", func(*lifeScene) func(string, string) (bool, error) {
+ return func(string, string) (bool, error) { return false, nil }
+ }, false},
+ {"held predecessor by a vanished op, failed", func(*lifeScene) func(string, string) (bool, error) {
+ return func(string, string) (bool, error) { return false, errors.New("registry busy") }
+ }, false},
+ {"held predecessor by a vanished op, dry run", func(x *lifeScene) func(string, string) (bool, error) {
+ return func(host, op string) (bool, error) {
+ x.dryDeleted = true
+ return x.reg.DeleteIfHoldWithTimers(host, op)
+ }
+ }, true},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ x := held(t)
+ x.cfg.DeleteIfHold, x.cfg.DryRun = c.delete(x), c.dry
+ x.lifetime(t, 2, 0)
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 1}, "moving", pairMove)
+ if x.dryDeleted {
+ t.Fatal("a dry run called DeleteIfHold")
+ }
+ })
+ }
+
+ t.Run("held predecessor, successor with its own move", func(t *testing.T) {
+ x := held(t)
+ var calls []string
+ x.cfg.DeleteIfHold = func(host, op string) (bool, error) {
+ calls = append(calls, op)
+ return x.reg.DeleteIfHoldWithTimers(host, op)
+ }
+ log := x.writes(t)
+ x.lifetime(t, 2, 0)
+ x.move(t, startAs(pairMove2))
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 1}, "moving", pairMove2)
+ if len(calls) != 1 || calls[0] != pairMove {
+ t.Fatalf("DeleteIfHold calls %q, want the vanished op's", calls)
+ }
+ ws := log()
+ if len(ws) == 0 || ws[0] != (rowWrite{"moving", pairMove2, registry.Pair{Incarnation: 2, ServingSeq: 1}}) {
+ t.Fatalf("writes %+v, want the held mint first", ws)
+ }
+ for _, w := range ws {
+ if w.Op == "" {
+ t.Fatalf("a ready row was written: %+v", ws)
+ }
+ }
+
+ // A stale setter: the predecessor's release, or a release naming
+ // lifetime 1 by the successor's own op, moves nothing.
+ for _, op := range []string{pairMove, pairMove2} {
+ if ok, err := x.reg.ReleaseHoldSeq(lifeHost, op, registry.Pair{Incarnation: 1, ServingSeq: 2}); err != nil || ok {
+ t.Fatalf("a release for lifetime 1 by %s: %v %v", op, ok, err)
+ }
+ }
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 1}, "moving", pairMove2)
+ })
+
+ t.Run("a stale setter", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.offline()
+ x.lifetime(t, 2, 0)
+ x.minted(t, "shop")
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 0}, "ready", "")
+ if ok, err := x.reg.ReleaseHoldSeq(lifeHost, pairMove, registry.Pair{Incarnation: 1, ServingSeq: 2}); err != nil || ok {
+ t.Fatalf("a release for lifetime 1: %v %v", ok, err)
+ }
+ if ok, err := x.reg.SetHoldIfSeq(lifeHost, "moving", pairMove, "", registry.Pair{Incarnation: 1, ServingSeq: 1}); err != nil || ok {
+ t.Fatalf("a hold for lifetime 1: %v %v", ok, err)
+ }
+ x.pairIs(t, registry.Pair{Incarnation: 2, ServingSeq: 0}, "ready", "")
+ })
+
+ t.Run("Phase 1 unchanged", func(t *testing.T) {
+ x := newLifeScene(t)
+ x.offline()
+ x.cfg.PairOf = nil
+ x.lifetime(t, 1, 3)
+ x.minted(t, "shop")
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 3}, "ready", "")
+ x.move(t, startAs(pairMove))
+ x.pass(t)
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 3}, "moving", pairMove)
+ if ok, err := x.reg.ReleaseHold(lifeHost, pairMove); err != nil || !ok {
+ t.Fatalf("release: %v %v", ok, err)
+ }
+ x.pairIs(t, registry.Pair{Incarnation: 1, ServingSeq: 3}, "ready", "")
+ })
+}