amadan

carlos / platform Public

instance moves P0 (#175): fence the cut against tick wakes, refuse abort past ready, refuse moving a replica

Squashed from instance-moves-p0 into main.
claude 408db5effd3e65e19ec09c383faa3ce16378c383 parent 4d5eab8
9 files changed, +388 −4
  • internal/activator/activator.go +12 −0
  • internal/activator/activator_test.go +55 −0
  • internal/activator/maintenance.go +19 −0
  • internal/console/apiinstancemove.go +8 −1
  • internal/console/apiinstancemove_test.go +15 −0
  • internal/host/movespass.go +95 −3
  • internal/host/movespass_test.go +137 −0
  • internal/provision/provision.go +17 −0
  • internal/provision/provision_test.go +30 −0

CI cancelled — superseded by a newer push — run details

landed from instance-moves-p0

diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index b8889e66a..16ef6357a 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -2774,7 +2774,19 @@ func (a *Service) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.
// request is the only place a signal could ride. A tick never leaves the
// process (schedule_wiring.go: "the tick never leaves the box"), so the call
// site IS the signal, and there is no header for an outsider to send.
+//
+// A tick never wakes a managed route this box has stopped serving (a status
+// other than ready: the residence-move cut's "moving", instance-moves design
+// G12). The due-wake sweep holds the same gate. The edge never hands the
+// activator such a row, but the runner fires whatever row the registry
+// returns, and a tick that woke a host mid-cut would restore it and let it
+// write behind the move's final mirror round. The move worker's fence closes
+// the race in-process; this gate is the half that survives an agent restart.
func (a *Service) ServeScheduled(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
+ if rt.Kind == "instance" && rt.Hibernate && rt.Status != "" && rt.Status != "ready" {
+ maintenanceResponse(w, r)
+ return true
+ }
return a.serve(w, r, rt, proxy, false)
}
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
index 9039e321c..93f38aac7 100644
--- a/internal/activator/activator_test.go
+++ b/internal/activator/activator_test.go
@@ -1776,6 +1776,61 @@ func TestServeScheduledWakesASleepingInstance(t *testing.T) {
}
}
+// A tick never wakes a route the box has stopped serving (G12): the move
+// cut sets "moving", and a tick that woke the host there would restore it
+// and let it write behind the move's final mirror round. This gate is the
+// half that survives an agent restart, when the in-memory fence is gone.
+func TestServeScheduledRefusesARouteThatIsNotReady(t *testing.T) {
+ const host = "moving.bab.oncarlos.com"
+ a, _, _, _ := testActivator(t, time.Hour)
+ rt := testRoute(host)
+ rt.Status = "moving"
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil)
+ if !a.ServeScheduled(rec, req, rt, okHandler()) {
+ t.Fatal("ServeScheduled returned false: the runner would deliver to the socket itself")
+ }
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status %d, want 503", rec.Code)
+ }
+ a.mu.Lock()
+ e := a.entries[rt.Host]
+ a.mu.Unlock()
+ if e != nil {
+ t.Fatal("a tick woke a moving route")
+ }
+}
+
+// LiftFence reopens only the fence it names: a stale generation leaves a
+// newer fence closed, and the right one lets the next tick wake the host.
+func TestLiftFenceReopensOnlyItsOwnGeneration(t *testing.T) {
+ const host = "fenced.bab.oncarlos.com"
+ a, _, _, _ := testActivator(t, time.Hour)
+ rt := testRoute(host)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ if _, err := a.FenceAndStop(ctx, host, 7); err != nil && !errors.Is(err, ErrStopUnproven) {
+ t.Fatalf("FenceAndStop = %v", err)
+ }
+ if a.LiftFence(host, 6) || a.LiftFence(host, 0) {
+ t.Fatal("a stale generation lifted the fence")
+ }
+ rec := httptest.NewRecorder()
+ a.ServeScheduled(rec, httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil), rt, okHandler())
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("fenced tick status %d, want 503", rec.Code)
+ }
+ if !a.LiftFence(host, 7) {
+ t.Fatal("the fence's own generation did not lift it")
+ }
+ rec = httptest.NewRecorder()
+ a.ServeScheduled(rec, httptest.NewRequest("POST", "http://carlos/jobs/sweep", nil), rt, okHandler())
+ if rec.Code != http.StatusOK {
+ t.Fatalf("tick after the lift: status %d, want 200", rec.Code)
+ }
+}
+
// The guard: a host whose next platform tick is nearer than the idle window
// stays resident. Sleeping would buy a few minutes of absence at the price of
// a full checkpoint-and-restore round trip, and then an immediate wake.
diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go
index 7717dd27e..ca1b1206a 100644
--- a/internal/activator/maintenance.go
+++ b/internal/activator/maintenance.go
@@ -124,6 +124,25 @@ func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint
}
}
+// LiftFence removes the fence FenceAndStop installed at generation, and only
+// that one: a fence at any other generation stays, so a caller holding a stale
+// generation cannot reopen admission somebody else closed. It reports whether
+// it lifted anything. The caller owns the proof that nothing on the far side
+// of the fence still needs it closed; the move worker lifts only once this
+// box no longer holds the host's route (instance-moves design §6.1, G12).
+func (a *Service) LiftFence(host string, generation uint64) bool {
+ if a == nil || generation == 0 {
+ return false
+ }
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ if a.fences[host] != generation {
+ return false
+ }
+ delete(a.fences, host)
+ return true
+}
+
func (a *Service) lockFence(ctx context.Context) error {
timer := time.NewTicker(time.Millisecond)
defer timer.Stop()
diff --git a/internal/console/apiinstancemove.go b/internal/console/apiinstancemove.go
index 0c7466c91..0204869e8 100644
--- a/internal/console/apiinstancemove.go
+++ b/internal/console/apiinstancemove.go
@@ -177,10 +177,17 @@ func (s *Server) handleAPIInstanceMoveStatus(w http.ResponseWriter, r *http.Requ
}
func (s *Server) handleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Request) {
- acct, app, host, _, ok := s.moveGate(w, r)
+ acct, app, host, rec, ok := s.moveGate(w, r)
if !ok {
return
}
+ // A replica is refused at the request only (G14): moveGate also fronts
+ // the status and abort doors, and a replica that got a move before this
+ // rail existed must still be able to read and abort it.
+ if rec.Role != "" {
+ jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region")
+ return
+ }
var req struct {
Region string `json:"region"`
}
diff --git a/internal/console/apiinstancemove_test.go b/internal/console/apiinstancemove_test.go
index be75ef761..a92ecb25e 100644
--- a/internal/console/apiinstancemove_test.go
+++ b/internal/console/apiinstancemove_test.go
@@ -186,6 +186,21 @@ func TestAPIInstanceMoveClassRails(t *testing.T) {
}
_ = fmt.Sprint()
_ = provisionPooledMoveRefusal(t, fx, token, u)
+
+ // A replica (G14): refused at the request, while the status door still
+ // answers, since the gate it shares must not strand a replica's move.
+ rep := provision.Record{Schema: 1, Host: "rep.go.tito.io", Slug: "rep", App: "titogo",
+ Account: "bab", Region: "ie", Role: "replica", Channel: "stable", CreatedAt: time.Now().UTC()}
+ if err := provision.CreateRecord(context.Background(), fx.rel, rep); err != nil {
+ t.Fatal(err)
+ }
+ code, _, out = rawReq(t, http.MethodPost, u+"/rep.go.tito.io/move", token, `{"region":"uk"}`)
+ if code != http.StatusBadRequest || !strings.Contains(out, "replica") {
+ t.Fatalf("move replica = %d %s, want 400 mentioning the replica rail", code, out)
+ }
+ if code, _, out = rawReq(t, http.MethodGet, u+"/rep.go.tito.io/move", token, ""); code != http.StatusOK {
+ t.Fatalf("move status of a replica = %d %s, want 200", code, out)
+ }
}
// provisionPooledMoveRefusal pins the pool rail: a pooled record's data
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index ad66f0ffe..6ec19b166 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -35,6 +35,7 @@ import (
// The cut ordering is the never-two-writers argument, mechanically:
//
// route Status "moving" (edge stops serving; RetireExcept stops the child)
+// → wake fence (no tick or sweep can wake it again — G12)
// → child quiesced (Quiesced(host) — retries next pass until true)
// → final mirror round (own replica is byte-final; staging matches it)
// → local purge (the warm-cache trap: a stale local file must
@@ -50,11 +51,17 @@ import (
// the once-per-host status nudge, and the per-host trouble throttle.
// RunAgent writes the pointer once; both converge loops read it.
type movesWorker struct {
- cfg moveCfg
+ cfg moveCfg
+ // pass runs one movesPass at a time. The fast converge and the
+ // instances loop both call runMovesPass, and two concurrent passes
+ // could drive one cut twice, or lift a fence from a record list read
+ // before the cut that installed it.
+ pass sync.Mutex
mu sync.Mutex
mirrors map[string]lease.MirrorStore // dstRegion → client pair
nudged sync.Map // host → true: status→quiesce hop nudged
trouble sync.Map // host → time.Time of last logged error
+ fenced sync.Map // host → uint64: the cut's wake-fence generation
}
var armedMovesWorker atomic.Pointer[movesWorker]
@@ -92,7 +99,20 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
Mirror: func(ctx context.Context, dstRegion string) (lease.MirrorStore, error) {
return lease.NewS3Mirror(ctx, hib.Region, dstRegion)
},
- Quiesced: svc.Quiesced,
+ Quiesced: svc.Quiesced,
+ Fence: func(ctx context.Context, host string, generation uint64) error {
+ // A short deadline: FenceAndStop waits for an awake child to
+ // stop, and one slow stop must not hold up every other move in
+ // the pass. ErrStopUnproven still means the fence is installed,
+ // which is all the cut asks of it; Quiesced answers the rest.
+ fctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ if _, err := svc.FenceAndStop(fctx, host, generation); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
+ return err
+ }
+ return nil
+ },
+ LiftFence: svc.LiftFence,
SetStatus: reg.SetStatus,
OwnsRoute: func(host string) bool {
rt, err := reg.Get(host)
@@ -134,7 +154,17 @@ type moveCfg struct {
DataDir string // /data — where the local DB lives
Mirror func(ctx context.Context, dstRegion string) (lease.MirrorStore, error)
Quiesced func(host string) bool
- SetStatus func(host, status string) error
+ // Fence installs the activator's wake fence for host (G12, the
+ // instance-moves design §6.1): status "moving" stops the edge, but the
+ // tick runner fires whatever row the registry returns, and a tick that
+ // woke the source after the purge would restore it from its own replica
+ // and write where the target never looks. A nil error means the fence
+ // is installed, whether or not the child has finished stopping. Nil
+ // skips (tests without an activator).
+ Fence func(ctx context.Context, host string, generation uint64) error
+ // LiftFence removes a fence Fence installed, at that generation only.
+ LiftFence func(host string, generation uint64) bool
+ SetStatus func(host, status string) error
// OwnsRoute answers whether THIS box's registry serves the host
// (review round 1): region equality alone would let a standby or
// second same-region box drive a cut whose child runs on its sibling —
@@ -158,6 +188,8 @@ type moveCfg struct {
// movesPass runs one round over every in-flight move this box owns.
func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
+ w.pass.Lock()
+ defer w.pass.Unlock()
cfg := w.cfg
recs, err := provision.ListAllRecords(ctx, s)
if err != nil {
@@ -196,9 +228,58 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
}
w.trouble.Delete(rec.Host)
}
+ w.liftFences(recs)
return nil
}
+// moveFenceGeneration is the cut fence's generation: the move's request
+// time, so every pass of one cut installs the same fence (FenceAndStop is
+// idempotent at an equal generation) and a later move of the same host
+// fences above an earlier one.
+func moveFenceGeneration(m *provision.MoveState) uint64 {
+ if n := m.RequestedAt.UnixNano(); n > 0 {
+ return uint64(n)
+ }
+ return 1
+}
+
+// liftFences lifts each cut fence once nothing on this box needs it: the
+// host's route is gone from this box's registry (the flip retired it), or
+// the record is at rest here again (a move back landed). Before either,
+// the fence holds, including after the flip while the "moving" row waits
+// for reconcile to retire it. The fences live in the activator's memory, so
+// an agent restart drops them all; ServeScheduled's status gate covers that
+// window, and a cut still in flight re-fences on its next pass.
+func (w *movesWorker) liftFences(recs []provision.Record) {
+ if w.cfg.LiftFence == nil {
+ return
+ }
+ atRest := map[string]bool{}
+ for _, rec := range recs {
+ if rec.Move == nil && rec.Region == w.cfg.Region {
+ atRest[rec.Host] = true
+ }
+ }
+ w.fenced.Range(func(k, v any) bool {
+ host, gen := k.(string), v.(uint64)
+ held := true
+ if w.cfg.Route != nil {
+ rt, err := w.cfg.Route(host)
+ if err != nil {
+ return true // an unreadable registry is not a retired route
+ }
+ held = rt != nil
+ }
+ if held && !atRest[host] {
+ return true
+ }
+ w.cfg.LiftFence(host, gen)
+ w.fenced.CompareAndDelete(host, gen)
+ log.Printf("moves: %s: wake fence lifted", host)
+ return true
+ })
+}
+
// moveTargets folds the fleet's edges files into placement → coordinates.
// A duplicate placement code keeps the lexically-first box (deterministic
// everywhere, same rule as foldHomes) — and a target with no data bucket is
@@ -278,6 +359,17 @@ func runMoveRound(ctx context.Context, s release.Store, w *movesWorker, rec prov
if err := cfg.SetStatus(rec.Host, "moving"); err != nil {
return fmt.Errorf("set route status: %w", err)
}
+ // Fence before the quiesce check, so the answer Quiesced gives stays
+ // true: without it a tick could wake the host between this pass's
+ // check and the purge (G12). Every pass re-installs the same fence,
+ // which is also how a restarted agent gets it back.
+ if cfg.Fence != nil {
+ gen := moveFenceGeneration(rec.Move)
+ if err := cfg.Fence(ctx, rec.Host, gen); err != nil {
+ return fmt.Errorf("fence against wakes: %w", err)
+ }
+ w.fenced.Store(rec.Host, gen)
+ }
if !cfg.Quiesced(rec.Host) {
if _, done := w.nudged.LoadOrStore(rec.Host, true); !done {
nudgeSerial(ctx, s)
diff --git a/internal/host/movespass_test.go b/internal/host/movespass_test.go
index 19fd26b2f..9fca637a1 100644
--- a/internal/host/movespass_test.go
+++ b/internal/host/movespass_test.go
@@ -2,6 +2,7 @@ package host
import (
"context"
+ "errors"
"os"
"path/filepath"
"strings"
@@ -339,6 +340,142 @@ func TestMovesPassCutUsesRouteDBPath(t *testing.T) {
}
}
+// The cut fences the host against wakes before it trusts Quiesced (G12),
+// holds the fence through the flip while this box still has the row, and
+// lifts it once the row is retired.
+func TestMovesPassCutFencesWakesUntilTheRouteRetires(t *testing.T) {
+ h := newMoveHarness(t)
+ host := "a.go.example.com"
+ rec := h.record(t, host, provision.MoveRequest("uk", time.Now()), provision.MovePromoteReady(time.Now()), provision.MoveMarkCutting())
+ h.f.put("data-ie", "instances/"+host+"/generations/g/snapshot", "e1")
+ fences := map[string]uint64{}
+ h.cfg.Fence = func(_ context.Context, host string, gen uint64) error {
+ fences[host] = gen
+ return nil
+ }
+ h.cfg.LiftFence = func(host string, gen uint64) bool {
+ if fences[host] != gen {
+ return false
+ }
+ delete(fences, host)
+ return true
+ }
+ routeHeld := true
+ h.cfg.Route = func(string) (*registry.Route, error) {
+ if !routeHeld {
+ return nil, nil
+ }
+ return &registry.Route{Host: host, Status: "moving"}, nil
+ }
+ h.cfg.Quiesced = func(string) bool {
+ if fences[host] == 0 {
+ t.Fatal("Quiesced was asked before the wake fence was installed")
+ }
+ return h.quiet
+ }
+ w := h.worker()
+
+ // Not quiesced: fenced, nothing else.
+ h.quiet = false
+ if err := movesPass(context.Background(), h.s, w); err != nil {
+ t.Fatal(err)
+ }
+ want := uint64(rec.Move.RequestedAt.UnixNano())
+ if fences[host] != want {
+ t.Fatalf("fence generation = %d, want the move's request time %d", fences[host], want)
+ }
+ if len(h.f.ops) != 0 {
+ t.Fatalf("mirror ran before quiesce: %v", h.f.ops)
+ }
+
+ // Quiesced: the cut completes and the fence stays, since the row is
+ // still here waiting for reconcile to retire it.
+ h.quiet = true
+ if err := movesPass(context.Background(), h.s, w); err != nil {
+ t.Fatal(err)
+ }
+ if got := h.read(t, host); got.Region != "uk" || got.Move != nil {
+ t.Fatalf("after cut: region=%q move=%+v", got.Region, got.Move)
+ }
+ if fences[host] != want {
+ t.Fatal("the fence was lifted while this box still held the moving row")
+ }
+
+ // Reconcile retires the row: the next pass lifts the fence.
+ routeHeld = false
+ if err := movesPass(context.Background(), h.s, w); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := fences[host]; ok {
+ t.Fatal("the fence outlived the route")
+ }
+}
+
+// A fence on a host whose record is at rest here again (a move back landed)
+// is lifted even though this box holds the row: the row serves now.
+func TestMovesPassLiftsTheFenceOfARecordAtRestHere(t *testing.T) {
+ h := newMoveHarness(t)
+ host := "a.go.example.com"
+ h.record(t, host) // region ie, no move: at rest on this box
+ lifted := uint64(0)
+ h.cfg.LiftFence = func(_ string, gen uint64) bool { lifted = gen; return true }
+ h.cfg.Route = func(string) (*registry.Route, error) { return &registry.Route{Host: host, Status: "ready"}, nil }
+ w := h.worker()
+ w.fenced.Store(host, uint64(42))
+ if err := movesPass(context.Background(), h.s, w); err != nil {
+ t.Fatal(err)
+ }
+ if lifted != 42 {
+ t.Fatalf("lifted generation %d, want 42", lifted)
+ }
+ if _, ok := w.fenced.Load(host); ok {
+ t.Fatal("the worker still remembers a lifted fence")
+ }
+}
+
+// Passes run one at a time: the fast converge and the instances loop both
+// call runMovesPass, and a pass that lifted fences from a record list read
+// before another pass's cut would reopen wakes mid-cut.
+func TestMovesPassRunsOneAtATime(t *testing.T) {
+ h := newMoveHarness(t)
+ w := h.worker()
+ w.pass.Lock()
+ done := make(chan error, 1)
+ go func() { done <- movesPass(context.Background(), h.s, w) }()
+ select {
+ case <-done:
+ t.Fatal("a second pass ran while the first held the worker")
+ case <-time.After(50 * time.Millisecond):
+ }
+ w.pass.Unlock()
+ if err := <-done; err != nil {
+ t.Fatal(err)
+ }
+}
+
+// A fence that cannot be installed stops the cut before it looks at the
+// child: nothing is mirrored, purged or flipped.
+func TestMovesPassCutStopsWhenTheFenceFails(t *testing.T) {
+ h := newMoveHarness(t)
+ host := "a.go.example.com"
+ h.record(t, host, provision.MoveRequest("uk", time.Now()), provision.MovePromoteReady(time.Now()), provision.MoveMarkCutting())
+ h.f.put("data-ie", "instances/"+host+"/generations/g/snapshot", "e1")
+ h.cfg.Fence = func(context.Context, string, uint64) error { return errors.New("fence not installed") }
+ h.cfg.Quiesced = func(string) bool {
+ t.Fatal("Quiesced was asked with no fence in place")
+ return true
+ }
+ if err := movesPass(context.Background(), h.s, h.worker()); err != nil {
+ t.Fatal(err)
+ }
+ if len(h.f.ops) != 0 {
+ t.Fatalf("the cut ran without a fence: %v", h.f.ops)
+ }
+ if got := h.read(t, host); got.Region != "ie" || got.Move == nil || got.Move.State != provision.MoveCutting {
+ t.Fatalf("after a failed fence: region=%q move=%+v, want still cutting in ie", got.Region, got.Move)
+ }
+}
+
func writeFileForTest(path, content string) error { return os.WriteFile(path, []byte(content), 0o600) }
func fileExistsForTest(path string) bool { _, err := os.Stat(path); return err == nil }
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index 34be6f41f..2b399c424 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -971,6 +971,13 @@ func MoveRequest(to string, now time.Time) func(*Record) error {
if rec.Region == to {
return fmt.Errorf("%w: the instance is already in %s", ErrMoveConflict, to)
}
+ // A replica never gets a route on a platform box (reconcile's mint
+ // gate), so no box owns its move and it would wedge in preparing
+ // (G14). Checked here as well as at the console's door because
+ // records have other writers.
+ if rec.Role != "" {
+ return fmt.Errorf("%w: a %s does not move. delete it and create one in the new region", ErrMoveConflict, rec.Role)
+ }
rec.Move = &MoveState{To: to, State: MovePreparing, RequestedAt: now.UTC()}
return nil
}
@@ -1005,16 +1012,26 @@ func MoveMarkCutting() func(*Record) error {
// MoveAbort marks a move for abort — legal before the point of no return.
// The clear itself is the source box's (MoveAbortComplete), after it has
// deleted the staged copy.
+//
+// The legal states are listed, not the illegal ones, so a state this
+// console has never heard of is refused (G16, instance-moves design §6.1).
+// A newer box writes "landing" after the flip, while the target restores
+// from staging. Aborting it would send the target down the aborting arm to
+// delete that staging before it promoted, and the first wake would seed an
+// empty database.
func MoveAbort() func(*Record) error {
return func(rec *Record) error {
if rec.Move == nil {
return fmt.Errorf("%w: no move in flight", ErrMoveConflict)
}
switch rec.Move.State {
+ case MovePreparing, MoveReady:
case MoveCutting:
return fmt.Errorf("%w: past the point of no return — the cut completes forward", ErrMoveConflict)
case MoveAborting:
return fmt.Errorf("%w: already aborting", ErrMoveConflict)
+ default:
+ return fmt.Errorf("%w: a move in state %q cannot be aborted", ErrMoveConflict, rec.Move.State)
}
rec.Move.State = MoveAborting
return nil
diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go
index 0f6609ea2..a97933440 100644
--- a/internal/provision/provision_test.go
+++ b/internal/provision/provision_test.go
@@ -976,6 +976,36 @@ func TestSetRecordMoveTransitions(t *testing.T) {
}
}
+// Abort names the states it may leave, so a state a newer box writes is
+// refused here rather than aborted (G16): "landing" is the target restoring
+// from the staging an abort would delete.
+func TestMoveAbortRefusesStatesItDoesNotKnow(t *testing.T) {
+ for _, state := range []string{"landing", "freezing", "done", ""} {
+ rec := &Record{Move: &MoveState{To: "uk", State: state}}
+ if err := MoveAbort()(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Errorf("abort from %q = %v, want ErrMoveConflict", state, err)
+ }
+ if rec.Move.State != state {
+ t.Errorf("abort from %q rewrote the state to %q", state, rec.Move.State)
+ }
+ }
+ for _, state := range []string{MovePreparing, MoveReady} {
+ rec := &Record{Move: &MoveState{To: "uk", State: state}}
+ if err := MoveAbort()(rec); err != nil || rec.Move.State != MoveAborting {
+ t.Errorf("abort from %q = %v, state %q; want aborting", state, err, rec.Move.State)
+ }
+ }
+}
+
+// A replica never gets a route on a platform box, so a move of one would
+// wedge in preparing with no box to drive it (G14).
+func TestMoveRequestRefusesAReplica(t *testing.T) {
+ rec := &Record{Region: "ie", Role: "replica"}
+ if err := MoveRequest("uk", time.Now())(rec); !errors.Is(err, ErrMoveConflict) || rec.Move != nil {
+ t.Fatalf("move of a replica = %v, move %+v; want ErrMoveConflict and no move", err, rec.Move)
+ }
+}
+
// A record written before the move field existed round-trips byte-identical
// through the CAS helper's read-marshal cycle — the wire-shape rule.
func TestMoveFieldAbsentOnPreMoveRecords(t *testing.T) {