carlos / platform Public

moves wait for a seed's placement or an import's commit, in both doors and in place (Plan 8 Task 3)

Spec §5.8: a --from seed moves only once placed, an import only once
committed. provision.seedMoveRefusal (SD-N100, SD-N101 in seed.go) is the
first rule of MoveRequest and MoveStartTransition, under the record CAS,
so the claim-mode start (hostops/move_enrollment.go) gets it too. Both
move doors call refuseSeedMove (seedmove.go) after their gate, before the
Phase 1 door's probe transition and claim CAS, so each answers the
sentence itself.

carlos apps place re-homes every record of the app (preflight M8(a)):
placeAppTarget reads the records with ListRecordsStrict before any write
and refuses 409 with the sentence; a record it cannot read refuses the
place (502, the existing "could not update this app's instances"
sentence). SetRecordRegion refuses a blocked seed under its CAS when the
region changes, and the place answers that sentence too
(provision.SeedMoveError).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 367ad94d6d1da1e5009fe64317be094e7511d6d2 parent cf39691

9 files changed, +490 −8

approved_seeded_copy_test.go +4 −2
internal/console/apiinstancemove.go +6 −0
internal/console/fleets.go +20 −1
internal/console/fleets_test.go +7 −3
internal/console/seedmove.go +72 −0
internal/console/seedmove_test.go +240 −0
internal/provision/provision.go +11 −0
internal/provision/seed.go +32 −0
internal/provision/seed_test.go +98 −2

CI passed — run details

diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go

index 09012e136..01bc28133 100644

--- a/approved_seeded_copy_test.go

+++ b/approved_seeded_copy_test.go

@@ -37,7 +37,8 @@ var seededCopyIDs = func() []string {

// Task 5 SD-N20 to SD-N26, SD-N30 to SD-N39 and SD-N41; Plan 5 Task 6 SD-N40, SD-N42 to SD-N47, SD-N83 to SD-N88 and SD-N114;

// Plan 6 Task 1 SD-N48 to SD-N50, SD-N62 and SD-N67; Plan 6 Task 2 SD-N51 to SD-N53, SD-N59, SD-N61 and SD-N63;

// Plan 6 Task 3 SD-N54, SD-N55, SD-N60, SD-N64, SD-N65 and SD-N115; Plan 6 Task 5 SD-N56 to SD-N58 and SD-N66;

-// Plan 6 Task 6 SD-N68 to SD-N82, SD-N89 and SD-N90; Plan 7 Task 3 SD-N91 to SD-N98; Plan 7 Task 6 SD-N99.

+// Plan 6 Task 6 SD-N68 to SD-N82, SD-N89 and SD-N90; Plan 7 Task 3 SD-N91 to SD-N98; Plan 7 Task 6 SD-N99;

+// Plan 8 Task 3 SD-N100 and SD-N101.

var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19", "SD-N27", "SD-N28", "SD-N29",

"SD-N20", "SD-N21", "SD-N22", "SD-N23", "SD-N24", "SD-N25", "SD-N26", "SD-N30", "SD-N31", "SD-N32", "SD-N33", "SD-N34", "SD-N35", "SD-N36", "SD-N37", "SD-N38", "SD-N39", "SD-N41",

"SD-N40", "SD-N42", "SD-N43", "SD-N44", "SD-N45", "SD-N46", "SD-N47", "SD-N114", "SD-N83", "SD-N84", "SD-N85", "SD-N86", "SD-N87", "SD-N88",

@@ -48,7 +49,8 @@ var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N

"SD-N68", "SD-N69", "SD-N70", "SD-N71", "SD-N72", "SD-N73", "SD-N74", "SD-N75", "SD-N76", "SD-N77", "SD-N78", "SD-N79",

"SD-N80", "SD-N81", "SD-N82", "SD-N89", "SD-N90",

"SD-N91", "SD-N92", "SD-N93", "SD-N94", "SD-N95", "SD-N96", "SD-N97", "SD-N98",

- "SD-N99"}

+ "SD-N99",

+ "SD-N100", "SD-N101"}

func mustParseSeededRecord(t *testing.T) []copyRow {

t.Helper()

diff --git a/internal/console/apiinstancemove.go b/internal/console/apiinstancemove.go

index dd020172d..52e5761bb 100644

--- a/internal/console/apiinstancemove.go

+++ b/internal/console/apiinstancemove.go

@@ -262,6 +262,9 @@ func (s *Server) legacyHandleAPIInstanceMoveRequest(w http.ResponseWriter, r *ht

if !ok {

return

}

+ if s.refuseSeedMove(w, rec) {

+ return

+ }

// A replica is refused at the request only (G14): moveGate also fronts

// the status and abort doors, and a replica that got a move before this

// rail existed must still be able to read and abort it.

@@ -405,6 +408,9 @@ func (s *Server) handleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Req

if !ok {

return

}

+ if s.refuseSeedMove(w, rec) {

+ return

+ }

// Host-claim consoles start the move under the app's admission: the

// claim runtime's authority registers it (RegisterMove), and the start

// is hostops.BeginClaimMove then hostops.StartClaimMoveRecord, never a

diff --git a/internal/console/fleets.go b/internal/console/fleets.go

index 472c8ab20..fc257c2e0 100644

--- a/internal/console/fleets.go

+++ b/internal/console/fleets.go

@@ -502,6 +502,10 @@ func (s *Server) appPlacement(reg consoleRegistry, acct *Account, app string) (s

return "", nil

}

+// placeInstancesFailed is placeAppTarget's answer when it cannot read the

+// app's instance records.

+const placeInstancesFailed = "could not update this app's instances — try again"

// placeAppTarget is placement's whole mechanism, and the ONLY place it

// lives: repoints where an app's routes (and its already-provisioned

// instances) should run. target "" clears placement back to

@@ -601,6 +605,18 @@ func (s *Server) placeAppTarget(ctx context.Context, acct *Account, app, target

}

+ // A place re-homes every record of the app, so a seed that may not move

+ // yet (spec §5.8) refuses it, before any write. A record that cannot be

+ // read refuses it too. SetRecordRegion refuses such a seed again under

+ // its CAS, for a record created after this read.

+ switch block, err := placeSeedBlock(ctx, s.cfg.Releases, acct.Sqid, app); {

+ case err != nil:

+ log.Printf("console: place %s/%s: read instances: %v", acct.Sqid, app, err)

+ return http.StatusBadGateway, placeInstancesFailed

+ case block != "":

+ return http.StatusConflict, provision.SeedMoveSentence(block)

+ }

// The store-of-record write (2026-08-06 ruling): appPlacement consults

// THIS before it ever walks a route. It has to land before routes/records

// are stamped below, not after — the same "never observe an app in a

@@ -661,7 +677,7 @@ func (s *Server) placeAppTarget(ctx context.Context, acct *Account, app, target

keys, err := s.cfg.Releases.List(ctx, prefix)

if err != nil {

log.Printf("console: place %s/%s: list instances: %v", acct.Sqid, app, err)

- return http.StatusBadGateway, "could not update this app's instances — try again"

+ return http.StatusBadGateway, placeInstancesFailed

}

for _, key := range keys {

rest, ok := strings.CutPrefix(key, prefix)

@@ -683,6 +699,9 @@ func (s *Server) placeAppTarget(ctx context.Context, acct *Account, app, target

log.Printf("console: place %s/%s: instance %s vanished mid-walk, skipping", acct.Sqid, app, host)

continue

}

+ if msg := seedMoveConflict(err); msg != "" {

+ return http.StatusConflict, msg

+ }

log.Printf("console: place %s/%s: stamp instance %s: %v", acct.Sqid, app, host, err)

return http.StatusBadGateway, "could not update every instance — try again"

}

diff --git a/internal/console/fleets_test.go b/internal/console/fleets_test.go

index adaf2e77c..37e3d2b54 100644

--- a/internal/console/fleets_test.go

+++ b/internal/console/fleets_test.go

@@ -487,7 +487,7 @@ func TestPlaceStampsRoutesAndInstances(t *testing.T) {

}

// vanishOnGetStore wraps a release.Store and deletes ONE key out from under

-// itself the first time that key is Get — a stand-in for I4's race: an

+// itself the first time that key is Get after skip Gets — a stand-in for I4's race: an

// instance record retired (DeleteRecord) in the gap between handleAPIPlace's

// List of a prefix and its per-key SetRecordRegion write reaching that same

// key. Every other key, and every other method, is the embedded Store's real

@@ -495,11 +495,14 @@ func TestPlaceStampsRoutesAndInstances(t *testing.T) {

type vanishOnGetStore struct {

release.Store

vanish string

+ skip int // Gets of vanish to let through first

deleted bool

}

func (v *vanishOnGetStore) Get(ctx context.Context, key string) ([]byte, string, error) {

- if key == v.vanish && !v.deleted {

+ if key == v.vanish && !v.deleted && v.skip > 0 {

+ v.skip--

+ } else if key == v.vanish && !v.deleted {

v.deleted = true

if err := v.Store.Delete(ctx, key); err != nil {

return nil, "", err

@@ -546,7 +549,8 @@ func TestPlaceSkipsVanishedRecordMidWalk(t *testing.T) {

// found it, exactly the race SetRecordRegion's os.ErrNotExist-wrapped

// return exists to name.

ghostKey := provision.RecordKey(fx.sqid, "titogo", "ghost.go.tito.io")

- fx.srv.cfg.Releases = &vanishOnGetStore{Store: fx.rel, vanish: ghostKey}

+ // skip 1: the place's strict read (seedmove.go) reads ghost first.

+ fx.srv.cfg.Releases = &vanishOnGetStore{Store: fx.rel, vanish: ghostKey, skip: 1}

status, body := postJSON(t, placeURL(fx, "titogo"), map[string]string{"target": "fleet/pi"}, token)

if status != http.StatusOK {

diff --git a/internal/console/seedmove.go b/internal/console/seedmove.go

new file mode 100644

index 000000000..534b725b0

--- /dev/null

+++ b/internal/console/seedmove.go

@@ -0,0 +1,72 @@

+package console

+import (

+ "context"

+ "errors"

+ "fmt"

+ "math"

+ "net/http"

+ "os"

+ "slices"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+// refuseSeedMove answers a move of a seed that is not placed (--from) or

+// not committed (import) with the deck's sentence (spec §5.8). Both move

+// doors call it after their gate and before their transition, so each

+// answers the sentence itself; the transitions refuse it again under the

+// record CAS.

+func (s *Server) refuseSeedMove(w http.ResponseWriter, rec *provision.Record) bool {

+ if b := rec.Seed.MoveBlock(); b != "" {

+ jsonError(w, http.StatusConflict, provision.SeedMoveSentence(b))

+ return true

+ }

+ return false

+// placeSeedBlock is carlos apps place's seed rule (preflight M8(a)): a

+// place re-homes every record of the app, so it is a move of each. It reads

+// the app's records strictly and returns the first record's MoveBlock, ""

+// when none blocks. A record it cannot read could be anything, so it is an

+// error and the place is refused. A record deleted between the listing and

+// its read (the race the place's own walk skips) is read again once.

+//

+// No cap: the place walks every record of the app, and a cap would refuse

+// a large app's place for good.

+func placeSeedBlock(ctx context.Context, st release.Store, acct, app string) (string, error) {

+ ctx, cancel := context.WithTimeout(ctx, instanceStoreTimeout)

+ defer cancel()

+ recs, unreadable, err := provision.ListRecordsStrict(ctx, st, acct, app, math.MaxInt)

+ if errors.Is(err, os.ErrNotExist) {

+ recs, unreadable, err = provision.ListRecordsStrict(ctx, st, acct, app, math.MaxInt)

+ }

+ if err != nil {

+ return "", err

+ }

+ if len(unreadable) > 0 {

+ hosts := make([]string, 0, len(unreadable))

+ for h := range unreadable {

+ hosts = append(hosts, h)

+ }

+ slices.Sort(hosts)

+ return "", fmt.Errorf("records that do not read: %v", hosts)

+ }

+ for _, rec := range recs {

+ if b := rec.Seed.MoveBlock(); b != "" {

+ return b, nil

+ }

+ return "", nil

+// seedMoveConflict is the sentence of a write the record's seed refused

+// under its CAS (provision.SeedMoveError), "" for any other error.

+func seedMoveConflict(err error) string {

+ var refusal *provision.SeedMoveError

+ if errors.As(err, &refusal) {

+ return provision.SeedMoveSentence(refusal.Block)

+ }

+ return ""

diff --git a/internal/console/seedmove_test.go b/internal/console/seedmove_test.go

new file mode 100644

index 000000000..651992956

--- /dev/null

+++ b/internal/console/seedmove_test.go

@@ -0,0 +1,240 @@

+package console

+import (

+ "context"

+ "errors"

+ "net/http"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/seeds"

+const seedMoveHost = "canary.example.com"

+// seedMoveFixture is a console whose ops account runs the shop app with

+// instances on, and a member's CLI token (an operator's when phase1, with

+// the Phase 1 switch on).

+func seedMoveFixture(t *testing.T, code string, phase1 bool) (*consoleFixture, string) {

+ t.Helper()

+ fx := newConsoleFixture(t)

+ now := time.Now()

+ if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", now); err != nil {

+ t.Fatal(err)

+ }

+ if err := fx.st.AddMember(OpsAccountID, "alice@keymail.dev", RoleMember, now); err != nil {

+ t.Fatal(err)

+ }

+ token := cliToken(t, fx, "alice@keymail.dev", code)

+ if phase1 {

+ if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {

+ t.Fatal(err)

+ }

+ enableMovesPhase1(fx)

+ }

+ enableInstances(t, fx, "shop", provision.Manifest{Schema: 1, App: "shop", Domain: "example.com",

+ Channel: "stable", Regions: []string{"ie"}})

+ return fx, token

+// putSeededRecord creates an instance record of the shop app in region,

+// holding a seed of kind in state.

+func putSeededRecord(t *testing.T, st release.Store, host, region, kind, state string) {

+ t.Helper()

+ id, err := seeds.NewID()

+ if err != nil {

+ t.Fatal(err)

+ }

+ rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], App: "shop", Account: release.OpsAccount,

+ Region: region, Channel: "stable", CreatedAt: time.Now().UTC(),

+ Seed: &provision.SeedSource{ID: id, Kind: kind, State: state, By: "paul@keymail.dev", At: time.Now().UTC()}}

+ if err := provision.CreateRecord(context.Background(), st, rec); err != nil {

+ t.Fatal(err)

+ }

+// Both move doors answer a move of an unplaced --from seed or an

+// uncommitted import with the deck's sentence (spec §5.8), before any

+// write: no journal, and on the Phase 1 door no claim.

+func TestMoveDoorsRefuseAnUnplacedSeed(t *testing.T) {

+ for _, door := range []struct {

+ name string

+ phase1 bool

+ }{{"legacy", false}, {"phase 1", true}} {

+ for _, c := range []struct{ kind, state, want string }{

+ {seeds.KindReplica, seeds.StateReady, "wake the instance once before moving it"},

+ {seeds.KindImport, seeds.StateSwapped, "commit or abort the import first"},

+ } {

+ fx, token := seedMoveFixture(t, "code-seed-move", door.phase1)

+ putSeededRecord(t, fx.rel, seedMoveHost, "ie", c.kind, c.state)

+ code, _, raw := rawReq(t, http.MethodPost, instancesURL(fx, release.OpsAccount, "shop")+"/"+seedMoveHost+"/move", token, `{"region":"uk"}`)

+ if code != http.StatusConflict || errorText(raw) != c.want {

+ t.Fatalf("%s door, %s %s = %d %s", door.name, c.kind, c.state, code, raw)

+ }

+ got, err := provision.GetRecord(context.Background(), fx.rel, release.OpsAccount, "shop", seedMoveHost)

+ if err != nil || got.Move != nil {

+ t.Fatalf("%s door: a refused move left a journal: %+v, %v", door.name, got, err)

+ }

+ if claim := claimBytes(t, fx, seedMoveHost); claim != "" {

+ t.Fatalf("%s door: a refused move wrote a claim: %s", door.name, claim)

+ }

+// A placed seed is not the helper's to refuse: the legacy door goes on to

+// its own checks (here, no armed regions).

+func TestMoveDoorLetsAPlacedSeedThrough(t *testing.T) {

+ fx, token := seedMoveFixture(t, "code-seed-move-placed", false)

+ putSeededRecord(t, fx.rel, seedMoveHost, "ie", seeds.KindReplica, seeds.StatePlaced)

+ code, _, raw := rawReq(t, http.MethodPost, instancesURL(fx, release.OpsAccount, "shop")+"/"+seedMoveHost+"/move", token, `{"region":"uk"}`)

+ if code != http.StatusBadRequest || errorText(raw) != "this deployment publishes no armed regions — nothing to move to" {

+ t.Fatalf("placed seed = %d %s", code, raw)

+ }

+// placeSeedFixture is fleetFixture with the shop app, one route and an

+// owned fleet "pi" to place it on.

+func placeSeedFixture(t *testing.T, code string) (*consoleFixture, string) {

+ t.Helper()

+ fx, token := fleetFixture(t, code)

+ if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", time.Now()); err != nil {

+ t.Fatal(err)

+ }

+ if status, body := postJSON(t, fleetsURL(fx, fx.sqid)+"/new",

+ map[string]string{"name": "pi", "bucket": "pi-fleet-data"}, token); status != http.StatusOK {

+ t.Fatalf("create fleet = %d %v", status, body)

+ }

+ return fx, token

+// assertNotPlaced checks a refused place wrote nothing: the app's placement

+// and every record's region are as before.

+func assertNotPlaced(t *testing.T, fx *consoleFixture, hosts ...string) {

+ t.Helper()

+ if p, err := fx.st.AppPlacement(OpsAccountID, "shop"); err != nil || p != "" {

+ t.Fatalf("a refused place recorded placement %q, %v", p, err)

+ }

+ for _, h := range hosts {

+ got, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "shop", h)

+ if err != nil || got.Region != "ie" {

+ t.Fatalf("a refused place re-homed %s: %+v, %v", h, got, err)

+ }

+// carlos apps place re-homes every record of the app, so it is a move too

+// (preflight M8(a)): an unplaced --from seed or an uncommitted import

+// refuses it, with the deck's sentence, before any write.

+func TestPlaceRefusesAnUnplacedSeed(t *testing.T) {

+ for _, c := range []struct{ kind, state, want string }{

+ {seeds.KindReplica, seeds.StateReady, "wake the instance once before moving it"},

+ {seeds.KindImport, seeds.StatePlaced, "commit or abort the import first"},

+ } {

+ fx, token := placeSeedFixture(t, "code-place-seed")

+ putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced)

+ putSeededRecord(t, fx.rel, seedMoveHost, "ie", c.kind, c.state)

+ code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`)

+ if code != http.StatusConflict || errorText(raw) != c.want {

+ t.Fatalf("%s %s: place = %d %s", c.kind, c.state, code, raw)

+ }

+ assertNotPlaced(t, fx, "plain.example.com", seedMoveHost)

+ }

+// A record the place cannot read could be a seed that may not move, so the

+// place is refused (it fails closed): a record that will not decode, and a

+// read that fails.

+func TestPlaceRefusesWhenARecordCannotBeRead(t *testing.T) {

+ fx, token := placeSeedFixture(t, "code-place-unreadable")

+ putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced)

+ bad := provision.RecordKey(fx.sqid, "shop", "broken.example.com")

+ if _, err := fx.rel.Put(context.Background(), bad, []byte("{not json"), ""); err != nil {

+ t.Fatal(err)

+ }

+ code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`)

+ if code != http.StatusBadGateway || errorText(raw) != "could not update this app's instances — try again" {

+ t.Fatalf("an undecodable record: place = %d %s", code, raw)

+ }

+ assertNotPlaced(t, fx, "plain.example.com")

+ if err := fx.rel.Delete(context.Background(), bad); err != nil {

+ t.Fatal(err)

+ }

+ fx.srv.cfg.Releases = &failGetStore{Store: fx.rel, key: provision.RecordKey(fx.sqid, "shop", "plain.example.com")}

+ code, _, raw = rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`)

+ if code != http.StatusBadGateway || errorText(raw) != "could not update this app's instances — try again" {

+ t.Fatalf("a failed read: place = %d %s", code, raw)

+ }

+ assertNotPlaced(t, fx, "plain.example.com")

+// A record deleted between the strict read's listing and its read is the

+// I4 race one step earlier: the read is taken again, and the place lands.

+func TestPlaceReadsAgainWhenARecordVanishes(t *testing.T) {

+ fx, token := placeSeedFixture(t, "code-place-seed-vanish")

+ putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced)

+ putSeededRecord(t, fx.rel, "gone.example.com", "ie", seeds.KindReplica, seeds.StatePlaced)

+ fx.srv.cfg.Releases = &vanishOnGetStore{Store: fx.rel, vanish: provision.RecordKey(fx.sqid, "shop", "gone.example.com")}

+ code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`)

+ if code != http.StatusOK {

+ t.Fatalf("place = %d %s", code, raw)

+ }

+ got, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "shop", "plain.example.com")

+ if err != nil || got.Region != "fleet/pi" {

+ t.Fatalf("after the place: %+v, %v", got, err)

+ }

+// The backstop: a seeded record created after the place's read is refused

+// under SetRecordRegion's CAS, and the place answers the same sentence.

+func TestPlaceAnswersTheBackstopsSentence(t *testing.T) {

+ fx, token := placeSeedFixture(t, "code-place-seed-backstop")

+ putSeededRecord(t, fx.rel, "plain.example.com", "ie", seeds.KindReplica, seeds.StatePlaced)

+ fx.srv.cfg.Releases = &createOnSecondListStore{Store: fx.rel, prefix: provision.RecordsPrefix(fx.sqid, "shop"), create: func(st release.Store) {

+ putSeededRecord(t, st, seedMoveHost, "ie", seeds.KindImport, seeds.StateSwapped)

+ }}

+ code, _, raw := rawReq(t, http.MethodPost, placeURL(fx, "shop"), token, `{"target":"fleet/pi"}`)

+ if code != http.StatusConflict || errorText(raw) != "commit or abort the import first" {

+ t.Fatalf("place = %d %s", code, raw)

+ }

+ got, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "shop", seedMoveHost)

+ if err != nil || got.Region != "ie" {

+ t.Fatalf("the late import was re-homed: %+v, %v", got, err)

+ }

+// failGetStore fails every Get of one key.

+type failGetStore struct {

+ release.Store

+ key string

+func (f *failGetStore) Get(ctx context.Context, key string) ([]byte, string, error) {

+ if key == f.key {

+ return nil, "", errors.New("injected read failure")

+ }

+ return f.Store.Get(ctx, key)

+// createOnSecondListStore runs create once, just before the second List of

+// prefix: after the place's strict read, before its walk.

+type createOnSecondListStore struct {

+ release.Store

+ prefix string

+ create func(release.Store)

+ lists int

+func (c *createOnSecondListStore) List(ctx context.Context, prefix string) ([]string, error) {

+ if prefix == c.prefix {

+ c.lists++

+ if c.lists == 2 {

+ c.create(c.Store)

+ }

+ return c.Store.List(ctx, prefix)

diff --git a/internal/provision/provision.go b/internal/provision/provision.go

index 22e7c51ca..2e5cc45e8 100644

--- a/internal/provision/provision.go

+++ b/internal/provision/provision.go

@@ -892,6 +892,11 @@ func CreateRecord(ctx context.Context, s release.Store, rec Record) error {

// nothing.

func SetRecordRegion(ctx context.Context, s release.Store, acct, app, host, region string) error {

_, err := updateRecord(ctx, s, acct, app, host, func(rec *Record) error {

+ if rec.Region != region {

+ if err := seedMoveRefusal(rec); err != nil {

+ return err

+ }

rec.Region = region

return nil

})

@@ -1170,6 +1175,9 @@ func SetRecordMove(ctx context.Context, s release.Store, acct, app, host string,

// flight — the caller aborts first or waits.

func MoveRequest(to string, now time.Time) func(*Record) error {

return func(rec *Record) error {

+ if err := seedMoveRefusal(rec); err != nil {

+ return err

+ }

if rec.ActiveRun.Open() {

return &MoveRunOpenError{Run: *rec.ActiveRun}

}

@@ -1204,6 +1212,9 @@ type MoveStart struct {

// without an id has no journal to run.

func MoveStartTransition(p MoveStart) func(*Record) error {

return func(rec *Record) error {

+ if err := seedMoveRefusal(rec); err != nil {

+ return err

+ }

if rec.ActiveRun.Open() {

return &MoveRunOpenError{Run: *rec.ActiveRun}

}

diff --git a/internal/provision/seed.go b/internal/provision/seed.go

index 5378a0733..854c885ac 100644

--- a/internal/provision/seed.go

+++ b/internal/provision/seed.go

@@ -275,3 +275,35 @@ func SeedColumn(rec Record) string {

return seeds.RowSeed{ID: rec.Seed.ID, Kind: rec.Seed.Kind, SHA256: rec.Seed.SHA256,

Bytes: rec.Seed.Bytes, State: rec.Seed.State}.Encode()

}

+// SeedMoveSentence is a move refusal's sentence (spec §5.8), "" for any

+// block but the two.

+func SeedMoveSentence(block string) string {

+ switch block {

+ case SeedMoveUnplaced:

+ return "wake the instance once before moving it"

+ case SeedMoveUncommitted:

+ return "commit or abort the import first"

+ }

+ return ""

+// SeedMoveError is a move the record's seed refuses: an ErrMoveConflict

+// whose sentence is SeedMoveSentence(Block). It is typed so a door that

+// meets it under a CAS (carlos apps place) can answer the sentence alone.

+type SeedMoveError struct{ Block string }

+func (e *SeedMoveError) Error() string {

+ return ErrMoveConflict.Error() + ": " + SeedMoveSentence(e.Block)

+func (e *SeedMoveError) Unwrap() error { return ErrMoveConflict }

+// seedMoveRefusal is the seed rule of every write that moves a record (the

+// two move-start transitions and SetRecordRegion), under the record CAS.

+func seedMoveRefusal(rec *Record) error {

+ if b := rec.Seed.MoveBlock(); b != "" {

+ return &SeedMoveError{Block: b}

+ }

+ return nil

diff --git a/internal/provision/seed_test.go b/internal/provision/seed_test.go

index 3058d3ce1..3a200c3d6 100644

--- a/internal/provision/seed_test.go

+++ b/internal/provision/seed_test.go

@@ -78,8 +78,9 @@ func TestOrdinaryWritersCannotChangeTheSeed(t *testing.T) {

if !errors.Is(err, ErrSeedProtected) {

t.Fatalf("dropping the seed through updateRecord: err = %v, want ErrSeedProtected", err)

}

- // An ordinary write that leaves the seed alone carries it through.

- if err := SetRecordRegion(ctx, s, rec.Account, rec.App, rec.Host, "ie"); err != nil {

+ // An ordinary write that leaves the seed alone carries it through. It

+ // keeps the region: re-homing an unplaced seed is refused (Plan 8 Task 3).

+ if err := SetRecordRegion(ctx, s, rec.Account, rec.App, rec.Host, rec.Region); err != nil {

t.Fatal(err)

}

got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)

@@ -491,3 +492,98 @@ func TestSeedAbortStaleOnlyFromWhatWasRead(t *testing.T) {

t.Fatalf("placed --from: err = %v, want ErrSeedConflict", err)

}

+func TestSeedMoveSentence(t *testing.T) {

+ for block, want := range map[string]string{

+ SeedMoveUnplaced: "wake the instance once before moving it",

+ SeedMoveUncommitted: "commit or abort the import first",

+ "": "",

+ "other": "",

+ } {

+ if got := SeedMoveSentence(block); got != want {

+ t.Errorf("SeedMoveSentence(%q) = %q, want %q", block, got, want)

+ }

+// validMoveStart is a start MoveStartTransition accepts on its own: the

+// seed refusal is then the only reason it can refuse (preflight m4).

+func validMoveStart(from string) MoveStart {

+ return MoveStart{ID: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Incarnation: strings.Repeat("a", 64), From: from, To: "uk", Cut: "auto", By: "op", Now: time.Now()}

+func TestMoveTransitionsRefuseAnUnplacedSeed(t *testing.T) {

+ ctx := context.Background()

+ for _, c := range []struct {

+ kind, state, block, want string

+ }{

+ {seeds.KindReplica, seeds.StateReady, SeedMoveUnplaced, "wake the instance once before moving it"},

+ {seeds.KindReplica, seeds.StateAborted, SeedMoveUnplaced, "wake the instance once before moving it"},

+ {seeds.KindImport, seeds.StatePlaced, SeedMoveUncommitted, "commit or abort the import first"},

+ {seeds.KindImport, seeds.StateAwaitingUpload, SeedMoveUncommitted, "commit or abort the import first"},

+ } {

+ s := release.NewDirStore(t.TempDir())

+ rec := seedTestRecord(t, s, testSeed(t, c.kind, c.state))

+ for name, mut := range map[string]func(*Record) error{

+ "request": MoveRequest("uk", time.Now()),

+ "start": MoveStartTransition(validMoveStart(rec.Region)),

+ } {

+ _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut)

+ var refusal *SeedMoveError

+ if !errors.Is(err, ErrMoveConflict) || !strings.Contains(err.Error(), c.want) || !errors.As(err, &refusal) || refusal.Block != c.block {

+ t.Fatalf("%s %s %s: err = %v", c.kind, c.state, name, err)

+ }

+ got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)

+ if err != nil || got.Move != nil {

+ t.Fatalf("%s %s: a refused move left a journal: %+v, %v", c.kind, c.state, got, err)

+ }

+ for _, c := range []struct{ kind, state string }{

+ {seeds.KindReplica, seeds.StatePlaced},

+ {seeds.KindImport, seeds.StateCommitted},

+ } {

+ for name, mut := range map[string]func(*Record) error{

+ "request": MoveRequest("uk", time.Now()),

+ "start": MoveStartTransition(validMoveStart("")),

+ } {

+ s := release.NewDirStore(t.TempDir())

+ rec := seedTestRecord(t, s, testSeed(t, c.kind, c.state))

+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil {

+ t.Fatalf("%s %s %s: %v", c.kind, c.state, name, err)

+ }

+// SetRecordRegion is carlos apps place's write: it re-homes a record, so it

+// refuses a seed that may not move, under its CAS (preflight M8(a)). A

+// write that leaves the region as it is moves nothing and is let through.

+func TestSetRecordRegionRefusesAnUnplacedSeed(t *testing.T) {

+ ctx := context.Background()

+ for _, c := range []struct{ kind, state, want string }{

+ {seeds.KindReplica, seeds.StateReady, "wake the instance once before moving it"},

+ {seeds.KindImport, seeds.StateSwapped, "commit or abort the import first"},

+ } {

+ s := release.NewDirStore(t.TempDir())

+ rec := seedTestRecord(t, s, testSeed(t, c.kind, c.state))

+ err := SetRecordRegion(ctx, s, rec.Account, rec.App, rec.Host, "fleet/pi")

+ if !errors.Is(err, ErrMoveConflict) || !strings.Contains(err.Error(), c.want) {

+ t.Fatalf("%s %s: err = %v", c.kind, c.state, err)

+ }

+ if got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host); err != nil || got.Region != rec.Region {

+ t.Fatalf("%s %s: a refused re-home wrote: %+v, %v", c.kind, c.state, got, err)

+ }

+ if err := SetRecordRegion(ctx, s, rec.Account, rec.App, rec.Host, rec.Region); err != nil {

+ t.Fatalf("%s %s: a write that keeps the region: %v", c.kind, c.state, err)

+ }

+ s := release.NewDirStore(t.TempDir())

+ rec := seedTestRecord(t, s, testSeed(t, seeds.KindReplica, seeds.StatePlaced))

+ if err := SetRecordRegion(ctx, s, rec.Account, rec.App, rec.Host, "fleet/pi"); err != nil {

+ t.Fatalf("a placed seed: %v", err)

+ }

+ if got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host); err != nil || got.Region != "fleet/pi" {

+ t.Fatalf("a placed seed after the re-home: %+v, %v", got, err)

+ }