carlos / platform Public

provision: the import swap, its predicates and its refusals (Plan 7 Task 3)

Reconcile swaps an import's addr row for a nested, gated exec row:
the journal first (a crash before the record CAS resumes as the CAS
only), then SD-N91 to SD-N98, then SwapForImport and the CAS to
swapped. A record the gate cannot protect, a transient layout or
nested-probe error, an unreadable assignment, the config gate and a
held row all defer. Before committed, no box mints an import's row.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude agent, operated by paul@keymail.dev bf34341bfaa93c84005db92f69e3c4a36cc138af parent c9754d6
6 files changed, +867 −5
  • approved_seeded_copy_test.go +3 −2
  • internal/provision/reconcile.go +7 −0
  • internal/provision/seedreconcile.go +23 −2
  • internal/provision/seedreconcile_test.go +0 −1
  • internal/provision/seedswap.go +240 −0
  • internal/provision/seedswap_test.go +594 −0
diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go
index 499dcfd02..5ae45f374 100644
--- a/approved_seeded_copy_test.go
+++ b/approved_seeded_copy_test.go
@@ -37,7 +37,7 @@ var seededCopyIDs = func() []string {
// Task 5 SD-N20 to SD-N26, SD-N30 to SD-N39 and SD-N41; Plan 5 Task 6 SD-N40, SD-N42 to SD-N47, SD-N83 to SD-N88 and SD-N114;
// Plan 6 Task 1 SD-N48 to SD-N50, SD-N62 and SD-N67; Plan 6 Task 2 SD-N51 to SD-N53, SD-N59, SD-N61 and SD-N63;
// Plan 6 Task 3 SD-N54, SD-N55, SD-N60, SD-N64, SD-N65 and SD-N115; Plan 6 Task 5 SD-N56 to SD-N58 and SD-N66;
-// Plan 6 Task 6 SD-N68 to SD-N82, SD-N89 and SD-N90.
+// Plan 6 Task 6 SD-N68 to SD-N82, SD-N89 and SD-N90; Plan 7 Task 3 SD-N91 to SD-N98.
var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19", "SD-N27", "SD-N28", "SD-N29",
"SD-N20", "SD-N21", "SD-N22", "SD-N23", "SD-N24", "SD-N25", "SD-N26", "SD-N30", "SD-N31", "SD-N32", "SD-N33", "SD-N34", "SD-N35", "SD-N36", "SD-N37", "SD-N38", "SD-N39", "SD-N41",
"SD-N40", "SD-N42", "SD-N43", "SD-N44", "SD-N45", "SD-N46", "SD-N47", "SD-N114", "SD-N83", "SD-N84", "SD-N85", "SD-N86", "SD-N87", "SD-N88",
@@ -46,7 +46,8 @@ var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N
"SD-N54", "SD-N55", "SD-N60", "SD-N64", "SD-N65", "SD-N115",
"SD-N56", "SD-N57", "SD-N58", "SD-N66",
"SD-N68", "SD-N69", "SD-N70", "SD-N71", "SD-N72", "SD-N73", "SD-N74", "SD-N75", "SD-N76", "SD-N77", "SD-N78", "SD-N79",
- "SD-N80", "SD-N81", "SD-N82", "SD-N89", "SD-N90"}
+ "SD-N80", "SD-N81", "SD-N82", "SD-N89", "SD-N90",
+ "SD-N91", "SD-N92", "SD-N93", "SD-N94", "SD-N95", "SD-N96", "SD-N97", "SD-N98"}
func mustParseSeededRecord(t *testing.T) []copyRow {
t.Helper()
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index d8c5428a4..cd0a4ccf8 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -1102,6 +1102,13 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
switch seedAction(rec, ok, cfg, mintLayout) {
case seedSkip:
continue
+ case seedSwap:
+ var row *registry.Route
+ if ok {
+ row = &existing
+ }
+ swapImport(ctx, s, reg, cfg, rec, row, mintLayout)
+ continue
case seedFinishAbort:
// A --from seed this box minted is still running: its row
// goes first, and a later pass that finds no row rolls the
diff --git a/internal/provision/seedreconcile.go b/internal/provision/seedreconcile.go
index ad41d2f2e..abdb6688d 100644
--- a/internal/provision/seedreconcile.go
+++ b/internal/provision/seedreconcile.go
@@ -18,6 +18,15 @@ import (
type SeedHooks struct {
// Data is this region's data bucket (seeds/ lives there).
Data release.Store
+ // LegacyOwners reads the operator assignment of legacy route names
+ // (spec §6.4): the swap's only authority over a row filed under the
+ // operator account. Nil, or an error, defers such a swap.
+ LegacyOwners func(ctx context.Context) (*seeds.LegacyOwners, error)
+ // NestedOK is the box's live nested probe, asked after the layout
+ // intent says nested (spec §5.4 "Layout"). An error defers the swap to
+ // the next pass; it never refuses (only the intent does). Nil: the
+ // intent alone decides.
+ NestedOK func() error
}
type seedVerdict int
@@ -26,6 +35,7 @@ const (
seedOrdinary seedVerdict = iota
seedSkip
seedFinishAbort
+ seedSwap
)
// seedAction is §3.2's dispatch, run before any other per-record handling.
@@ -46,7 +56,7 @@ func seedAction(rec Record, hasRow bool, cfg ReconcileConfig, nested func() (boo
return seedSkip
case seeds.StateReady:
if src.Import() {
- return seedSkip // the swap is Plan 7's
+ return seedSwap // seedswap.go
}
if hasRow {
return seedOrdinary
@@ -67,7 +77,18 @@ func seedAction(rec Record, hasRow bool, cfg ReconcileConfig, nested func() (boo
return seedSkip
}
return seedOrdinary
- case seeds.StateSwapped, seeds.StatePlaced, seeds.StateCommitted:
+ case seeds.StateSwapped, seeds.StatePlaced:
+ // Before committed, an import's row is made only by the swap,
+ // gated (spec §5.4, §8). A box with no row, or the swapping box
+ // after its gated row was removed, must not mint one: routeFor
+ // never sets the gate, so ordinary traffic would reach an
+ // unverified import.
+ if src.Import() && !hasRow {
+ log.Printf("provision: %s: not minting import %s: it is %s but not committed, and only its swap makes its row", rec.Host, src.ID, src.State)
+ return seedSkip
+ }
+ return seedOrdinary
+ case seeds.StateCommitted:
return seedOrdinary
}
log.Printf("provision: %s: seed %s is in an unknown state %q; leaving it alone", rec.Host, src.ID, src.State)
diff --git a/internal/provision/seedreconcile_test.go b/internal/provision/seedreconcile_test.go
index 56a2fafcf..a95173f35 100644
--- a/internal/provision/seedreconcile_test.go
+++ b/internal/provision/seedreconcile_test.go
@@ -89,7 +89,6 @@ func TestReconcileSkipsSeedsThatAreNotReady(t *testing.T) {
{seeds.KindImport, seeds.StateAwaitingUpload},
{seeds.KindImport, seeds.StateUploaded},
{seeds.KindImport, seeds.StateRefused},
- {seeds.KindImport, seeds.StateReady}, // Plan 7 swaps; here, nothing
{seeds.KindReplica, seeds.StateRolledBack},
} {
s, sqid := setup(t)
diff --git a/internal/provision/seedswap.go b/internal/provision/seedswap.go
new file mode 100644
index 000000000..6019780fd
--- /dev/null
+++ b/internal/provision/seedswap.go
@@ -0,0 +1,240 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// importShape is the legacy row's shape check (spec §5.4, codex 10, R1-10):
+// "" when the row is an addr-backed instance or service this import may
+// take over, else what makes it not one.
+func importShape(rt registry.Route) string {
+ switch {
+ case rt.Kind != "instance" && rt.Kind != "service":
+ return "kind " + rt.Kind
+ case rt.Socket != "":
+ return "socket"
+ case rt.Addr == "" && rt.Addrs == "":
+ return "no address"
+ case rt.Provisioned:
+ return "provisioned"
+ case rt.Backing != "":
+ return "backing"
+ case rt.Tunnel != "":
+ return "tunnel"
+ case rt.Caps != "":
+ return "caps"
+ case rt.Platform:
+ return "platform"
+ case rt.Canary:
+ return "canary"
+ }
+ return ""
+}
+
+// importAddr is the address the row serves, as the edge reads it
+// (registry.Route.AddrList): addrs when set, else addr. A row whose addr
+// is Replaces but whose addrs names something else serves the addrs, so
+// it is not the route the import recorded.
+func importAddr(rt registry.Route) string {
+ return strings.Join(rt.AddrList(), ",")
+}
+
+// importRefusal is §5.4's predicates in order: "" to swap, else the
+// refusal reason the record keeps. owners is consulted only for a row
+// filed under the operator account: release.RouteAccount folds a legacy
+// row's "" (and any account that is not a sqid) into release.OpsAccount,
+// so such a row is the operator's too (preflight M5).
+func importRefusal(rec Record, existing *registry.Route, cfg ReconcileConfig, owners *seeds.LegacyOwners, nested bool) string {
+ switch {
+ case cfg.ClaimEnabled:
+ return "claim control is on for this box"
+ case !nested:
+ return "this box does not run instances nested"
+ case existing == nil:
+ return fmt.Sprintf("there is no route at %s to replace", rec.Host)
+ }
+ if what := importShape(*existing); what != "" {
+ return fmt.Sprintf("the route is not an address route this import can take over (%s)", what)
+ }
+ if addr := importAddr(*existing); addr != rec.Seed.Replaces {
+ return fmt.Sprintf("the route's address is %s, not %s", addr, rec.Seed.Replaces)
+ }
+ if existing.App != rec.App {
+ return fmt.Sprintf("the route belongs to app %s", existing.App)
+ }
+ switch release.RouteAccount(*existing) {
+ case rec.Account:
+ return ""
+ case release.OpsAccount:
+ if o, ok := owners.Lookup(existing.App); ok && o.Account == rec.Account && o.App == rec.App {
+ return ""
+ }
+ return fmt.Sprintf("the route is filed under the operator account, and no assignment maps %s to this app", existing.App)
+ }
+ return "the route is filed under another account"
+}
+
+// operatorFiled reports a row the swap may take only through the operator
+// assignment: filed under the operator account, for a record that is not
+// the operator's own.
+func operatorFiled(rec Record, rt *registry.Route) bool {
+ if rt == nil {
+ return false
+ }
+ acct := release.RouteAccount(*rt)
+ return acct != rec.Account && acct == release.OpsAccount
+}
+
+var verifyHashShape = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// importUngateable is why the swap must not run for this record at all
+// (preflight M3), or "": the gate needs a hash (an empty one would swap in
+// an ungated row), the host and seed id name the snapshot and must pass
+// seeds.Check, and the new row must be one the activator manages, nested
+// and hibernating, since only that row places the snapshot before it
+// serves. These are not refusals: the record stays ready, logged, until
+// it is aborted.
+func importUngateable(rec Record, next registry.Route, dataDir string) string {
+ if err := seeds.Check(rec.Host, rec.Seed.ID); err != nil {
+ return fmt.Sprintf("its host or seed id is malformed (%v)", err)
+ }
+ switch {
+ case !verifyHashShape.MatchString(rec.Seed.VerifyHash):
+ return "its verify hash is not a sha256"
+ case !rec.DerivesPaths() || !next.Hibernate || !NestedPaths(dataDir, next):
+ return "its record does not describe a nested, hibernating instance"
+ }
+ return ""
+}
+
+// importRow is the exec row the swap installs (spec §5.4): the record's
+// route at the nested paths, explicitly, never NestedPaths(existing),
+// which reads the addr row as flat (codex 12); no seed command; filed
+// under the record's account; gated by the verify hash. Its seed column
+// already says swapped, the state the record's CAS moves it to.
+func importRow(rec Record, dataDir string) registry.Route {
+ swapped := *rec.Seed
+ swapped.State = seeds.StateSwapped
+ r := rec
+ r.Seed = &swapped
+ next := routeFor(r, dataDir, "", true)
+ next.ImportGate = rec.Seed.VerifyHash
+ return next
+}
+
+// swapImport is the swap (spec §5.4): the journal first, then the
+// predicates, then one registry transaction, then the record's CAS to
+// swapped. Transient failures defer to the next pass and write nothing; a
+// failed predicate is a durable refusal.
+func swapImport(ctx context.Context, s release.Store, reg *registry.Registry, cfg ReconcileConfig, rec Record, existing *registry.Route, nested func() (bool, error)) {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would consider swapping %s for import %s", rec.Host, rec.Seed.ID)
+ return
+ }
+ // The journal comes first (codex R2-3): a row there means this box has
+ // already swapped, so the legacy-row predicates never judge the row it
+ // made. This seed's own row at swapped is a crash between the swap's
+ // transaction and the record's CAS: only the CAS is redone.
+ j, err := reg.ImportJournal(rec.Host)
+ if err != nil {
+ log.Printf("provision: %s: import deferred: the import journal could not be read: %v", rec.Host, err)
+ return
+ }
+ if j != nil {
+ if j.SeedID != rec.Seed.ID || j.State != registry.JournalSwapped {
+ log.Printf("provision: %s: import %s left alone: the import journal holds seed %s at %s", rec.Host, rec.Seed.ID, j.SeedID, j.State)
+ return
+ }
+ markSwapped(ctx, s, rec)
+ return
+ }
+
+ next := importRow(rec, cfg.DataDir)
+ if why := importUngateable(rec, next, cfg.DataDir); why != "" {
+ log.Printf("provision: %s: not swapping for import %s: %s", rec.Host, rec.Seed.ID, why)
+ return
+ }
+ isNested := false
+ if !cfg.ClaimEnabled {
+ ok, err := nested()
+ if err != nil {
+ log.Printf("provision: %s: import deferred: the box's layout intent could not be read: %v", rec.Host, err)
+ return
+ }
+ // Only the intent refuses (SD-N97). The live probe can fail for a
+ // moment, and a refusal is terminal: an error waits for the next
+ // pass (preflight M4).
+ if ok && cfg.Seeds.NestedOK != nil {
+ if err := cfg.Seeds.NestedOK(); err != nil {
+ log.Printf("provision: %s: import deferred: the box's nested probe did not answer: %v", rec.Host, err)
+ return
+ }
+ }
+ isNested = ok
+ }
+ if existing != nil && existing.HoldOp != "" {
+ log.Printf("provision: %s: import deferred: the route is held by operation %s", rec.Host, existing.HoldOp)
+ return
+ }
+ var owners *seeds.LegacyOwners
+ if operatorFiled(rec, existing) {
+ if cfg.Seeds.LegacyOwners == nil {
+ log.Printf("provision: %s: import deferred: this box cannot read the legacy route assignments", rec.Host)
+ return
+ }
+ o, err := cfg.Seeds.LegacyOwners(ctx)
+ if err != nil {
+ log.Printf("provision: %s: import deferred: the legacy route assignments could not be read: %v", rec.Host, err)
+ return
+ }
+ owners = o
+ }
+ if reason := importRefusal(rec, existing, cfg, owners, isNested); reason != "" {
+ if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedRefuse(rec.Seed.ID, reason)); err != nil {
+ log.Printf("provision: %s: record the import's refusal (%s): %v", rec.Host, reason, err)
+ return
+ }
+ log.Printf("provision: %s: import %s refused: %s", rec.Host, rec.Seed.ID, reason)
+ return
+ }
+ if cfg.ConfigGate != nil {
+ if err := cfg.ConfigGate(ctx, next); err != nil {
+ return // the gate logs its own refusal; retried every pass
+ }
+ }
+ now := time.Now
+ if cfg.Now != nil {
+ now = cfg.Now
+ }
+ journal := registry.ImportJournal{Host: rec.Host, SeedID: rec.Seed.ID, Original: *existing,
+ RecordKey: RecordKey(rec.Account, rec.App, rec.Host), At: now().Unix()}
+ if err := reg.SwapForImport(journal, next); err != nil {
+ log.Printf("provision: %s: import swap deferred: %v", rec.Host, err)
+ return
+ }
+ log.Printf("provision: %s: swapped for import %s (the original route is journaled)", rec.Host, rec.Seed.ID)
+ markSwapped(ctx, s, rec)
+}
+
+// markSwapped is the swap's record CAS, ready to swapped. It runs only
+// after the registry transaction committed; a failure leaves the journal
+// at swapped and the record at ready, which the next pass resumes.
+func markSwapped(ctx context.Context, s release.Store, rec Record) {
+ if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedSwapped(rec.Seed.ID)); err != nil {
+ if errors.Is(err, ErrSeedConflict) {
+ log.Printf("provision: %s: import %s swapped, but the record moved on: %v", rec.Host, rec.Seed.ID, err)
+ return
+ }
+ log.Printf("provision: %s: import %s swapped, record CAS deferred to the next pass: %v", rec.Host, rec.Seed.ID, err)
+ }
+}
diff --git a/internal/provision/seedswap_test.go b/internal/provision/seedswap_test.go
new file mode 100644
index 000000000..c8dc81177
--- /dev/null
+++ b/internal/provision/seedswap_test.go
@@ -0,0 +1,594 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// shopSetup is the import tests' neutral fixture: an account manifest
+// (id acctID) and its app "shop" (domain shop.example.com, the shared
+// pool, no seed command).
+func shopSetup(t *testing.T) (*release.DirStore, string) {
+ t.Helper()
+ s := dirStore(t)
+ sqid := writeAccountManifest(t, s, acctID, "Shop")
+ if err := PutManifest(ctx, s, sqid, "shop", Manifest{Schema: 1, App: "shop", Domain: "shop.example.com"}); err != nil {
+ t.Fatal(err)
+ }
+ return s, sqid
+}
+
+func shopRecord(sqid string) Record {
+ return Record{Schema: 1, Host: "a.shop.example.com", Slug: "a", App: "shop",
+ Account: sqid, Region: "ie", Channel: "stable", CreatedAt: time.Now()}
+}
+
+func newImport(sqid string) Record {
+ rec := shopRecord(sqid)
+ id, _ := seeds.NewID()
+ rec.Seed = &SeedSource{ID: id, Kind: seeds.KindImport, State: seeds.StateReady, SHA256: strings.Repeat("a", 64), Bytes: 10,
+ Replaces: "127.0.0.1:8401", VerifyHash: strings.Repeat("b", 64), By: "t", At: time.Now()}
+ return rec
+}
+
+func importRecord(t *testing.T, s release.Store, sqid string) Record {
+ t.Helper()
+ rec := newImport(sqid)
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ return rec
+}
+
+// addrRow is the legacy row an import replaces: an addr route to the
+// app's own process, not minted from any record.
+func addrRow(rec Record, account string) registry.Route {
+ return registry.Route{Host: rec.Host, Kind: "service", Addr: "127.0.0.1:8401", App: rec.App, Account: account,
+ Region: "ie", Channel: "stable", Status: "ready", Unit: "shop@a.service"}
+}
+
+func importCfg(t *testing.T, owners map[string]seeds.LegacyOwner) ReconcileConfig {
+ t.Helper()
+ cfg := nestedSeedCfg(dirStore(t))
+ cfg.Pools = nil
+ cfg.Seeds.LegacyOwners = func(context.Context) (*seeds.LegacyOwners, error) {
+ return &seeds.LegacyOwners{V: 1, Names: owners}, nil
+ }
+ return cfg
+}
+
+func seedState(t *testing.T, s release.Store, rec Record) *SeedSource {
+ t.Helper()
+ r, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return r.Seed
+}
+
+// wantUntouched asserts a deferred swap: the record still ready with no
+// reason, the stored row exactly row, and no journal.
+func wantUntouched(t *testing.T, s release.Store, reg *registry.Registry, rec Record, row *registry.Route) {
+ t.Helper()
+ if got := seedState(t, s, rec); got.State != seeds.StateReady || got.Reason != "" {
+ t.Fatalf("seed = %s %q, want ready (deferred, not refused)", got.State, got.Reason)
+ }
+ got, err := reg.Get(rec.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if row == nil {
+ if got != nil {
+ t.Fatalf("a row appeared: %+v", got)
+ }
+ } else if got == nil || !reflect.DeepEqual(*got, *row) {
+ t.Fatalf("row = %+v, want it unchanged %+v", got, row)
+ }
+ if j, _ := reg.ImportJournal(rec.Host); j != nil {
+ t.Fatalf("a deferred import left a journal row: %+v", j)
+ }
+}
+
+func TestSwapReplacesTheAddrRowAndJournalsIt(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ original := mustGet(t, reg, rec.Host)
+ cfg := importCfg(t, nil)
+ at := time.Unix(1_800_000_000, 0)
+ cfg.Now = func() time.Time { return at }
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ got := mustGet(t, reg, rec.Host)
+ if got.Kind != "instance" || got.Socket != NestedSocket(rec.Host) || got.Addr != "" || got.Addrs != "" ||
+ got.ImportGate != rec.Seed.VerifyHash || got.DBPath != NestedDBPath("/data", rec.Host) || got.Account != sqid ||
+ !got.Provisioned || got.Seed != "" || !got.Hibernate {
+ t.Fatalf("swapped row = %+v", got)
+ }
+ if col, _ := seeds.ParseRowSeed(got.SeedSource); col == nil || col.ID != rec.Seed.ID || col.State != seeds.StateSwapped {
+ t.Fatalf("swapped row's seed column = %+v, want this seed at swapped", col)
+ }
+ j, err := reg.ImportJournal(rec.Host)
+ if err != nil || j == nil || j.SeedID != rec.Seed.ID || j.State != registry.JournalSwapped || !reflect.DeepEqual(j.Original, *original) ||
+ j.RecordKey != RecordKey(rec.Account, rec.App, rec.Host) || j.At != at.Unix() {
+ t.Fatalf("journal = %+v, %v", j, err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateSwapped {
+ t.Fatalf("record = %s", st.State)
+ }
+
+ // The next pass converges the row as an ordinary owned row and leaves
+ // the gate and the journal alone.
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ again := mustGet(t, reg, rec.Host)
+ if again.ImportGate != rec.Seed.VerifyHash || again.Socket != got.Socket || again.DBPath != got.DBPath || again.Addr != "" {
+ t.Fatalf("row after the next pass = %+v", again)
+ }
+ if j2, _ := reg.ImportJournal(rec.Host); j2 == nil || !reflect.DeepEqual(*j2, *j) {
+ t.Fatalf("journal after the next pass = %+v", j2)
+ }
+}
+
+// A row backed by an addrs set of exactly Replaces is the same address.
+func TestSwapAcceptsAnAddrsRowOfExactlyReplaces(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ row := addrRow(rec, sqid)
+ row.Addr, row.Addrs = "", "127.0.0.1:8401"
+ reg := regWith(t, row)
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, rec.Host); got.ImportGate != rec.Seed.VerifyHash || got.Kind != "instance" {
+ t.Fatalf("row = %+v", got)
+ }
+}
+
+func TestSwapRefusesEachCondition(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ row func(Record, string) *registry.Route
+ owners map[string]seeds.LegacyOwner
+ cfg func(*ReconcileConfig)
+ want string
+ }{
+ {"no row", func(Record, string) *registry.Route { return nil }, nil, nil, "there is no route at a.shop.example.com to replace"},
+ {"addr changed", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Addr = "127.0.0.1:9"; return &x }, nil, nil,
+ "the route's address is 127.0.0.1:9, not 127.0.0.1:8401"},
+ {"addrs wider than replaces", func(r Record, a string) *registry.Route {
+ x := addrRow(r, a)
+ x.Addr, x.Addrs = "", "127.0.0.1:8401,127.0.0.1:8402"
+ return &x
+ }, nil, nil, "the route's address is 127.0.0.1:8401,127.0.0.1:8402, not 127.0.0.1:8401"},
+ {"addrs behind a matching addr", func(r Record, a string) *registry.Route {
+ x := addrRow(r, a)
+ x.Addrs = "127.0.0.1:9"
+ return &x
+ }, nil, nil, "the route's address is 127.0.0.1:9, not 127.0.0.1:8401"},
+ {"another app", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.App = "other"; return &x }, nil, nil,
+ "the route belongs to app other"},
+ {"a third account", func(r Record, _ string) *registry.Route { x := addrRow(r, "zzz"); return &x }, nil, nil,
+ "the route is filed under another account"},
+ {"operator row, no assignment", func(r Record, _ string) *registry.Route { x := addrRow(r, release.OpsAccount); return &x }, nil, nil,
+ "the route is filed under the operator account, and no assignment maps shop to this app"},
+ {"legacy operator row, no assignment", func(r Record, _ string) *registry.Route { x := addrRow(r, ""); return &x }, nil, nil,
+ "no assignment maps shop to this app"},
+ {"operator row, assigned to another account", func(r Record, _ string) *registry.Route { x := addrRow(r, release.OpsAccount); return &x },
+ map[string]seeds.LegacyOwner{"shop": {Account: "other", App: "shop"}}, nil, "no assignment maps shop to this app"},
+ {"operator row, assigned to another app", func(r Record, _ string) *registry.Route { x := addrRow(r, release.OpsAccount); return &x },
+ map[string]seeds.LegacyOwner{"shop": {Account: "SELF", App: "store"}}, nil, "no assignment maps shop to this app"},
+ {"socket backed", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Socket = "/run/x.sock"; return &x }, nil, nil, "(socket)"},
+ {"no address", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Addr = ""; return &x }, nil, nil, "(no address)"},
+ {"provisioned", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Provisioned = true; return &x }, nil, nil, "(provisioned)"},
+ {"static", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Kind = "static"; return &x }, nil, nil, "(kind static)"},
+ {"backing", func(r Record, a string) *registry.Route {
+ x := addrRow(r, a)
+ x.Backing = registry.BackingUnit
+ return &x
+ }, nil, nil, "(backing)"},
+ {"tunnel", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Tunnel = "t"; return &x }, nil, nil, "(tunnel)"},
+ {"caps", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Caps = "host-role"; return &x }, nil, nil, "(caps)"},
+ {"platform", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Platform = true; return &x }, nil, nil, "(platform)"},
+ {"canary", func(r Record, a string) *registry.Route { x := addrRow(r, a); x.Canary = true; return &x }, nil, nil, "(canary)"},
+ {"flat", func(r Record, a string) *registry.Route { x := addrRow(r, a); return &x }, nil, func(c *ReconcileConfig) { c.Layout = nil },
+ "this box does not run instances nested"},
+ {"flat intent", func(r Record, a string) *registry.Route { x := addrRow(r, a); return &x }, nil, func(c *ReconcileConfig) {
+ c.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+ }, "this box does not run instances nested"},
+ {"claimed", func(r Record, a string) *registry.Route { x := addrRow(r, a); return &x }, nil, func(c *ReconcileConfig) { c.ClaimEnabled = true },
+ "claim control is on for this box"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ var reg *registry.Registry
+ var before *registry.Route
+ if row := c.row(rec, sqid); row != nil {
+ reg = regWith(t, *row)
+ before = mustGet(t, reg, rec.Host)
+ } else {
+ reg = regWith(t)
+ }
+ owners := c.owners
+ for k, v := range owners { // "SELF" is the record's own account
+ if v.Account == "SELF" {
+ owners[k] = seeds.LegacyOwner{Account: sqid, App: v.App}
+ }
+ }
+ cfg := importCfg(t, owners)
+ if c.cfg != nil {
+ c.cfg(&cfg)
+ }
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateRefused || !strings.Contains(st.Reason, c.want) {
+ t.Fatalf("seed = %s %q, want refused with %q", st.State, st.Reason, c.want)
+ }
+ if j, _ := reg.ImportJournal(rec.Host); j != nil {
+ t.Fatalf("a refused import left a journal row: %+v", j)
+ }
+ after, _ := reg.Get(rec.Host)
+ if (before == nil) != (after == nil) || (before != nil && !reflect.DeepEqual(*before, *after)) {
+ t.Fatalf("a refused import changed the row: %+v -> %+v", before, after)
+ }
+ // A refused import is skipped from then on.
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateRefused {
+ t.Fatalf("seed after the next pass = %s", st.State)
+ }
+ })
+ }
+}
+
+// M5: an operator-filed row ("bab", or a legacy row with no account) is
+// accepted only through the assignment, and the new row is filed under
+// the record's account.
+func TestSwapAcceptsAnAssignedOperatorRow(t *testing.T) {
+ for _, acct := range []string{release.OpsAccount, ""} {
+ t.Run("account "+acct, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, acct))
+ if err := Reconcile(ctx, s, reg, importCfg(t, map[string]seeds.LegacyOwner{"shop": {Account: sqid, App: "shop"}})); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, rec.Host); got.Account != sqid || got.Kind != "instance" || got.ImportGate == "" {
+ t.Fatalf("row = %+v; want it filed under the record's account, gated", got)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateSwapped {
+ t.Fatalf("record = %s", st.State)
+ }
+ })
+ }
+}
+
+// The assignment is an authority check: an unreadable file or an unwired
+// reader defers the swap, never refuses it and never accepts the row.
+func TestSwapDefersWhenTheAssignmentCannotBeRead(t *testing.T) {
+ for name, hook := range map[string]func(context.Context) (*seeds.LegacyOwners, error){
+ "read error": func(context.Context) (*seeds.LegacyOwners, error) { return nil, errors.New("store down") },
+ "unwired": nil,
+ } {
+ t.Run(name, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, release.OpsAccount))
+ row := mustGet(t, reg, rec.Host)
+ cfg := importCfg(t, nil)
+ cfg.Seeds.LegacyOwners = hook
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ wantUntouched(t, s, reg, rec, row)
+ })
+ }
+}
+
+// M3: the swap is skipped, the record left as it is, unless the verify
+// hash is a hash, the host and seed id pass seeds.Check, and the new row
+// is nested and hibernating.
+func TestSwapSkipsARecordItCannotGate(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ edit func(*Record)
+ }{
+ {"empty verify hash", func(r *Record) { r.Seed.VerifyHash = "" }},
+ {"verify hash not a hash", func(r *Record) { r.Seed.VerifyHash = "closed" }},
+ {"verify hash in capitals", func(r *Record) { r.Seed.VerifyHash = strings.Repeat("B", 64) }},
+ {"bad seed id", func(r *Record) { r.Seed.ID = "sd_../../x" }},
+ {"bad host", func(r *Record) { r.Host = "A.shop.example.com/.." }},
+ {"does not hibernate", func(r *Record) { off := false; r.Hibernate = &off }},
+ {"not an instance", func(r *Record) { r.Kind = "service" }},
+ {"proxies to an address", func(r *Record) { r.Addr = "127.0.0.1:9000" }},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ stored := importRecord(t, s, sqid)
+ rec := stored
+ seed := *stored.Seed
+ rec.Seed = &seed
+ c.edit(&rec)
+ reg := regWith(t, addrRow(stored, sqid))
+ row := mustGet(t, reg, stored.Host)
+ cfg := importCfg(t, nil)
+ got, _ := reg.Get(stored.Host)
+ swapImport(ctx, s, reg, cfg, rec, got, mintLayoutReader(ctx, s, cfg))
+ wantUntouched(t, s, reg, stored, row)
+ })
+ }
+}
+
+// The same rule through Reconcile, for a record shape CreateRecord stores.
+func TestReconcileDoesNotSwapAnImportThatWouldNotHibernate(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := newImport(sqid)
+ off := false
+ rec.Hibernate = &off
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t, addrRow(rec, sqid))
+ row := mustGet(t, reg, rec.Host)
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ wantUntouched(t, s, reg, rec, row)
+}
+
+// M4: a layout read error or a NestedOK error defers to the next pass and
+// writes nothing; only a layout intent that says flat refuses (above).
+func TestSwapDefersATransientLayoutAnswer(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ cfg func(*ReconcileConfig)
+ }{
+ {"layout read error", func(c *ReconcileConfig) {
+ c.Layout = func(context.Context, release.Store) (string, error) { return "", errors.New("store down") }
+ }},
+ {"nested probe error", func(c *ReconcileConfig) {
+ c.Seeds.NestedOK = func() error { return errors.New("ready probe timed out") }
+ }},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ row := mustGet(t, reg, rec.Host)
+ cfg := importCfg(t, nil)
+ c.cfg(&cfg)
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ wantUntouched(t, s, reg, rec, row)
+
+ // The next pass, with the box answering, swaps.
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateSwapped {
+ t.Fatalf("record after the box answered = %s %q", st.State, st.Reason)
+ }
+ })
+ }
+}
+
+func TestSwapRunsWhenTheNestedProbeAnswers(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ cfg := importCfg(t, nil)
+ asked := 0
+ cfg.Seeds.NestedOK = func() error { asked++; return nil }
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if asked != 1 || mustGet(t, reg, rec.Host).ImportGate == "" {
+ t.Fatalf("probe asked %d times, row %+v", asked, mustGet(t, reg, rec.Host))
+ }
+}
+
+// Transient refusals of the write itself defer: the config gate, a row
+// held by a running operation, and a dry run.
+func TestSwapDefersWhatTheWriteCannotTakeYet(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ held bool
+ cfg func(*ReconcileConfig)
+ }{
+ {"config gate", false, func(c *ReconcileConfig) {
+ c.ConfigGate = func(context.Context, registry.Route) error { return errors.New("no key yet") }
+ }},
+ {"held", true, nil},
+ {"dry run", false, func(c *ReconcileConfig) { c.DryRun = true }},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ if c.held {
+ if err := reg.SetHold(rec.Host, "moving", "op_1"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ row := mustGet(t, reg, rec.Host)
+ if c.held && row.HoldOp == "" {
+ t.Fatalf("fixture: the row is not held: %+v", row)
+ }
+ cfg := importCfg(t, nil)
+ if c.cfg != nil {
+ c.cfg(&cfg)
+ }
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ wantUntouched(t, s, reg, rec, row)
+ })
+ }
+}
+
+// A crash between the swap's transaction and the record's CAS: the
+// journal names this seed and the record still says ready. The next pass
+// sees the journal first, never judges the row it made as a legacy row,
+// and only redoes the CAS.
+func TestSwapResumesTheRecordCASAfterACrash(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ original := mustGet(t, reg, rec.Host)
+ next := routeFor(rec, "/data", "", true)
+ next.ImportGate = rec.Seed.VerifyHash
+ if err := reg.SwapForImport(registry.ImportJournal{Host: rec.Host, SeedID: rec.Seed.ID, Original: *original,
+ RecordKey: RecordKey(rec.Account, rec.App, rec.Host), At: 1}, next); err != nil {
+ t.Fatal(err)
+ }
+ swapped := mustGet(t, reg, rec.Host)
+ j, _ := reg.ImportJournal(rec.Host)
+
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateSwapped || st.Reason != "" {
+ t.Fatalf("record = %s %q, want swapped", st.State, st.Reason)
+ }
+ if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *swapped) {
+ t.Fatalf("row changed on the resume: %+v -> %+v", swapped, got)
+ }
+ if j2, _ := reg.ImportJournal(rec.Host); j2 == nil || !reflect.DeepEqual(*j2, *j) {
+ t.Fatalf("journal changed on the resume: %+v -> %+v", j, j2)
+ }
+}
+
+// A journal row this record's seed does not own (another seed's, or one
+// already rolling back) leaves the record alone: no refusal, no swap.
+func TestSwapLeavesAnotherSeedsJournalAlone(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ original := mustGet(t, reg, rec.Host)
+ other, _ := seeds.NewID()
+ next := routeFor(rec, "/data", "", true)
+ next.ImportGate = rec.Seed.VerifyHash
+ if err := reg.SwapForImport(registry.ImportJournal{Host: rec.Host, SeedID: other, Original: *original, At: 1}, next); err != nil {
+ t.Fatal(err)
+ }
+ row := mustGet(t, reg, rec.Host)
+ j, _ := reg.ImportJournal(rec.Host)
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateReady || st.Reason != "" {
+ t.Fatalf("record = %s %q, want ready", st.State, st.Reason)
+ }
+ if got := mustGet(t, reg, rec.Host); !reflect.DeepEqual(*got, *row) {
+ t.Fatalf("row changed: %+v", got)
+ }
+ if j2, _ := reg.ImportJournal(rec.Host); !reflect.DeepEqual(j2, j) {
+ t.Fatalf("journal changed: %+v", j2)
+ }
+}
+
+// M2: before committed, an import's row is made only by the swap. A box
+// with no row never mints one for a swapped or placed import, and neither
+// does the swapping box after its gated row was removed.
+func TestReconcileNeverMintsAnUncommittedImport(t *testing.T) {
+ for _, state := range []string{seeds.StateSwapped, seeds.StatePlaced} {
+ t.Run("no row, "+state, func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := newImport(sqid)
+ rec.Seed.State = state
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ if rt, _ := reg.Get(rec.Host); rt != nil {
+ t.Fatalf("minted %+v", rt)
+ }
+ })
+ }
+ t.Run("after the gated row was removed", func(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := importRecord(t, s, sqid)
+ reg := regWith(t, addrRow(rec, sqid))
+ cfg := importCfg(t, nil)
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if st := seedState(t, s, rec); st.State != seeds.StateSwapped {
+ t.Fatalf("record = %s", st.State)
+ }
+ if err := reg.Delete(rec.Host); err != nil { // carlos remove
+ t.Fatal(err)
+ }
+ for _, mut := range []func(*Record) error{nil, SeedPlace(rec.Seed.ID)} {
+ if mut != nil {
+ if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if rt, _ := reg.Get(rec.Host); rt != nil {
+ t.Fatalf("%s: minted %+v", seedState(t, s, rec).State, rt)
+ }
+ }
+ })
+}
+
+// After commit the import is an ordinary instance: a box with no row
+// mints it (ungated), as any other record.
+func TestReconcileMintsACommittedImport(t *testing.T) {
+ s, sqid := shopSetup(t)
+ rec := newImport(sqid)
+ rec.Seed.State = seeds.StateCommitted
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, importCfg(t, nil)); err != nil {
+ t.Fatal(err)
+ }
+ if got := mustGet(t, reg, rec.Host); got.ImportGate != "" || got.DBPath != NestedDBPath("/data", rec.Host) {
+ t.Fatalf("row = %+v", got)
+ }
+}
+
+func TestImportRefusalOrder(t *testing.T) {
+ rec := newImport("abc")
+ row := addrRow(rec, "zzz")
+ row.App = "other"
+ cfg := ReconcileConfig{ClaimEnabled: true}
+ if got := importRefusal(rec, &row, cfg, nil, false); got != "claim control is on for this box" {
+ t.Fatalf("claims first: %q", got)
+ }
+ if got := importRefusal(rec, &row, ReconcileConfig{}, nil, false); got != "this box does not run instances nested" {
+ t.Fatalf("then layout: %q", got)
+ }
+ if got := importRefusal(rec, nil, ReconcileConfig{}, nil, true); got != "there is no route at a.shop.example.com to replace" {
+ t.Fatalf("then the row: %q", got)
+ }
+ if got := importRefusal(rec, &row, ReconcileConfig{}, nil, true); got != "the route belongs to app other" {
+ t.Fatalf("app before account: %q", got)
+ }
+ ok := addrRow(rec, "abc")
+ if got := importRefusal(rec, &ok, ReconcileConfig{}, nil, true); got != "" {
+ t.Fatalf("the record's own row: %q", got)
+ }
+}