amadan

carlos / platform Public

provision: reconcile dispatches on the seed state first (Plan 4 Task 5)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 7bd503a55f7102a71d5d1210092acf3c8fbcf1ef parent bdcf811
3 files changed, +287 −0
  • internal/provision/reconcile.go +25 −0
  • internal/provision/seedreconcile.go +103 −0
  • internal/provision/seedreconcile_test.go +159 −0

CI passed — run details

diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index 61d394e17..87a9fee57 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -132,6 +132,9 @@ type ReconcileConfig struct {
// notes the record and acts after the pass, by which time a row this
// pass minted is in the registry. Nil: no one is told.
RunSeen func(rec Record)
+ // Seeds is the seeded-instances wiring (seedreconcile.go). Nil: every
+ // seeded record is left alone.
+ Seeds *SeedHooks
// beforeHoldWrite is the tests' pause under the host's hold lock, just
// before holdExisting's conditional write (TestReconcileHoldWrite...).
@@ -440,6 +443,7 @@ func routeFor(rec Record, dataDir, seed string, nested bool) registry.Route {
Replica: rec.Role == "replica",
Gate: rec.Gate,
Provisioned: true, Seed: seed, Status: "ready",
+ SeedSource: SeedColumn(rec),
}
if rec.DerivesPaths() {
rt.Socket = DefaultSocket(rec.Host)
@@ -1089,6 +1093,20 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// clobber, and a later pass converges it once the manifest reads.
seed, seedIsKnown := seedFor(rec.Account, rec.App)
existing, ok := byHost[rec.Host]
+ // A seed is dispatched before anything else (seeded-instances spec
+ // §3.2, codex 13): an aborted one is finished, an unready one is
+ // skipped, and a seeded row never carries the manifest's seed
+ // command.
+ if rec.Seed != nil {
+ switch seedAction(rec, ok, cfg, mintLayout) {
+ case seedSkip:
+ continue
+ case seedFinishAbort:
+ finishAbortedSeed(ctx, s, cfg, rec)
+ continue
+ }
+ seed, seedIsKnown = "", true
+ }
// A move's hold is judged first (moves spec §9.6). A missing row is
// minted held by mintHeld, which has a ready mint's refusals and
// the same claim hook (below) around its write, so the hold and the
@@ -1246,6 +1264,13 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
log.Printf("provision: converge seed on %s: %v", rec.Host, err)
}
}
+ if col := SeedColumn(rec); existing.SeedSource != col {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would converge %s seed source", rec.Host)
+ } else if err := reg.SetSeedSource(rec.Host, col); err != nil {
+ log.Printf("provision: converge seed source on %s: %v", rec.Host, err)
+ }
+ }
// Unit is compared only when the RECORD states one: `carlos
// route -unit` (issue #110) is the supported way to stamp the
// unit owning a route's process out of band, and a record that
diff --git a/internal/provision/seedreconcile.go b/internal/provision/seedreconcile.go
new file mode 100644
index 000000000..4475952c9
--- /dev/null
+++ b/internal/provision/seedreconcile.go
@@ -0,0 +1,103 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "log"
+ "os"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// SeedHooks is what the provision pass needs to act on a seeded record
+// (seeded-instances spec §3.2). Nil on ReconcileConfig: no seeded record is
+// minted or acted on, which is the safe answer for a box that cannot place
+// one.
+type SeedHooks struct {
+ // Data is this region's data bucket (seeds/ lives there).
+ Data release.Store
+}
+
+type seedVerdict int
+
+const (
+ seedOrdinary seedVerdict = iota
+ seedSkip
+ seedFinishAbort
+)
+
+// seedAction is §3.2's dispatch, run before any other per-record handling.
+// hasRow is whether this box already holds a row for the host. nested is
+// asked only for a mint, through the pass's lazy layout reader.
+func seedAction(rec Record, hasRow bool, cfg ReconcileConfig, nested func() (bool, error)) seedVerdict {
+ src := rec.Seed
+ if src == nil {
+ return seedOrdinary
+ }
+ if cfg.Seeds == nil {
+ return seedSkip
+ }
+ switch src.State {
+ case seeds.StateAborted:
+ return seedFinishAbort
+ case seeds.StateRolledBack, seeds.StateReserving, seeds.StateAwaitingUpload, seeds.StateUploaded, seeds.StateRefused:
+ return seedSkip
+ case seeds.StateReady:
+ if src.Import() {
+ return seedSkip // the swap is Plan 7's
+ }
+ if hasRow {
+ return seedOrdinary
+ }
+ if cfg.ClaimEnabled {
+ log.Printf("provision: %s: not minting a seeded instance: claim control is on for this box", rec.Host)
+ return seedSkip
+ }
+ // Only an activator-managed row is minted nested (routeFor): any
+ // other row would start without its seed and without the seed
+ // command, which is an empty database.
+ if !rec.DerivesPaths() || !rec.HibernateOrDefault() {
+ log.Printf("provision: %s: not minting a seeded instance: its record does not describe a hibernating instance", rec.Host)
+ return seedSkip
+ }
+ if ok, err := nested(); err != nil || !ok {
+ log.Printf("provision: %s: not minting a seeded instance: this box does not mint nested (%v)", rec.Host, err)
+ return seedSkip
+ }
+ return seedOrdinary
+ case seeds.StateSwapped, seeds.StatePlaced, seeds.StateCommitted:
+ return seedOrdinary
+ }
+ log.Printf("provision: %s: seed %s is in an unknown state %q; leaving it alone", rec.Host, src.ID, src.State)
+ return seedSkip
+}
+
+// finishAbortedSeed is §3.2's no-journal path for an aborted seed: delete
+// this seed's snapshot (only this seed's id, codex R4-1), then CAS the
+// record to rolled-back. Idempotent: an absent snapshot and a lost CAS are
+// both done. The console finishes from rolled-back (Plan 8).
+//
+// A host or id that fails seeds.Check names no snapshot (no valid key
+// could hold one), so nothing is deleted and the record is still rolled
+// back.
+func finishAbortedSeed(ctx context.Context, s release.Store, cfg ReconcileConfig, rec Record) {
+ if cfg.DryRun {
+ log.Printf("provision: dry-run: would finish aborted seed %s of %s", rec.Seed.ID, rec.Host)
+ return
+ }
+ if err := seeds.Check(rec.Host, rec.Seed.ID); err != nil {
+ log.Printf("provision: %s: aborted seed %q names no snapshot: %v", rec.Host, rec.Seed.ID, err)
+ } else {
+ key := seeds.SnapshotKey(rec.Host, rec.Seed.ID)
+ if err := cfg.Seeds.Data.Delete(ctx, key); err != nil && !errors.Is(err, os.ErrNotExist) {
+ log.Printf("provision: %s: delete aborted seed's snapshot: %v", rec.Host, err)
+ return
+ }
+ }
+ if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedRolledBack(rec.Seed.ID)); err != nil && !errors.Is(err, ErrSeedConflict) {
+ log.Printf("provision: %s: mark seed %s rolled back: %v", rec.Host, rec.Seed.ID, err)
+ return
+ }
+ log.Printf("provision: %s: aborted seed %s rolled back (no box had swapped it)", rec.Host, rec.Seed.ID)
+}
diff --git a/internal/provision/seedreconcile_test.go b/internal/provision/seedreconcile_test.go
new file mode 100644
index 000000000..080dae8aa
--- /dev/null
+++ b/internal/provision/seedreconcile_test.go
@@ -0,0 +1,159 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+func seededRecord(t *testing.T, s release.Store, sqid, kind, state string) Record {
+ t.Helper()
+ rec := acmeRecord(sqid)
+ id, _ := seeds.NewID()
+ rec.Seed = &SeedSource{ID: id, Kind: kind, State: state, SHA256: "a", Bytes: 1, By: "t", At: time.Now()}
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ return rec
+}
+
+func nestedSeedCfg(data release.Store) ReconcileConfig {
+ return ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data",
+ Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil },
+ Seeds: &SeedHooks{Data: data}}
+}
+
+func TestReconcileMintsAFromSeedNestedWithoutTheSeedCommand(t *testing.T) {
+ s, sqid := setup(t) // the manifest declares Seed ["seed"]
+ rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady)
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
+ t.Fatal(err)
+ }
+ got := mustGet(t, reg, rec.Host)
+ if got.Seed != "" || got.DBPath != NestedDBPath("/data", rec.Host) || got.SeedSource != SeedColumn(rec) {
+ t.Fatalf("row = seed %q db %q col %q", got.Seed, got.DBPath, got.SeedSource)
+ }
+ // The record moves on (placed), and the next pass converges the column
+ // and still never stamps the seed command.
+ if _, err := SetRecordSeed(ctx, s, rec.Account, rec.App, rec.Host, SeedPlace(rec.Seed.ID)); err != nil {
+ t.Fatal(err)
+ }
+ if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
+ t.Fatal(err)
+ }
+ got = mustGet(t, reg, rec.Host)
+ row, _ := seeds.ParseRowSeed(got.SeedSource)
+ if got.Seed != "" || row == nil || row.State != seeds.StatePlaced {
+ t.Fatalf("after placed: seed %q col %+v", got.Seed, row)
+ }
+}
+
+func TestReconcileDoesNotMintASeedFlatOrClaimed(t *testing.T) {
+ s, sqid := setup(t)
+ rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateReady)
+ for name, cfg := range map[string]ReconcileConfig{
+ "flat": {Region: "ie", Pools: []string{"tito"}, DataDir: "/data", Seeds: &SeedHooks{Data: dirStore(t)}},
+ // The claim hook admits the write, so only the seed dispatch can
+ // refuse it.
+ "claimed": func() ReconcileConfig {
+ c := nestedSeedCfg(dirStore(t))
+ c.ClaimEnabled = true
+ c.ClaimInstall = func(_ context.Context, _ ClaimRecord, write func(registry.ClaimOwnership) error) error {
+ return write(registry.ClaimOwnership{ClaimIncarnation: "inc", ClaimStamp: 1, ClaimOwner: "owner", ClaimRowID: "row"})
+ }
+ return c
+ }(),
+ "unwired": {Region: "ie", Pools: []string{"tito"}, DataDir: "/data",
+ Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }},
+ } {
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ if rt, _ := reg.Get(rec.Host); rt != nil {
+ t.Errorf("%s: minted %+v", name, rt)
+ }
+ }
+}
+
+func TestReconcileSkipsSeedsThatAreNotReady(t *testing.T) {
+ for _, c := range []struct{ kind, state string }{
+ {seeds.KindReplica, seeds.StateReserving},
+ {seeds.KindImport, seeds.StateAwaitingUpload},
+ {seeds.KindImport, seeds.StateUploaded},
+ {seeds.KindImport, seeds.StateRefused},
+ {seeds.KindImport, seeds.StateReady}, // Plan 7 swaps; here, nothing
+ {seeds.KindReplica, seeds.StateRolledBack},
+ } {
+ s, sqid := setup(t)
+ rec := seededRecord(t, s, sqid, c.kind, c.state)
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
+ t.Fatal(err)
+ }
+ if rt, _ := reg.Get(rec.Host); rt != nil {
+ t.Errorf("%s %s: minted %+v", c.kind, c.state, rt)
+ }
+ }
+}
+
+func TestReconcileFinishesAnAbortedSeedWithNoJournal(t *testing.T) {
+ s, sqid := setup(t)
+ data := dirStore(t)
+ rec := seededRecord(t, s, sqid, seeds.KindReplica, seeds.StateAborted)
+ other, _ := seeds.NewID()
+ mine := seeds.SnapshotKey(rec.Host, rec.Seed.ID)
+ theirs := seeds.SnapshotKey(rec.Host, other)
+ for _, k := range []string{mine, theirs} {
+ if _, err := data.Put(ctx, k, []byte("db"), ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := data.Get(ctx, mine); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("this seed's snapshot: %v, want it deleted", err)
+ }
+ if _, _, err := data.Get(ctx, theirs); err != nil {
+ t.Fatalf("another seed's snapshot was touched: %v", err)
+ }
+ got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
+ if err != nil || got.Seed.State != seeds.StateRolledBack {
+ t.Fatalf("record = %+v, %v; want rolled-back", got.Seed, err)
+ }
+ // Idempotent: a second pass over a rolled-back record does nothing.
+ if err := Reconcile(ctx, s, reg, nestedSeedCfg(data)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// A record that would mint a flat row even on a nested box (it does not
+// hibernate, so the activator never manages it) would start without its
+// seed and without the seed command: it is not minted.
+func TestReconcileDoesNotMintASeedThatWouldNotHibernate(t *testing.T) {
+ s, sqid := setup(t)
+ rec := acmeRecord(sqid)
+ off := false
+ rec.Hibernate = &off
+ id, _ := seeds.NewID()
+ rec.Seed = &SeedSource{ID: id, Kind: seeds.KindReplica, State: seeds.StateReady, SHA256: "a", Bytes: 1, By: "t", At: time.Now()}
+ if err := CreateRecord(ctx, s, rec); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, nestedSeedCfg(dirStore(t))); err != nil {
+ t.Fatal(err)
+ }
+ if rt, _ := reg.Get(rec.Host); rt != nil {
+ t.Errorf("minted %+v", rt)
+ }
+}