amadan

carlos / platform Public

activator: the seed fallback in Restore, marker, purge and checks (Plan 4 Task 3)

Restore dispatches a row whose seed is ready, swapped or placed to the
seed fallback (seedplace.go); every other row takes restorePlain, the old
body unchanged. The fallback refuses on claim-armed and non-nested boxes,
fails the wake when the replica listing cannot tell, restores plainly once
the host has a replica of its own (and fails rather than report fresh if
that restore finds nothing), keeps a local database only when its .seed
marker names the current seed, and otherwise purges through instancefs,
fetches through a reader bounded by the recorded size, checks length,
sha256 and integrity, writes the marker and renames the snapshot into
place. It never reports fresh, so the manifest's seed command never runs
for a seeded row. SD-N14 to SD-N19 wired; memberSafeWakeReason passes
their sentences through whole.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude e9e88033e744e7a4f414e699ce36382248a4b222 parent c555b2e
5 files changed, +647 −2
  • approved_seeded_copy_test.go +2 −2
  • internal/activator/backend_exec.go +30 −0
  • internal/activator/logsink.go +7 −0
  • internal/activator/seedplace.go +288 −0
  • internal/activator/seedplace_linux_test.go +320 −0
diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go
index d97e6d7c9..d1c4d105d 100644
--- a/approved_seeded_copy_test.go
+++ b/approved_seeded_copy_test.go
@@ -33,8 +33,8 @@ var seededCopyIDs = func() []string {
// seededWiredIDs grows as the plans wire rows. Plan 1 wires none; Plan 3
// Task 3 wires SD-N1, Task 5 SD-N2 to SD-N8 and SD-N113, Task 6 SD-N9 to SD-N13,
-// SD-N111 and SD-N112.
-var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112"}
+// SD-N111 and SD-N112; Plan 4 Task 3 SD-N14 to SD-N19.
+var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19"}
func mustParseSeededRecord(t *testing.T) []copyRow {
t.Helper()
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 54b7655b6..832feed1d 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -29,6 +29,7 @@ import (
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/schedule"
+ "github.com/carlosframework/platform/internal/seeds"
"github.com/carlosframework/platform/internal/unixdial"
)
@@ -150,6 +151,16 @@ type ExecBackend struct {
// gate (every unit test and any box that predates the check). Wired in
// production by Task 6.
NestedOK func() error
+ // Seeds reads a seed's snapshot from this region's data bucket
+ // (seeded-instances spec §3.4). Nil: no seed can be placed here, and a
+ // seeded row's wake fails rather than starting empty.
+ Seeds SeedObjects
+ // ClaimArmed is CARLOS_HOST_CLAIM on this box. The seed fallback is the
+ // plain path only; with claims armed it refuses (spec §3.4, #250).
+ ClaimArmed bool
+ // OnSeedPlaced records a seed placed (Task 4): the record's CAS to
+ // placed, then the snapshot's delete. Nil: not recorded here.
+ OnSeedPlaced func(ctx context.Context, account, app, host, id string) error
// ConfigGate answers whether this box can provide rt's config (amadan
// #213): nil means yes, an error means refuse. Acquire asks it BEFORE
// lease.Acquire, so a box that would start the instance without its
@@ -612,7 +623,26 @@ func (l *execLease) Replicate(ctx context.Context, dbPath string, syncInterval t
func (l *execLease) Release(ctx context.Context) error { return l.held.Release(ctx) }
+// Restore puts the row's database in place for a wake. A row whose seed is
+// placeable (ready, swapped or placed) goes through the seed fallback
+// (seedplace.go), which never reports fresh; every other row restores
+// plainly. A seed column that does not parse fails the wake: reading it as
+// "no seed" would boot a seeded instance empty.
func (b *ExecBackend) Restore(ctx context.Context, rt registry.Route) (bool, error) {
+ src, err := seeds.ParseRowSeed(rt.SeedSource)
+ if err != nil {
+ return false, err
+ }
+ if src != nil && src.Placeable() {
+ return b.seededRestore(ctx, rt, src)
+ }
+ return b.restorePlain(ctx, rt)
+}
+
+// restorePlain is Restore for a row with no seed to place: the warm cache,
+// then the staging fallback, then the host's own replica, and fresh only
+// when there is none of the three.
+func (b *ExecBackend) restorePlain(ctx context.Context, rt registry.Route) (bool, error) {
host, dbPath := rt.Host, b.pathsFor(rt).dbPath
if _, err := os.Stat(dbPath); err == nil {
return false, nil // warm cache — see doWake
diff --git a/internal/activator/logsink.go b/internal/activator/logsink.go
index 6cfd20f78..76291b09f 100644
--- a/internal/activator/logsink.go
+++ b/internal/activator/logsink.go
@@ -7,6 +7,7 @@
package activator
import (
+ "errors"
"regexp"
"strings"
@@ -47,6 +48,12 @@ var absPathToken = regexp.MustCompile(`(^|[\s"(=])(/[\w@.\-/]+)`)
// text itself originates in our own code (backends, materialize, lease) —
// this is a rendering pass, not a secrecy boundary.
func memberSafeWakeReason(err error) string {
+ // The seed fallback's refusals are composed for the member (SD-N14 to
+ // SD-N19) and pass through whole, without the wrapping around them.
+ var seed *SeedPlacementError
+ if errors.As(err, &seed) {
+ return "wake failed: " + seed.Sentence
+ }
s := err.Error()
switch {
case strings.Contains(s, "address already in use"):
diff --git a/internal/activator/seedplace.go b/internal/activator/seedplace.go
new file mode 100644
index 000000000..aa30ba116
--- /dev/null
+++ b/internal/activator/seedplace.go
@@ -0,0 +1,288 @@
+package activator
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/instancefs"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// SeedObjects reads a seed's snapshot, seeds/<host>/<id>/snapshot.db, from
+// this box's own region data bucket.
+type SeedObjects interface {
+ Open(ctx context.Context, host, id string) (io.ReadCloser, error)
+}
+
+// SeedMarkerSuffix names the installation evidence beside a placed
+// database: <db>.seed holds the seed's id (spec §3.4 step 3, codex R2-1).
+const SeedMarkerSuffix = ".seed"
+
+// seedMarkerMax bounds the marker read: it holds a 29-byte id and a newline,
+// and it sits in tenant-writable ground.
+const seedMarkerMax = 128
+
+// hasOwnReplica and checkSeedDB are seams for the tests.
+var (
+ hasOwnReplica = lease.HasReplica
+ checkSeedDB = lease.CheckDBContext
+)
+
+// SeedPlacementError is a wake the seed fallback failed or refused. Its
+// Sentence is the member's (SD-N14 to SD-N19): memberSafeWakeReason passes
+// it to carlos logs whole. Err is the cause, for the box log.
+type SeedPlacementError struct {
+ Sentence string
+ Err error
+}
+
+func (e *SeedPlacementError) Error() string { return e.Sentence }
+func (e *SeedPlacementError) Unwrap() error { return e.Err }
+
+func seedMismatch(host string) error {
+ return &SeedPlacementError{Sentence: fmt.Sprintf("%s could not start from its seed: the snapshot's length or sha256 does not match its record.", host)}
+}
+
+func seedNotDB(host string, err error) error {
+ return &SeedPlacementError{Sentence: fmt.Sprintf("%s could not start from its seed: the snapshot is not a database that passes an integrity check.", host), Err: err}
+}
+
+func seedUnread(host string, err error) error {
+ return &SeedPlacementError{Sentence: fmt.Sprintf("%s could not start from its seed: the snapshot could not be read. The next wake tries again.", host), Err: err}
+}
+
+func seedRefusedFlat(host string) error {
+ return &SeedPlacementError{Sentence: fmt.Sprintf("%s could not start from its seed: this box does not run instances nested.", host)}
+}
+
+func seedRefusedClaim(host string) error {
+ return &SeedPlacementError{Sentence: fmt.Sprintf("%s could not start from its seed: claim control is on for this box.", host)}
+}
+
+func seedPlaceFailed(host string, err error) error {
+ return &SeedPlacementError{Sentence: fmt.Sprintf("%s could not start from its seed: it could not be put in place. The next wake tries again.", host), Err: err}
+}
+
+// seededRestore is the plain-path seed fallback (spec §3.4). It never
+// reports fresh: a seeded row never runs the manifest's seed command, and
+// every failure fails the wake instead of falling through to an empty
+// database. Every file it touches in the instance directory goes through
+// instancefs, so no symlink there is followed.
+func (b *ExecBackend) seededRestore(ctx context.Context, rt registry.Route, src *seeds.RowSeed) (bool, error) {
+ host := rt.Host
+ if err := seeds.Check(host, src.ID); err != nil {
+ return false, fmt.Errorf("seed fallback for %s: %w", host, err)
+ }
+ fail := func(err error) (bool, error) {
+ if cause := errors.Unwrap(err); cause != nil {
+ log.Printf("activator: %s: seed %s not placed: %v: %v", host, src.ID, err, cause)
+ } else {
+ log.Printf("activator: %s: seed %s not placed: %v", host, src.ID, err)
+ }
+ return false, err
+ }
+ if b.ClaimArmed {
+ return fail(seedRefusedClaim(host))
+ }
+ paths := b.pathsFor(rt)
+ if paths.layout != LayoutNested {
+ return fail(seedRefusedFlat(host))
+ }
+ dir, base := filepath.Dir(paths.dbPath), filepath.Base(paths.dbPath)
+ // The instance directory may not exist yet on a first wake. Created as
+ // prepare and lease.Restore create it.
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ return fail(seedPlaceFailed(host, err))
+ }
+
+ // Step 1: a replica of its own means this host was placed already. A
+ // listing that cannot tell fails the wake: read as "no replica", it
+ // would place the snapshot over a lineage that exists.
+ has, err := hasOwnReplica(ctx, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host), b.Cfg.Region)
+ if err != nil {
+ return false, fmt.Errorf("seed fallback for %s: cannot tell whether it has a replica of its own: %w", host, err)
+ }
+ if has {
+ fresh, err := b.restorePlain(ctx, rt)
+ if err != nil {
+ return false, err
+ }
+ if fresh {
+ // Listed a replica, then restored none: never boot this row
+ // empty. The next wake asks again.
+ return false, fmt.Errorf("seed fallback for %s: its replica was listed but nothing was restored", host)
+ }
+ if src.State != seeds.StatePlaced {
+ b.finishPlacement(ctx, rt, src)
+ }
+ return false, nil
+ }
+
+ // This seed, placed by an interrupted first wake: keep it. Replication
+ // starts in doWake, as generation one of the host's own lineage.
+ placed, err := seedPlacedAt(dir, base, src.ID)
+ if err != nil {
+ return fail(seedPlaceFailed(host, err))
+ }
+ if placed {
+ log.Printf("activator: %s: keeping seed %s placed by an earlier wake", host, src.ID)
+ return false, nil
+ }
+ if b.Seeds == nil {
+ return fail(seedUnread(host, errors.New("no seed reader on this box")))
+ }
+ // Anything else is purged before the fetch: another seed's database,
+ // one with no marker, their WAL and SHM, the marker, litestream's
+ // metadata, and an earlier fetch's leftovers.
+ if err := purgeSeedTarget(dir, base); err != nil {
+ return fail(seedPlaceFailed(host, err))
+ }
+ if err := b.placeSeed(ctx, host, src, dir, base); err != nil {
+ return fail(err)
+ }
+ log.Printf("activator: %s: placed seed %s (%d bytes)", host, src.ID, src.Bytes)
+ return false, nil
+}
+
+// seedPlacedAt reports whether dir/base is a regular file whose marker names
+// id. A missing, symlinked or oversized marker or database is "not placed",
+// and is purged; any other error is returned.
+func seedPlacedAt(dir, base, id string) (bool, error) {
+ m, err := instancefs.ReadBeneathLimit(dir, base+SeedMarkerSuffix, seedMarkerMax)
+ if err != nil {
+ if notAPlacedFile(err) {
+ return false, nil
+ }
+ return false, err
+ }
+ if strings.TrimSpace(string(m)) != id {
+ return false, nil
+ }
+ f, err := instancefs.OpenBeneath(dir, base, os.O_RDONLY, 0)
+ if err != nil {
+ if notAPlacedFile(err) {
+ return false, nil
+ }
+ return false, err
+ }
+ f.Close()
+ return true, nil
+}
+
+func notAPlacedFile(err error) bool {
+ return errors.Is(err, os.ErrNotExist) || errors.Is(err, instancefs.ErrSymlink) ||
+ errors.Is(err, instancefs.ErrNotRegular) || errors.Is(err, instancefs.ErrTooLarge)
+}
+
+// seedTempName is where a seed's snapshot is fetched to, in the instance
+// directory. The id is a seed id (seeds.Check), one clean path element.
+func seedTempName(id string) string { return ".seed-" + id + ".tmp" }
+
+// purgeSeedTarget removes everything at and beside dir/base that could
+// stand in for a seed's database: what PurgeLocalDBArtifacts removes in its
+// full form, the marker, and any seed temp file, through instancefs.
+func purgeSeedTarget(dir, base string) error {
+ names := []string{base, base + "-wal", base + "-shm", base + SeedMarkerSuffix,
+ base + "-litestream", ".activator-" + base + "-litestream"}
+ ents, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ for _, e := range ents {
+ if n := e.Name(); strings.HasPrefix(n, ".seed-sd_") && strings.Contains(n, ".tmp") {
+ names = append(names, n)
+ }
+ }
+ for _, n := range names {
+ if err := instancefs.RemoveAllBeneath(dir, n); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// placeSeed fetches and checks the snapshot, then writes the marker and
+// renames the snapshot onto dir/base. The marker goes first, so a database
+// at that path always has one. The temp file and any sidecar the check left
+// are removed on every path.
+func (b *ExecBackend) placeSeed(ctx context.Context, host string, src *seeds.RowSeed, dir, base string) error {
+ tmp := seedTempName(src.ID)
+ defer func() {
+ for _, n := range []string{tmp, tmp + "-wal", tmp + "-shm"} {
+ _ = instancefs.RemoveAllBeneath(dir, n)
+ }
+ }()
+ if err := b.fetchSeed(ctx, host, src, dir, tmp); err != nil {
+ return err
+ }
+ if err := checkSeedDB(ctx, filepath.Join(dir, tmp)); err != nil {
+ return seedNotDB(host, err)
+ }
+ if err := instancefs.WriteAtomic(dir, base+SeedMarkerSuffix, []byte(src.ID+"\n"), 0o600); err != nil {
+ return seedPlaceFailed(host, err)
+ }
+ if err := instancefs.RenameBetween(dir, tmp, dir, base); err != nil {
+ return seedPlaceFailed(host, err)
+ }
+ return nil
+}
+
+// fetchSeed copies the snapshot into dir/tmp, checking its length and
+// sha256 against the row as it writes. It reads at most one byte more than
+// the row records, so an oversized object is refused without being read.
+func (b *ExecBackend) fetchSeed(ctx context.Context, host string, src *seeds.RowSeed, dir, tmp string) error {
+ rc, err := b.Seeds.Open(ctx, host, src.ID)
+ if err != nil {
+ return seedUnread(host, err)
+ }
+ defer rc.Close()
+ f, err := instancefs.OpenBeneath(dir, tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+ if err != nil {
+ return seedPlaceFailed(host, err)
+ }
+ h := sha256.New()
+ r := &readErrs{r: io.LimitReader(rc, src.Bytes+1)}
+ n, cerr := io.Copy(io.MultiWriter(f, h), r)
+ serr := f.Sync()
+ ferr := f.Close()
+ if r.err != nil {
+ return seedUnread(host, r.err)
+ }
+ if cerr != nil || serr != nil || ferr != nil {
+ return seedPlaceFailed(host, errors.Join(cerr, serr, ferr))
+ }
+ if n != src.Bytes || hex.EncodeToString(h.Sum(nil)) != src.SHA256 {
+ return seedMismatch(host)
+ }
+ return nil
+}
+
+// readErrs keeps the read side's error apart from the write side's, which
+// io.Copy returns alike: a bucket that fails is SD-N16, a disk that fails is
+// SD-N19.
+type readErrs struct {
+ r io.Reader
+ err error
+}
+
+func (e *readErrs) Read(p []byte) (int, error) {
+ n, err := e.r.Read(p)
+ if err != nil && err != io.EOF {
+ e.err = err
+ }
+ return n, err
+}
+
+// finishPlacement is Task 4's: the record's CAS to placed and the
+// snapshot's delete, best effort, once the host has a replica of its own.
+func (b *ExecBackend) finishPlacement(context.Context, registry.Route, *seeds.RowSeed) {}
diff --git a/internal/activator/seedplace_linux_test.go b/internal/activator/seedplace_linux_test.go
new file mode 100644
index 000000000..ec0fbeade
--- /dev/null
+++ b/internal/activator/seedplace_linux_test.go
@@ -0,0 +1,320 @@
+//go:build linux
+
+package activator
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+type fakeSeedObjects struct {
+ body map[string][]byte // id -> snapshot
+ err error
+ opens int
+ read int64 // bytes the fallback pulled from the object
+}
+
+func (f *fakeSeedObjects) Open(_ context.Context, host, id string) (io.ReadCloser, error) {
+ f.opens++
+ if f.err != nil {
+ return nil, f.err
+ }
+ b, ok := f.body[id]
+ if !ok {
+ return nil, os.ErrNotExist
+ }
+ return io.NopCloser(&countingReader{r: bytes.NewReader(b), n: &f.read}), nil
+}
+
+type countingReader struct {
+ r io.Reader
+ n *int64
+}
+
+func (c *countingReader) Read(p []byte) (int, error) {
+ n, err := c.r.Read(p)
+ *c.n += int64(n)
+ return n, err
+}
+
+// seededRoute is nestedBackend's row with a seed column for a snapshot
+// made by writeMinimalSQLite.
+func seededRoute(t *testing.T, b *ExecBackend, rt registry.Route, state string) (registry.Route, *fakeSeedObjects, string) {
+ t.Helper()
+ id, _ := seeds.NewID()
+ snap := filepath.Join(t.TempDir(), "snap.db")
+ if err := writeMinimalSQLite(snap); err != nil {
+ t.Fatal(err)
+ }
+ body, _ := os.ReadFile(snap)
+ sum := sha256.Sum256(body)
+ rt.SeedSource = seeds.RowSeed{ID: id, Kind: seeds.KindReplica, SHA256: hex.EncodeToString(sum[:]), Bytes: int64(len(body)), State: state}.Encode()
+ objs := &fakeSeedObjects{body: map[string][]byte{id: body}}
+ b.Seeds = objs
+ return rt, objs, id
+}
+
+func noOwnReplica(t *testing.T) {
+ t.Helper()
+ old := hasOwnReplica
+ hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return false, nil }
+ t.Cleanup(func() { hasOwnReplica = old })
+}
+
+func TestSeededRestorePlacesTheSnapshot(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
+ fresh, err := b.Restore(context.Background(), rt)
+ if err != nil || fresh {
+ t.Fatalf("Restore = fresh %v, %v; want placed, never fresh", fresh, err)
+ }
+ if objs.opens != 1 {
+ t.Fatalf("snapshot opened %d times", objs.opens)
+ }
+ marker, err := os.ReadFile(rt.DBPath + SeedMarkerSuffix)
+ if err != nil || strings.TrimSpace(string(marker)) != id {
+ t.Fatalf("marker = %q, %v; want the seed id", marker, err)
+ }
+ if err := checkSeedDB(context.Background(), rt.DBPath); err != nil {
+ t.Fatalf("the placed database does not check: %v", err)
+ }
+ // Nothing of the fetch is left beside the database.
+ ents, _ := os.ReadDir(filepath.Dir(rt.DBPath))
+ for _, e := range ents {
+ if strings.HasPrefix(e.Name(), ".seed-") {
+ t.Errorf("%s left behind", e.Name())
+ }
+ }
+ // A second wake (a crash after the rename, before the first upload)
+ // keeps the placed file and does not fetch again.
+ if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 1 {
+ t.Fatalf("resume: fresh %v err %v opens %d", fresh, err, objs.opens)
+ }
+}
+
+func TestSeededRestorePurgesAnotherSeedsDatabase(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, _, id := seededRoute(t, b, rt, seeds.StateSwapped)
+ os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
+ os.WriteFile(rt.DBPath, []byte("an aborted import's data"), 0o600)
+ os.WriteFile(rt.DBPath+"-wal", []byte("w"), 0o600)
+ os.WriteFile(rt.DBPath+"-shm", []byte("s"), 0o600)
+ os.WriteFile(rt.DBPath+SeedMarkerSuffix, []byte("sd_bbbbbbbbbbbbbbbbbbbbbbbbbb\n"), 0o600)
+ meta := filepath.Join(filepath.Dir(rt.DBPath), ".activator-"+filepath.Base(rt.DBPath)+"-litestream")
+ os.MkdirAll(meta, 0o700)
+ if _, err := b.Restore(context.Background(), rt); err != nil {
+ t.Fatal(err)
+ }
+ for _, p := range []string{rt.DBPath + "-wal", rt.DBPath + "-shm", meta} {
+ if _, err := os.Stat(p); !os.IsNotExist(err) {
+ t.Errorf("%s survived the purge", p)
+ }
+ }
+ if m, _ := os.ReadFile(rt.DBPath + SeedMarkerSuffix); strings.TrimSpace(string(m)) != id {
+ t.Fatalf("marker = %q, want %s", m, id)
+ }
+ if b, _ := os.ReadFile(rt.DBPath); bytes.Equal(b, []byte("an aborted import's data")) {
+ t.Fatal("the other seed's database stood in for this one")
+ }
+}
+
+func TestSeededRestorePurgesAnUnmarkedDatabase(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
+ os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
+ os.WriteFile(rt.DBPath, []byte("an unmarked database"), 0o600)
+ if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || objs.opens != 1 {
+ t.Fatalf("fresh %v err %v opens %d; want the unmarked database replaced by the snapshot", fresh, err, objs.opens)
+ }
+ if got, _ := os.ReadFile(rt.DBPath); !bytes.Equal(got, objs.body[mustSeedID(t, rt)]) {
+ t.Fatal("the unmarked database stood in for the seed")
+ }
+}
+
+func TestSeededRestoreRemovesASymlinkedDatabaseNotItsTarget(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, _, id := seededRoute(t, b, rt, seeds.StateReady)
+ os.MkdirAll(filepath.Dir(rt.DBPath), 0o700)
+ outside := filepath.Join(t.TempDir(), "outside.db")
+ os.WriteFile(outside, []byte("not the instance's"), 0o600)
+ outsideMarker := filepath.Join(t.TempDir(), "marker")
+ os.WriteFile(outsideMarker, []byte(id+"\n"), 0o600)
+ if err := os.Symlink(outside, rt.DBPath); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(outsideMarker, rt.DBPath+SeedMarkerSuffix); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := b.Restore(context.Background(), rt); err != nil {
+ t.Fatal(err)
+ }
+ if got, _ := os.ReadFile(outside); string(got) != "not the instance's" {
+ t.Fatalf("the link's target was touched: %q", got)
+ }
+ if fi, err := os.Lstat(rt.DBPath); err != nil || !fi.Mode().IsRegular() {
+ t.Fatalf("the database is not a placed file: %v %v", fi, err)
+ }
+ if err := checkSeedDB(context.Background(), rt.DBPath); err != nil {
+ t.Fatalf("the placed database does not check: %v", err)
+ }
+}
+
+func mustSeedID(t *testing.T, rt registry.Route) string {
+ t.Helper()
+ src, err := seeds.ParseRowSeed(rt.SeedSource)
+ if err != nil || src == nil {
+ t.Fatalf("seed column: %v", err)
+ }
+ return src.ID
+}
+
+func TestSeededRestoreRefusesABadSnapshot(t *testing.T) {
+ noOwnReplica(t)
+ for _, c := range []struct {
+ name string
+ mangle func(*fakeSeedObjects, string)
+ want string
+ }{
+ {"short", func(f *fakeSeedObjects, id string) { f.body[id] = f.body[id][:100] }, "length or sha256"},
+ {"long", func(f *fakeSeedObjects, id string) { f.body[id] = append(f.body[id], 'x') }, "length or sha256"},
+ {"wrong bytes", func(f *fakeSeedObjects, id string) { f.body[id] = bytes.Repeat([]byte{'x'}, len(f.body[id])) }, "length or sha256"},
+ {"unreadable", func(f *fakeSeedObjects, id string) { f.err = errors.New("bucket down") }, "could not be read"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
+ c.mangle(objs, id)
+ fresh, err := b.Restore(context.Background(), rt)
+ var spe *SeedPlacementError
+ if fresh || !errors.As(err, &spe) || !strings.Contains(spe.Sentence, c.want) {
+ t.Fatalf("Restore = fresh %v, %v; want a %q refusal", fresh, err, c.want)
+ }
+ if _, err := os.Stat(rt.DBPath); !os.IsNotExist(err) {
+ t.Fatal("a refused snapshot left a database at the row's path")
+ }
+ })
+ }
+}
+
+// The object is read through the recorded size and one byte more, never to
+// its end: an oversized object costs the box nothing beyond the record.
+func TestSeededRestoreReadsNoMoreThanTheRecordedSize(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, id := seededRoute(t, b, rt, seeds.StateReady)
+ want := int64(len(objs.body[id]))
+ objs.body[id] = append(objs.body[id], make([]byte, 4<<20)...)
+ var spe *SeedPlacementError
+ if _, err := b.Restore(context.Background(), rt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "length or sha256") {
+ t.Fatalf("Restore = %v; want the length refusal", err)
+ }
+ if objs.read > want+1 {
+ t.Fatalf("read %d bytes of the object; the record says %d", objs.read, want)
+ }
+}
+
+func TestSeededRestoreRefusesANonDatabase(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ id, _ := seeds.NewID()
+ body := []byte("not sqlite at all, but the right length and hash")
+ sum := sha256.Sum256(body)
+ rt.SeedSource = seeds.RowSeed{ID: id, Kind: seeds.KindImport, SHA256: hex.EncodeToString(sum[:]), Bytes: int64(len(body)), State: seeds.StateSwapped}.Encode()
+ b.Seeds = &fakeSeedObjects{body: map[string][]byte{id: body}}
+ _, err := b.Restore(context.Background(), rt)
+ var spe *SeedPlacementError
+ if !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "integrity check") {
+ t.Fatalf("Restore = %v; want the integrity refusal", err)
+ }
+}
+
+func TestSeededRestoreRefusesClaimAndFlat(t *testing.T) {
+ noOwnReplica(t)
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, _, _ = seededRoute(t, b, rt, seeds.StateReady)
+ b.ClaimArmed = true
+ var spe *SeedPlacementError
+ if _, err := b.Restore(context.Background(), rt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "claim control") {
+ t.Fatalf("claim-armed: %v", err)
+ }
+ b.ClaimArmed = false
+ flat := testExecBackend(t)
+ flat.Seeds = b.Seeds
+ frt := registry.Route{Host: rt.Host, Kind: "instance", Hibernate: true, SeedSource: rt.SeedSource,
+ Socket: filepath.Join(t.TempDir(), "x.sock")} // DBPath empty: the flat default
+ if _, err := flat.Restore(context.Background(), frt); !errors.As(err, &spe) || !strings.Contains(spe.Sentence, "nested") {
+ t.Fatalf("flat: %v", err)
+ }
+}
+
+func TestSeededRestoreFailsWhenTheReplicaCannotBeListed(t *testing.T) {
+ old := hasOwnReplica
+ hasOwnReplica = func(context.Context, string, string, string) (bool, error) {
+ return false, errors.New("listing failed")
+ }
+ defer func() { hasOwnReplica = old }()
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
+ if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh || objs.opens != 0 {
+ t.Fatalf("Restore = fresh %v, %v, opens %d; want a failed wake and no fetch", fresh, err, objs.opens)
+ }
+}
+
+func TestSeededRestoreUsesItsOwnReplicaOnceItHasOne(t *testing.T) {
+ old := hasOwnReplica
+ hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return true, nil }
+ defer func() { hasOwnReplica = old }()
+ oldRestore := restoreReplica
+ restored := 0
+ restoreReplica = func(_ context.Context, dbPath, _, _, _ string) (bool, error) {
+ restored++
+ return true, writeMinimalSQLite(dbPath)
+ }
+ defer func() { restoreReplica = oldRestore }()
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, _ := seededRoute(t, b, rt, seeds.StatePlaced)
+ if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh || restored != 1 || objs.opens != 0 {
+ t.Fatalf("fresh %v err %v restored %d opens %d; want the own replica, not the seed", fresh, err, restored, objs.opens)
+ }
+}
+
+// A listing that saw a replica and a restore that then found none must not
+// come back fresh: fresh would boot the instance empty and run the
+// manifest's seed command over a seeded row.
+func TestSeededRestoreNeverReportsFreshWhenItsReplicaVanishes(t *testing.T) {
+ old := hasOwnReplica
+ hasOwnReplica = func(context.Context, string, string, string) (bool, error) { return true, nil }
+ defer func() { hasOwnReplica = old }()
+ oldRestore := restoreReplica
+ restoreReplica = func(context.Context, string, string, string, string) (bool, error) { return false, nil }
+ defer func() { restoreReplica = oldRestore }()
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ rt, objs, _ := seededRoute(t, b, rt, seeds.StateReady)
+ if fresh, err := b.Restore(context.Background(), rt); err == nil || fresh || objs.opens != 0 {
+ t.Fatalf("Restore = fresh %v, %v, opens %d; want a failed wake", fresh, err, objs.opens)
+ }
+}
+
+func TestMemberSafeWakeReasonPassesTheSeedSentence(t *testing.T) {
+ err := fmtWrap(seedRefusedFlat("shop.example.com"))
+ if got := memberSafeWakeReason(err); got != "wake failed: shop.example.com could not start from its seed: this box does not run instances nested." {
+ t.Fatalf("reason = %q", got)
+ }
+}
+
+func fmtWrap(err error) error { return errors.Join(errors.New("restore shop.example.com"), err) }