amadan

carlos / platform Public

activator: a seed's processes are proven gone before its database goes, and an unproven seed leaves an obligation

A failed flat seed's cgroup (flag on its role child, flag off the host cgroup) is killed and proven empty before RemoveChild and removeSeedDB; a successful flag-on seed's role child is too. When the proof fails the database is kept, <db>.seed-unproven is written durably beside it and the seed answers ErrSeedUnproven. Restore answers the obligation before the database can be warm: it proves the host cgroup empty, then removes the database, its WAL and SHM and the obligation so the seed runs again, or refuses with ErrSeedUnproven, and doWake leaves the lease to its TTL. ErrNoDelegation keeps the leader-only proof with a log line.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 4c5e43b564f2dff765ba8a4580fc25e8c4edbfba parent 8530210
5 files changed, +414 −35
  • internal/activator/activator.go +10 −1
  • internal/activator/activator_test.go +25 −0
  • internal/activator/backend_exec.go +177 −28
  • internal/activator/containment.go +14 −6
  • internal/activator/containment_test.go +188 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index fc67b3c18..c4f42dda4 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -1926,7 +1926,16 @@ func (a *Service) doWake(ctx context.Context, e *actEntry) (*wokenInstance, erro
// and no marker is ever read to skip that.
fresh, err := a.backend.Restore(ctx, rt)
if err != nil {
- w.stopOutcome.leaseReleased = releaseLease(ctx, lease) == nil
+ if errors.Is(err, ErrSeedUnproven) {
+ // An earlier seed's processes are still not proven gone (its
+ // <db>.seed-unproven obligation, Task 8): they may still write
+ // the database, so the lease is left to its TTL, as the seed's
+ // own ErrSeedUnproven leaves it (step 3).
+ log.Printf("activator: %s: %v; leaving the lease to its TTL", host, err)
+ w.stopOutcome.uncertain = true
+ } else {
+ w.stopOutcome.leaseReleased = releaseLease(ctx, lease) == nil
+ }
return w, fmt.Errorf("restore %s: %w", host, err)
}
if ctx.Err() != nil || !a.wakeAllowed(host) {
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
index 50b137a29..4539f774b 100644
--- a/internal/activator/activator_test.go
+++ b/internal/activator/activator_test.go
@@ -1219,6 +1219,31 @@ func TestActivatorSeedUnprovenLeavesTheLeaseToItsTTL(t *testing.T) {
}
}
+// Task 8, plan 3: a wake whose Restore refuses because an earlier seed's
+// processes are still not proven gone (the <db>.seed-unproven obligation)
+// treats it as the seed's own ErrSeedUnproven: nothing is seeded or
+// started, and the lease is left to its TTL.
+func TestActivatorRestoreRefusedForAnUnprovenSeedLeavesTheLeaseToItsTTL(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Minute)
+ b.restoreErr = fmt.Errorf("unproven.example.com: %w: an earlier seed's processes are not proven gone", ErrSeedUnproven)
+ rt := testRoute("unproven.example.com")
+ rt.Seed = `["seed"]`
+
+ _, err := a.beginRequest(context.Background(), rt)
+ if !errors.Is(err, ErrSeedUnproven) {
+ t.Fatalf("wake error = %v, want it to wrap ErrSeedUnproven", err)
+ }
+ if got := b.releaseCount(); got != 0 {
+ t.Fatalf("lease releases = %d, want 0: something of the earlier seed may still write the database", got)
+ }
+ b.mu.Lock()
+ seeds, starts := b.seeds, b.starts
+ b.mu.Unlock()
+ if seeds != 0 || starts != 0 {
+ t.Fatalf("seeds=%d starts=%d after a refused restore, want neither", seeds, starts)
+ }
+}
+
// TestActivatorWakeRefusesFencedRowBeforeAcquire pins design §6.8: a fenced
// row is refused at the top of doWake, before the lease is ever acquired,
// not only inside Start after the lease is already held.
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index a7f6d8316..bf5c3cb93 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"log"
"net"
"net/http"
@@ -30,6 +31,7 @@ import (
"github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/safefile"
"github.com/carlosframework/platform/internal/schedule"
"github.com/carlosframework/platform/internal/unixdial"
)
@@ -46,6 +48,13 @@ var spawnHealthTimeout = 30 * time.Second
// that might still be happening.
var ErrSeedUnproven = errors.New("activator: seed unproven; processes remain")
+// seedUnprovenSuffix names a flat seed's unproven-cleanup obligation, kept
+// beside the database (<db>.seed-unproven): a seed failed, its processes
+// were not proven gone, and so its database was kept. While the file is
+// there no wake treats the database as warm (Restore). It does not end in
+// ".db", so the box-wide litestream glob never picks it up.
+const seedUnprovenSuffix = ".seed-unproven"
+
// quarantineSuffix names where a restore that failed its integrity check is
// parked. It deliberately does NOT end in ".db": the box-wide litestream
// generator globs <data>/*.db, and a quarantined corpse must never be picked
@@ -247,6 +256,11 @@ type ExecBackend struct {
// failed health wait). Nil: KillGen. Tests replace this seam, never the
// Cgroups field.
killGen func(ctx context.Context, rt registry.Route, n uint64) error
+ // cgroupKill is seedDescendantsGone's kill and proof: child "" is the
+ // host cgroup (KillHost), any other name one of its children. Nil: the
+ // cgroup manager's own. Tests replace this seam, never the Cgroups
+ // field.
+ cgroupKill func(ctx context.Context, rt registry.Route, child string) error
// strictPreflight is NextSpawnStrict's seam for PrepareStrict's runtime
// preflight. Nil: Cgroups.StrictPreflight, the very function
// PrepareStrict runs, so S2 and the spawn cannot disagree.
@@ -712,6 +726,12 @@ func (l *execLease) Err() error { return l.held.Err() }
func (b *ExecBackend) Restore(ctx context.Context, rt registry.Route) (bool, error) {
host, dbPath := rt.Host, b.pathsFor(rt).dbPath
+ // A failed seed whose processes were not proven gone left its database
+ // and an obligation (Task 8): it is answered before the database can
+ // count as a warm cache.
+ if err := b.settleSeedObligation(ctx, rt, dbPath); err != nil {
+ return false, err
+ }
if _, err := os.Stat(dbPath); err == nil {
return false, nil // warm cache — see doWake
} else if !os.IsNotExist(err) {
@@ -989,43 +1009,172 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route, exe stri
cmd.WaitDelay = seedDrainGrace
return cmd
}, nil)
- if err == nil {
+ spawned := err == nil
+ if spawned {
err = cmd.Wait()
settle()
- b.Cgroups.RemoveChild(host, child) // the seed's own child only; no-op for "" (flag off)
}
- if err != nil {
- // WaitDelay reports on the SUCCESS path too: a seed that exits 0 after
- // backgrounding something (a helper, a stray `&`) leaves the inherited
- // stderr pipe open, and Wait then returns ErrWaitDelay INSTEAD of nil
- // even though the process exited 0. Treating that as a failure would
- // fail the wake AND delete the database the seed had just correctly
- // built — leaving the host fresh again next wake, i.e. permanently
- // un-wakeable. The exit status is the verdict; the leaked pipe is a
- // log line.
- if cmd != nil && errors.Is(err, exec.ErrWaitDelay) && cmd.ProcessState != nil && cmd.ProcessState.Success() {
- log.Printf("activator: seeded %s (it exited 0 but left a child holding its output pipe open for more than %s)", host, seedDrainGrace)
- return nil
- }
+ // WaitDelay reports on the SUCCESS path too: a seed that exits 0 after
+ // backgrounding something (a helper, a stray `&`) leaves the inherited
+ // stderr pipe open, and Wait then returns ErrWaitDelay INSTEAD of nil
+ // even though the process exited 0. Treating that as a failure would
+ // fail the wake AND delete the database the seed had just correctly
+ // built — leaving the host fresh again next wake, i.e. permanently
+ // un-wakeable. The exit status is the verdict; the leaked pipe is a
+ // log line.
+ exitedZero := spawned && (err == nil || (errors.Is(err, exec.ErrWaitDelay) && cmd.ProcessState != nil && cmd.ProcessState.Success()))
+ var failure error
+ if !exitedZero {
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
err = fmt.Errorf("timed out after %s and was killed: %w", seedTimeout, err)
}
- // Take the half-built database with the failure. Failing the wake is
- // only half the promise: the file a crashed seed left behind would
- // look like a warm cache to the NEXT wake — DB present, so not fresh,
- // so no seed — and the instance would boot on it, which is the
- // half-seeded instance this whole path exists to prevent. Only a
- // database that wasn't there when we started is ours to remove, and
- // under the lease nothing else can have written it.
- if seedOwnsDB {
- removeSeedDB(dbPath)
- }
+ failure = fmt.Errorf("seeding %s: %w", host, err)
if tail := strings.TrimSpace(stderr.String()); tail != "" {
- return fmt.Errorf("seeding %s: %w: %s", host, err, tail)
+ failure = fmt.Errorf("seeding %s: %w: %s", host, err, tail)
+ }
+ }
+ // What the seed left is proven gone before its database is touched
+ // (Task 8). CommandContext and Wait speak for the leader only, and a
+ // process it forked keeps writing the database, a removed one included.
+ // A failed seed's cgroup is killed and proven empty: flag on its role
+ // child, flag off carlos-exec@<host>, which holds nothing else yet (a
+ // fresh host, before the spawn, under the wake's lease). A successful
+ // seed's role child is too (flag on): a helper that outlived the seed
+ // would otherwise run on beside the instance, out of reach of any
+ // generation kill. Flag off, a successful seed's helper stays in the
+ // host cgroup, where the terminal stop's proof (stopHostProven) finds
+ // it. A spawn that failed ran nothing.
+ if spawned && (failure != nil || child != "") {
+ pctx, pcancel := context.WithTimeout(context.WithoutCancel(ctx), actStopTimeout)
+ perr := b.seedDescendantsGone(pctx, rt, child)
+ pcancel()
+ if perr != nil {
+ // The database is kept, as seedUnitFailed keeps it: removing it
+ // under a live writer is the hazard itself. The obligation beside
+ // it keeps every later wake from serving it as warm until the
+ // proof holds (Restore), and the caller leaves the lease to its
+ // TTL (ErrSeedUnproven).
+ if failure == nil {
+ failure = fmt.Errorf("seed for %s: %w: processes remain after a successful seed: %v", host, ErrSeedUnproven, perr)
+ } else {
+ failure = fmt.Errorf("%w: %w (the seed's processes remain after stopping it; its database was left in place: %v)", failure, ErrSeedUnproven, perr)
+ }
+ leaveSeedObligation(host, dbPath, seedOwnsDB, failure)
+ return failure
}
- return fmt.Errorf("seeding %s: %w", host, err)
+ b.Cgroups.RemoveChild(host, child) // the seed's own child only; no-op for "" (flag off)
}
- log.Printf("activator: seeded %s", host)
+ if failure == nil {
+ if err != nil {
+ log.Printf("activator: seeded %s (it exited 0 but left a child holding its output pipe open for more than %s)", host, seedDrainGrace)
+ } else {
+ log.Printf("activator: seeded %s", host)
+ }
+ return nil
+ }
+ // Take the half-built database with the failure. Failing the wake is
+ // only half the promise: the file a crashed seed left behind would
+ // look like a warm cache to the NEXT wake — DB present, so not fresh,
+ // so no seed — and the instance would boot on it, which is the
+ // half-seeded instance this whole path exists to prevent. Only a
+ // database that wasn't there when we started is ours to remove, and
+ // under the lease, with the seed's processes proven gone above, nothing
+ // else can have written it.
+ if seedOwnsDB {
+ removeSeedDB(dbPath)
+ }
+ return failure
+}
+
+// seedDescendantsGone kills what a flat seed left and proves it gone: flag
+// on, its own role child (child); flag off, carlos-exec@<host> (""), which
+// holds nothing else yet: a fresh host, before the spawn, under the wake's
+// lease. An absent cgroup is empty. A box with no delegated cgroup
+// (cgroup.ErrNoDelegation) ran the seed unaccounted, with nothing that
+// could reach a descendant, so the leader's exit is the only proof there
+// is (ledger 127): it logs and answers nil. Any other failure is returned,
+// and the caller keeps the database (ErrSeedUnproven).
+func (b *ExecBackend) seedDescendantsGone(ctx context.Context, rt registry.Route, child string) error {
+ var err error
+ switch {
+ case b.cgroupKill != nil:
+ err = b.cgroupKill(ctx, rt, child)
+ case child == "":
+ err = b.KillHost(ctx, rt)
+ default:
+ err = b.killChild(ctx, rt.Host, child)
+ }
+ if errors.Is(err, cgroup.ErrNoDelegation) {
+ log.Printf("activator: %s: the seed ran with no delegated cgroup; its leader's exit is the only proof there is", rt.Host)
+ return nil
+ }
+ return err
+}
+
+// leaveSeedObligation records, durably and beside the database, that a
+// seed's processes were not proven gone (<db>.seed-unproven): Restore
+// refuses to treat the database as warm while it is there. Only a database
+// the seed created is the obligation's to remove later, so with seedOwnsDB
+// false (a database that was there before the seed) nothing is recorded.
+// A failed write is logged: the wake already fails, and the lease is left
+// to its TTL.
+func leaveSeedObligation(host, dbPath string, seedOwnsDB bool, cause error) {
+ if !seedOwnsDB {
+ log.Printf("activator: %s: the seed's processes are not proven gone; %s was there before the seed, so it is kept and no obligation is recorded", host, dbPath)
+ return
+ }
+ ob := dbPath + seedUnprovenSuffix
+ if err := safefile.WriteFile(ob, []byte(cause.Error()+"\n"), 0o600); err != nil {
+ log.Printf("activator: %s: COULD NOT RECORD the unproven seed at %s: %v; the next wake may treat its database as warm", host, ob, err)
+ return
+ }
+ log.Printf("activator: %s: the seed's processes are not proven gone; its database is kept and %s refuses it as warm until they are", host, ob)
+}
+
+// settleSeedObligation answers an earlier seed's obligation
+// (<db>.seed-unproven, leaveSeedObligation) before anything treats dbPath
+// as warm. With none there it does nothing. Otherwise the host's processes
+// are proven gone first, and only then are the database, its WAL and SHM
+// and, last, the obligation removed, so the wake goes on as a first boot
+// and the seed runs again. A proof that fails refuses the wake with
+// ErrSeedUnproven and removes nothing. The obligation is a file, so it
+// survives an agent restart.
+//
+// The proof is on carlos-exec@<host>, whatever the layout now: it holds
+// the seed's role child as well, so it covers the seed whichever way it
+// ran, including across a CARLOS_SWAP change between the two wakes. It is
+// safe to kill there: this runs under the wake's lease, before anything of
+// this wake has run, and no other entry of the host is live (an unproven
+// failed wake or stray sidecar refuses the wake before the lease).
+func (b *ExecBackend) settleSeedObligation(ctx context.Context, rt registry.Route, dbPath string) error {
+ host := rt.Host
+ ob := dbPath + seedUnprovenSuffix
+ if _, err := os.Lstat(ob); errors.Is(err, fs.ErrNotExist) {
+ return nil
+ } else if err != nil {
+ // "Couldn't tell" is never "no obligation".
+ return fmt.Errorf("%s: read the seed obligation %s: %w", host, ob, err)
+ }
+ pctx, cancel := context.WithTimeout(ctx, actStopTimeout)
+ err := b.seedDescendantsGone(pctx, rt, "")
+ cancel()
+ if err != nil {
+ return fmt.Errorf("%s: %w: an earlier seed's processes are not proven gone, so its database is not used: %v", host, ErrSeedUnproven, err)
+ }
+ for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} {
+ if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return fmt.Errorf("%s: remove %s after an unproven seed: %w", host, p, err)
+ }
+ }
+ // The removals are durable before the obligation goes, so a crash in
+ // between can only leave the obligation, never the database without it.
+ if err := syncDir(filepath.Dir(dbPath)); err != nil {
+ return fmt.Errorf("%s: sync %s after an unproven seed: %w", host, filepath.Dir(dbPath), err)
+ }
+ if err := os.Remove(ob); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return fmt.Errorf("%s: remove the seed obligation %s: %w", host, ob, err)
+ }
+ log.Printf("activator: %s: the earlier seed's processes are proven gone; its database is removed and the seed runs again", host)
return nil
}
diff --git a/internal/activator/containment.go b/internal/activator/containment.go
index 2d0f6bef3..9e907366d 100644
--- a/internal/activator/containment.go
+++ b/internal/activator/containment.go
@@ -299,7 +299,19 @@ func (b *ExecBackend) KillGen(ctx context.Context, rt registry.Route, n uint64)
return cgroup.ErrNoDelegation
}
child := cgroup.GenChild(n)
- c, err := b.Cgroups.OpenCleanupChild(rt.Host, child)
+ if err := b.killChild(ctx, rt.Host, child); err != nil {
+ return err
+ }
+ b.Cgroups.RemoveChild(rt.Host, child)
+ return nil
+}
+
+// killChild kills whatever is in one child of host's generation parent and
+// returns once that child is empty. It touches no sibling. An absent child
+// is empty; no delegated cgroup at all is cgroup.ErrNoDelegation. The child
+// is kept: the caller removes it (RemoveChild) once it is done with it.
+func (b *ExecBackend) killChild(ctx context.Context, host, child string) error {
+ c, err := b.Cgroups.OpenCleanupChild(host, child)
if errors.Is(err, cgroup.ErrNoCgroup) {
return nil // already removed, so already empty
}
@@ -310,9 +322,5 @@ func (b *ExecBackend) KillGen(ctx context.Context, rt registry.Route, n uint64)
_ = c.Close()
return err
}
- if err := c.Close(); err != nil {
- return err
- }
- b.Cgroups.RemoveChild(rt.Host, child)
- return nil
+ return c.Close()
}
diff --git a/internal/activator/containment_test.go b/internal/activator/containment_test.go
index 993471a27..7a85a1043 100644
--- a/internal/activator/containment_test.go
+++ b/internal/activator/containment_test.go
@@ -7,6 +7,9 @@ import (
"os"
"os/exec"
"path/filepath"
+ "strconv"
+ "strings"
+ "syscall"
"testing"
"github.com/carlosframework/platform/internal/cgroup"
@@ -339,3 +342,188 @@ func TestStartedThenExitedIsUncertainAndKeepsItsText(t *testing.T) {
t.Fatal("another uncertain start reads as a crash at the start")
}
}
+
+// Task 8: a failed flat seed's processes are killed and proven gone before
+// its database is removed. Here they cannot be (a temp dir is no cgroupfs,
+// so the host cgroup's kill is unproven): the seed answers ErrSeedUnproven,
+// its database stays, and a durable obligation is left beside it.
+func TestAFailedSeedWhoseCleanupIsUnprovenKeepsTheDBAndLeavesAnObligation(t *testing.T) {
+ const host = "unproven-seed.example.com"
+ b, _ := delegatedBackend(t, host)
+ stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nexit 1\n")
+ rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}
+
+ err := b.Seed(context.Background(), rt)
+ if !errors.Is(err, ErrSeedUnproven) {
+ t.Fatalf("Seed = %v, want ErrSeedUnproven: the seed's leftovers were never proven gone", err)
+ }
+ db := filepath.Join(b.DataDir, host+".db")
+ if got, err := os.ReadFile(db); err != nil || string(got) != "half" {
+ t.Fatalf("database = %q, %v; want it kept: a process may still be writing it", got, err)
+ }
+ if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
+ t.Fatalf("no obligation beside the database: %v", err)
+ }
+}
+
+// Task 8, plan 3: the obligation survives to the next wake. While the proof
+// still fails, Restore refuses and the database is never reported warm.
+// Once it passes, the database, its WAL and SHM and the obligation are
+// removed, the wake is fresh, and the seed runs again.
+func TestTheNextWakeAfterAnUnprovenSeedProvesItBeforeTheDBIsWarm(t *testing.T) {
+ const host = "obligation.example.com"
+ b, _ := delegatedBackend(t, host)
+ runs := filepath.Join(t.TempDir(), "runs")
+ stageSeedScript(t, b, host, "v1", fmt.Sprintf(
+ "echo run >> %q\nprintf half > \"$3\"\n[ \"$(wc -l < %q)\" -ge 2 ] && exit 0\nexit 1\n", runs, runs))
+ swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) {
+ return false, nil
+ })
+ rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}
+ db := filepath.Join(b.DataDir, host+".db")
+
+ if err := b.Seed(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) {
+ t.Fatalf("first Seed = %v, want ErrSeedUnproven", err)
+ }
+ for _, p := range []string{db + "-wal", db + "-shm"} {
+ if err := os.WriteFile(p, []byte("x"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // The proof still fails: refused, and the database is not warm.
+ fresh, err := b.Restore(context.Background(), rt)
+ if !errors.Is(err, ErrSeedUnproven) || fresh {
+ t.Fatalf("Restore = %v, %v; want a refusal with ErrSeedUnproven", fresh, err)
+ }
+ if got, _ := os.ReadFile(db); string(got) != "half" {
+ t.Fatalf("database = %q after a refused wake, want it untouched", got)
+ }
+ if _, err := os.Stat(db + seedUnprovenSuffix); err != nil {
+ t.Fatalf("the obligation went with a refused wake: %v", err)
+ }
+
+ // The proof passes: everything of the failed seed goes, and it is fresh.
+ var asked []string
+ b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
+ asked = append(asked, child)
+ return nil
+ }
+ fresh, err = b.Restore(context.Background(), rt)
+ if err != nil || !fresh {
+ t.Fatalf("Restore = %v, %v; want fresh once the proof holds", fresh, err)
+ }
+ if len(asked) != 1 || asked[0] != "" {
+ t.Fatalf("proof asked of %q, want the host cgroup once", asked)
+ }
+ for _, p := range []string{db, db + "-wal", db + "-shm", db + seedUnprovenSuffix} {
+ if _, err := os.Stat(p); !os.IsNotExist(err) {
+ t.Fatalf("%s still there (%v) after the proof held", filepath.Base(p), err)
+ }
+ }
+ if err := b.Seed(context.Background(), rt); err != nil {
+ t.Fatalf("second Seed: %v", err)
+ }
+ got, err := os.ReadFile(runs)
+ if err != nil || strings.Count(string(got), "run\n") != 2 {
+ t.Fatalf("seed runs = %q, %v; want the seed to have run twice", got, err)
+ }
+}
+
+// Task 8, plan 6: flag on, a successful seed's role child is killed and
+// proven empty before the seed returns, so a helper that redirected its
+// output (Wait returns a plain nil) does not outlive it.
+func TestAFlagOnSeedKillsAHelperThatOutlivesIt(t *testing.T) {
+ const host = "helper-seed.example.com"
+ b, _ := delegatedBackend(t, host)
+ cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a role child is never moved by PID
+ b.GenContainment = true
+ pidFile := filepath.Join(t.TempDir(), "helper.pid")
+ stageSeedScript(t, b, host, "v1", fmt.Sprintf(
+ "sleep 30 >/dev/null 2>&1 &\necho $! > %q\nprintf seeded > \"$3\"\nexit 0\n", pidFile))
+ helper := func() int {
+ raw, err := os.ReadFile(pidFile)
+ if err != nil {
+ return 0
+ }
+ pid, _ := strconv.Atoi(strings.TrimSpace(string(raw)))
+ return pid
+ }
+ t.Cleanup(func() {
+ if pid := helper(); pid > 0 {
+ _ = syscall.Kill(pid, syscall.SIGKILL)
+ }
+ })
+ var asked []string
+ b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
+ asked = append(asked, child)
+ if pid := helper(); pid > 0 {
+ _ = syscall.Kill(pid, syscall.SIGKILL) // what cgroup.kill does to the role child
+ }
+ return nil
+ }
+
+ if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err != nil {
+ t.Fatalf("Seed: %v", err)
+ }
+ if len(asked) != 1 || asked[0] != cgroup.RoleChild("seed") {
+ t.Fatalf("killed %q before the seed returned, want the seed's role child once", asked)
+ }
+ if got, err := os.ReadFile(filepath.Join(b.DataDir, host+".db")); err != nil || string(got) != "seeded" {
+ t.Fatalf("database = %q, %v; want the seed's work kept", got, err)
+ }
+}
+
+// Flag on, a successful seed whose role child cannot be proven empty is
+// unproven: the helper may still write the database, so the wake must not
+// go on to serve it, and the next wake must prove it first.
+func TestAFlagOnSeedWhoseHelperIsNotProvenGoneIsUnproven(t *testing.T) {
+ const host = "helper-unproven.example.com"
+ b, _ := delegatedBackend(t, host)
+ cgroup.FakePlacementForTest(t)
+ b.GenContainment = true
+ stageSeedScript(t, b, host, "v1", "printf seeded > \"$3\"\nexit 0\n")
+ b.cgroupKill = func(context.Context, registry.Route, string) error {
+ return errors.New("the role child still holds a process")
+ }
+
+ err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
+ if !errors.Is(err, ErrSeedUnproven) {
+ t.Fatalf("Seed = %v, want ErrSeedUnproven", err)
+ }
+ if _, err := os.Stat(filepath.Join(b.DataDir, host+".db"+seedUnprovenSuffix)); err != nil {
+ t.Fatalf("no obligation beside the database: %v", err)
+ }
+}
+
+// Flag on, a failed seed's role child is proven empty before the database
+// it built is removed.
+func TestAFlagOnFailedSeedProvesItsRoleChildBeforeRemovingTheDB(t *testing.T) {
+ const host = "failed-role-seed.example.com"
+ b, _ := delegatedBackend(t, host)
+ cgroup.FakePlacementForTest(t)
+ b.GenContainment = true
+ stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nexit 1\n")
+ db := filepath.Join(b.DataDir, host+".db")
+ var asked []string
+ b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error {
+ if _, err := os.Stat(db); err != nil {
+ t.Errorf("the database was gone (%v) before the seed's processes were proven gone", err)
+ }
+ asked = append(asked, child)
+ return nil
+ }
+
+ err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`})
+ if err == nil || errors.Is(err, ErrSeedUnproven) {
+ t.Fatalf("Seed = %v, want the seed's own failure", err)
+ }
+ if len(asked) != 1 || asked[0] != cgroup.RoleChild("seed") {
+ t.Fatalf("proved %q, want the seed's role child once", asked)
+ }
+ for _, p := range []string{db, db + seedUnprovenSuffix} {
+ if _, err := os.Stat(p); !os.IsNotExist(err) {
+ t.Fatalf("%s still there (%v) after a proven failed seed", filepath.Base(p), err)
+ }
+ }
+}