| index d4ea0c0a2..017ea974f 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -55,6 +55,10 @@ var ErrSeedUnproven = errors.New("activator: seed unproven; processes remain") |
| // ".db", so the box-wide litestream glob never picks it up. |
| const seedUnprovenSuffix = ".seed-unproven" |
| |
| +// writeSeedObligationFile is the obligation's durable write: a var so a |
| +// test can make it fail (a full disk). |
| +var writeSeedObligationFile = safefile.WriteFile |
| + |
| // quarantineSuffix names where a restore that failed its integrity check is |
| // parked. It deliberately does NOT end in ".db": the box-wide litestream |
| // generator globs <data>/*.db, and a quarantined corpse must never be picked |
| @@ -897,14 +901,23 @@ func seedArgv(rt registry.Route, dbPath string) ([]string, error) { |
| return append(seedArgs, "--db", dbPath), nil |
| } |
| |
| -// removeSeedDB deletes a failed seed's own half-built database and its |
| -// WAL/SHM siblings: the cleanup both spawn paths run when, and only when, |
| -// seedOwnsDB says the database is the seed's own work (seedCreatedTheDB). |
| -// Shared so the two paths cannot drift on which files "the database" means. |
| -func removeSeedDB(dbPath string) { |
| +// removeSeedDB deletes a seed's own half-built database and its WAL/SHM |
| +// siblings, proven out of reach of its processes: the cleanup both spawn |
| +// paths and the next wake's discharge run when, and only when, the |
| +// database is the seed's own work (seedCreatedTheDB). Shared so the three |
| +// cannot drift on which files "the database" means. A file already gone |
| +// is fine; any other failure is returned, and the directory is synced so |
| +// the removals are durable before the caller removes the obligation. |
| +func removeSeedDB(dbPath string) error { |
| for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { |
| - _ = os.Remove(p) |
| + if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) { |
| + return fmt.Errorf("remove the seed's %s: %w", p, err) |
| + } |
| + } |
| + if err := syncDir(filepath.Dir(dbPath)); err != nil { |
| + return fmt.Errorf("sync %s after removing the seed's database: %w", filepath.Dir(dbPath), err) |
| } |
| + return nil |
| } |
| |
| // seedChild is Seed's flat body. exe is the file the seed execs: the live |
| @@ -979,6 +992,14 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route, exe stri |
| if perr != nil { |
| return perr |
| } |
| + // The obligation goes on disk, durably, before the seed's first |
| + // instruction (Task 8 fix round 1): from here until its processes are |
| + // proven gone, nothing may treat the database as warm, whatever happens |
| + // to the seed, its disk or this agent. A write that fails refuses the |
| + // seed with nothing run. |
| + if err := recordSeedObligation(host, dbPath, seedObligation{ownsDB: seedOwnsDB, layout: LayoutFlat}); err != nil { |
| + return err |
| + } |
| cmd, settle, err := b.spawnInCgroupAtTier(rt, child, false, tier, func() *exec.Cmd { |
| cmd := exec.CommandContext(ctx, exe, argv...) |
| // The control-channel vars are agent-owned in BOTH directions here too |
| @@ -1054,20 +1075,38 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route, exe stri |
| pcancel() |
| if perr != nil { |
| // The database is kept, as seedUnitFailed keeps it: removing it |
| - // under a live writer is the hazard itself. The obligation beside |
| - // it keeps every later wake from serving it as warm until the |
| - // proof holds (Restore), and the caller leaves the lease to its |
| - // TTL (ErrSeedUnproven). |
| + // under a live writer is the hazard itself. The obligation |
| + // written ahead stays, so every later wake refuses it as warm |
| + // until the proof holds (Restore), and the caller leaves the |
| + // lease to its TTL (ErrSeedUnproven). |
| if failure == nil { |
| failure = fmt.Errorf("seed for %s: %w: processes remain after a successful seed: %v", host, ErrSeedUnproven, perr) |
| } else { |
| failure = fmt.Errorf("%w: %w (the seed's processes remain after stopping it; its database was left in place: %v)", failure, ErrSeedUnproven, perr) |
| } |
| - leaveSeedObligation(host, dbPath, seedOwnsDB, failure) |
| + log.Printf("activator: %s: the seed's processes are not proven gone; its database is kept and %s refuses it as warm until they are", host, dbPath+seedUnprovenSuffix) |
| return failure |
| } |
| b.Cgroups.RemoveChild(host, child) // the seed's own child only; no-op for "" (flag off) |
| } |
| + // From here the seed's processes are proven gone (or a spawn that |
| + // failed ran nothing, or a flag-off success's helper is the terminal |
| + // stop's to prove), so the obligation is discharged. A failure takes |
| + // the half-built database with it first. Failing the wake is only half |
| + // the promise: the file a crashed seed left behind would look like a |
| + // warm cache to the NEXT wake (DB present, so not fresh, so no seed) and |
| + // the instance would boot on it, which is the half-seeded instance this |
| + // whole path exists to prevent. Only a database that wasn't there when |
| + // we started is ours to remove, and under the lease, with the seed's |
| + // processes proven gone above, nothing else can have written it. A |
| + // discharge that fails keeps the obligation and fails the wake while |
| + // nothing serves; the next wake proves and removes again. |
| + if derr := dischargeSeedObligation(dbPath, failure != nil && seedOwnsDB); derr != nil { |
| + if failure == nil { |
| + return fmt.Errorf("seed for %s: %w", host, derr) |
| + } |
| + return fmt.Errorf("%w; and %w", failure, derr) |
| + } |
| if failure == nil { |
| if err != nil { |
| log.Printf("activator: seeded %s (it exited 0 but left a child holding its output pipe open for more than %s)", host, seedDrainGrace) |
| @@ -1076,17 +1115,6 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route, exe stri |
| } |
| return nil |
| } |
| - // Take the half-built database with the failure. Failing the wake is |
| - // only half the promise: the file a crashed seed left behind would |
| - // look like a warm cache to the NEXT wake — DB present, so not fresh, |
| - // so no seed — and the instance would boot on it, which is the |
| - // half-seeded instance this whole path exists to prevent. Only a |
| - // database that wasn't there when we started is ours to remove, and |
| - // under the lease, with the seed's processes proven gone above, nothing |
| - // else can have written it. |
| - if seedOwnsDB { |
| - removeSeedDB(dbPath) |
| - } |
| return failure |
| } |
| |
| @@ -1115,73 +1143,199 @@ func (b *ExecBackend) seedDescendantsGone(ctx context.Context, rt registry.Route |
| return err |
| } |
| |
| -// leaveSeedObligation records, durably and beside the database, that a |
| -// seed's processes were not proven gone (<db>.seed-unproven): Restore |
| -// refuses to treat the database as warm while it is there. Only a database |
| -// the seed created is the obligation's to remove later, so with seedOwnsDB |
| -// false (a database that was there before the seed) nothing is recorded. |
| -// A failed write is logged: the wake already fails, and the lease is left |
| -// to its TTL. |
| -func leaveSeedObligation(host, dbPath string, seedOwnsDB bool, cause error) { |
| - if !seedOwnsDB { |
| - log.Printf("activator: %s: the seed's processes are not proven gone; %s was there before the seed, so it is kept and no obligation is recorded", host, dbPath) |
| - return |
| +// seedObligation is what <db>.seed-unproven records, written ahead of a |
| +// seed (Task 8 fix round 1): before the seed's first instruction runs, so |
| +// a seed that dies of a full disk, or an agent that dies mid-seed, cannot |
| +// leave a partial database with nothing to refuse it. It is removed only |
| +// once the seed's processes are proven gone (dischargeSeedObligation). |
| +// |
| +// ownsDB is seedCreatedTheDB at the seed's start: only then is the |
| +// database the obligation's to remove. layout is where the seed ran: |
| +// LayoutNested is root's seed unit in the host slice, not |
| +// carlos-exec@<host>, so its proof is that unit's (nestedSeedGone); |
| +// LayoutFlat is an exec child; "" is an obligation that does not say. |
| +type seedObligation struct { |
| + ownsDB bool |
| + layout Layout |
| +} |
| + |
| +// The obligation's body: three lines, the first a version. Anything a |
| +// reader does not recognise reads as the cautious answer: a database that |
| +// was not the seed's (kept), and the layout the row reads now. |
| +const ( |
| + seedObligationHeader = "carlos seed obligation v1" |
| + seedObligationOwned = "db=owned-by-seed" |
| + seedObligationKept = "db=there-before-seed" |
| + seedObligationFlat = "seed=flat" |
| + seedObligationNested = "seed=nested" |
| +) |
| + |
| +func (o seedObligation) body() []byte { |
| + db, seed := seedObligationKept, seedObligationFlat |
| + if o.ownsDB { |
| + db = seedObligationOwned |
| } |
| + if o.layout == LayoutNested { |
| + seed = seedObligationNested |
| + } |
| + return []byte(seedObligationHeader + "\n" + db + "\n" + seed + "\n") |
| +} |
| + |
| +// recordSeedObligation writes the obligation ahead of a seed, durably |
| +// (safefile: file fsync, rename, dir fsync). One that is already there is |
| +// an earlier seed not yet answered (Restore answers it before any seed of |
| +// a wake): it is never overwritten, since that could lose what it says the |
| +// earlier seed owned, and the seed is refused with ErrSeedUnproven. A write |
| +// that fails refuses the seed as an ordinary error: nothing ran. |
| +func recordSeedObligation(host, dbPath string, o seedObligation) error { |
| ob := dbPath + seedUnprovenSuffix |
| - if err := safefile.WriteFile(ob, []byte(cause.Error()+"\n"), 0o600); err != nil { |
| - log.Printf("activator: %s: COULD NOT RECORD the unproven seed at %s: %v; the next wake may treat its database as warm", host, ob, err) |
| - return |
| + if _, err := os.Lstat(ob); err == nil { |
| + return fmt.Errorf("%s: refusing to seed: %w: an earlier seed's obligation %s is not answered", host, ErrSeedUnproven, ob) |
| + } else if !errors.Is(err, fs.ErrNotExist) { |
| + return fmt.Errorf("%s: refusing to seed: read the seed obligation %s: %w", host, ob, err) |
| + } |
| + if err := writeSeedObligationFile(ob, o.body(), 0o600); err != nil { |
| + return fmt.Errorf("%s: refusing to seed: record its obligation %s: %w", host, ob, err) |
| } |
| - log.Printf("activator: %s: the seed's processes are not proven gone; its database is kept and %s refuses it as warm until they are", host, ob) |
| + return nil |
| } |
| |
| -// settleSeedObligation answers an earlier seed's obligation |
| -// (<db>.seed-unproven, leaveSeedObligation) before anything treats dbPath |
| -// as warm. With none there it does nothing. Otherwise the host's processes |
| -// are proven gone first, and only then are the database, its WAL and SHM |
| -// and, last, the obligation removed, so the wake goes on as a first boot |
| -// and the seed runs again. A proof that fails refuses the wake with |
| -// ErrSeedUnproven and removes nothing. The obligation is a file, so it |
| -// survives an agent restart. |
| +// readSeedObligation reads dbPath's obligation. present is false only on a |
| +// definite "not there"; any other failure to look is an error, never "no |
| +// obligation". |
| // |
| -// The proof is on carlos-exec@<host>, whatever the layout now: it holds |
| -// the seed's role child as well, so it covers the seed whichever way it |
| -// ran, including across a CARLOS_SWAP change between the two wakes. It is |
| -// safe to kill there: this runs under the wake's lease, before anything of |
| -// this wake has run, and no other entry of the host is live (an unproven |
| -// failed wake or stray sidecar refuses the wake before the lease). |
| +// The name sits beside the database, on ground the instance itself can |
| +// write, so it is opened as the env files are (openOwnEnvNoFollow): a |
| +// symlink or a FIFO planted there is refused in the open itself, never |
| +// followed or waited on. Such an entry was not written by a seed, so it |
| +// reads as an obligation that does not say (the zero value): it is proven |
| +// and removed like any other, and the database beside it is kept, which |
| +// neither hangs the wake nor holds the row down. |
| +func readSeedObligation(dbPath string) (o seedObligation, present bool, err error) { |
| + f, err := openOwnEnvNoFollow(dbPath+seedUnprovenSuffix, os.O_RDONLY, 0) |
| + if errors.Is(err, fs.ErrNotExist) { |
| + return seedObligation{}, false, nil |
| + } |
| + if errors.Is(err, errOwnEnvSymlink) || errors.Is(err, errOwnEnvNotRegular) { |
| + return seedObligation{}, true, nil |
| + } |
| + if err != nil { |
| + return seedObligation{}, true, err |
| + } |
| + raw, err := io.ReadAll(io.LimitReader(f, 4<<10)) |
| + f.Close() |
| + if err != nil { |
| + return seedObligation{}, true, err |
| + } |
| + lines := strings.Split(string(raw), "\n") |
| + if len(lines) >= 3 && lines[0] == seedObligationHeader { |
| + o.ownsDB = lines[1] == seedObligationOwned |
| + switch lines[2] { |
| + case seedObligationNested: |
| + o.layout = LayoutNested |
| + case seedObligationFlat: |
| + o.layout = LayoutFlat |
| + } |
| + } |
| + return o, true, nil |
| +} |
| + |
| +// dischargeSeedObligation runs once a seed's processes are proven gone: |
| +// the database files go first when removeDB (removeSeedDB, every error |
| +// checked and the directory synced), and only then the obligation, synced |
| +// too, so a crash between the two leaves the obligation, never a database |
| +// without it. Any failure keeps the obligation, and the next wake proves |
| +// and removes again. |
| +func dischargeSeedObligation(dbPath string, removeDB bool) error { |
| + if removeDB { |
| + if err := removeSeedDB(dbPath); err != nil { |
| + return err |
| + } |
| + } |
| + ob := dbPath + seedUnprovenSuffix |
| + if err := os.Remove(ob); err != nil && !errors.Is(err, fs.ErrNotExist) { |
| + return fmt.Errorf("remove the seed obligation %s: %w", ob, err) |
| + } |
| + if err := syncDir(filepath.Dir(dbPath)); err != nil { |
| + return fmt.Errorf("sync %s after removing the seed obligation: %w", filepath.Dir(dbPath), err) |
| + } |
| + return nil |
| +} |
| + |
| +// settleSeedObligation answers an earlier seed's obligation before anything |
| +// treats dbPath as warm. With none there it does nothing. Otherwise the |
| +// seed's processes are proven gone first (seedObligationProof). A proof |
| +// that fails refuses the wake with ErrSeedUnproven and removes nothing. |
| +// One that holds discharges the obligation: a database the seed owned goes |
| +// with it, so the wake goes on as a first boot and the seed runs again; a |
| +// database that was there before the seed stays, and is warm. The |
| +// obligation is a file, so it survives an agent restart. |
| func (b *ExecBackend) settleSeedObligation(ctx context.Context, rt registry.Route, dbPath string) error { |
| host := rt.Host |
| - ob := dbPath + seedUnprovenSuffix |
| - if _, err := os.Lstat(ob); errors.Is(err, fs.ErrNotExist) { |
| + o, present, err := readSeedObligation(dbPath) |
| + if !present { |
| return nil |
| - } else if err != nil { |
| - // "Couldn't tell" is never "no obligation". |
| - return fmt.Errorf("%s: read the seed obligation %s: %w", host, ob, err) |
| } |
| - pctx, cancel := context.WithTimeout(ctx, actStopTimeout) |
| - err := b.seedDescendantsGone(pctx, rt, "") |
| - cancel() |
| if err != nil { |
| - return fmt.Errorf("%s: %w: an earlier seed's processes are not proven gone, so its database is not used: %v", host, ErrSeedUnproven, err) |
| + // "Couldn't tell" is never "no obligation". |
| + return fmt.Errorf("%s: read the seed obligation %s: %w", host, dbPath+seedUnprovenSuffix, err) |
| } |
| - for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { |
| - if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) { |
| - return fmt.Errorf("%s: remove %s after an unproven seed: %w", host, p, err) |
| - } |
| + if err := b.seedObligationProof(ctx, rt, o); err != nil { |
| + return fmt.Errorf("%s: %w: an earlier seed's processes are not proven gone, so its database is not used: %v", host, ErrSeedUnproven, err) |
| } |
| - // The removals are durable before the obligation goes, so a crash in |
| - // between can only leave the obligation, never the database without it. |
| - if err := syncDir(filepath.Dir(dbPath)); err != nil { |
| - return fmt.Errorf("%s: sync %s after an unproven seed: %w", host, filepath.Dir(dbPath), err) |
| + if err := dischargeSeedObligation(dbPath, o.ownsDB); err != nil { |
| + return fmt.Errorf("%s: after an earlier seed proven gone: %w", host, err) |
| } |
| - if err := os.Remove(ob); err != nil && !errors.Is(err, fs.ErrNotExist) { |
| - return fmt.Errorf("%s: remove the seed obligation %s: %w", host, ob, err) |
| + if o.ownsDB { |
| + log.Printf("activator: %s: the earlier seed's processes are proven gone; its database is removed and the seed runs again", host) |
| + } else { |
| + log.Printf("activator: %s: the earlier seed's processes are proven gone; the database that was there before it is kept", host) |
| } |
| - log.Printf("activator: %s: the earlier seed's processes are proven gone; its database is removed and the seed runs again", host) |
| return nil |
| } |
| |
| +// seedObligationProof proves an obligation's seed gone, by where it ran. A |
| +// nested seed ran as root's seed unit in the host slice: that unit is |
| +// stopped and its cgroup proven empty (nestedSeedGone). A flat seed ran in |
| +// carlos-exec@<host>, flag off in it and flag on in its role child, so the |
| +// host cgroup is proven, whatever CARLOS_SWAP says now: it holds the role |
| +// child too, so it covers the seed whichever way it ran. An obligation that |
| +// does not say (an unknown body) is proven by the layout the row reads now. |
| +// |
| +// Killing there is safe: this runs under the wake's lease, before anything |
| +// of this wake has run, and no other entry of the host is live (an unproven |
| +// failed wake or stray sidecar refuses the wake before the lease). |
| +func (b *ExecBackend) seedObligationProof(ctx context.Context, rt registry.Route, o seedObligation) error { |
| + layout := o.layout |
| + if layout == "" { |
| + layout = b.pathsFor(rt).layout |
| + } |
| + if layout == LayoutNested { |
| + return b.nestedSeedGone(ctx, rt) |
| + } |
| + pctx, cancel := context.WithTimeout(ctx, actStopTimeout) |
| + defer cancel() |
| + return b.seedDescendantsGone(pctx, rt, "") |
| +} |
| + |
| +// nestedSeedGone stops root's seed unit for rt and proves its cgroup empty |
| +// (waitSeedCgroupEmpty), each bounded and detached from ctx's own |
| +// cancellation, which may be why the seed failed. A failed stop is logged: |
| +// the empty cgroup is the proof. |
| +func (b *ExecBackend) nestedSeedGone(ctx context.Context, rt registry.Route) error { |
| + if b.Control == nil { |
| + return fmt.Errorf("%s: instance control channel not configured", rt.Host) |
| + } |
| + sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), instancectl.DeadlineControl) |
| + if err := b.Control.Stop(sctx, rt.Host, instancectl.RoleSeed); err != nil { |
| + log.Printf("activator: %s: stop seed unit: %v", rt.Host, err) |
| + } |
| + cancel() |
| + wctx, wcancel := context.WithTimeout(context.WithoutCancel(ctx), actStopTimeout) |
| + defer wcancel() |
| + return b.waitSeedCgroupEmpty(wctx, rt) |
| +} |
| + |
| // seedCreatedTheDB decides, from the stat taken just BEFORE a seed ran, |
| // whether the database at that path can only be the seed's own work — the |
| // single condition under which a failed seed's cleanup may delete it. |
| @@ -1298,6 +1452,11 @@ func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route, exe *inst |
| // unchanged for the unit path. |
| _, statErr := os.Stat(dbPath) |
| seedOwnsDB := seedCreatedTheDB(statErr) |
| + // The obligation is written ahead, as seedChild's is, before root is |
| + // asked to run anything; it names the seed unit as the proof to ask. |
| + if err := recordSeedObligation(host, dbPath, seedObligation{ownsDB: seedOwnsDB, layout: LayoutNested}); err != nil { |
| + return err |
| + } |
| |
| log.Printf("activator: seeding %s: %s", host, strings.Join(argv, " ")) |
| // The client already bounds a seed call at instancectl.DeadlineSeed. |
| @@ -1317,8 +1476,12 @@ func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route, exe *inst |
| if errors.Is(err, instancectl.ErrStagedExe) { |
| // Root refused the recorded build before it laid out or started |
| // anything (403), so no seed ran: nothing to stop, and the |
| - // database is not the seed's work. |
| - return fmt.Errorf("seed for %s: %w", host, err) |
| + // database is not the seed's work. Its obligation is discharged. |
| + cause := fmt.Errorf("seed for %s: %w", host, err) |
| + if derr := dischargeSeedObligation(dbPath, false); derr != nil { |
| + return fmt.Errorf("%w; and %w", cause, derr) |
| + } |
| + return cause |
| } |
| if err != nil { |
| return b.seedUnitFailed(ctx, rt, dbPath, seedOwnsDB, fmt.Errorf("seed for %s: %w", host, err)) |
| @@ -1329,13 +1492,17 @@ func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route, exe *inst |
| // A successful seed is not done until its cgroup is empty (design |
| // §6.2): the control service already waited before answering, but the |
| // activator's own proof is what lets the wake go on to Start with no |
| - // seed process still holding the database. |
| + // seed process still holding the database. Unproven, the obligation |
| + // written ahead stays. |
| wctx, wcancel := context.WithTimeout(context.WithoutCancel(ctx), actStopTimeout) |
| emptyErr := b.waitSeedCgroupEmpty(wctx, rt) |
| wcancel() |
| if emptyErr != nil { |
| return fmt.Errorf("seed for %s: %w: processes remain after a successful seed: %v", host, ErrSeedUnproven, emptyErr) |
| } |
| + if derr := dischargeSeedObligation(dbPath, false); derr != nil { |
| + return fmt.Errorf("seed for %s: %w", host, derr) |
| + } |
| log.Printf("activator: seeded %s", host) |
| return nil |
| } |
| @@ -1343,31 +1510,21 @@ func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route, exe *inst |
| // seedUnitFailed is seedUnit's shared failure tail: on EITHER failure |
| // shape, whether Control.Seed itself errored or it answered with a |
| // non-success result, the seed unit is stopped and its cgroup is proven |
| -// empty BEFORE any database cleanup runs, never after. Running the cleanup |
| -// on the bare result instead could race a seed process SIGKILL hadn't |
| -// actually reaped yet, exactly the class of bug Stop's own ErrStillRunning |
| -// discipline exists to rule out elsewhere. |
| -// removeSeedDB runs only when seedOwnsDB AND the wait proved the cgroup |
| -// empty; when it did not, nothing is removed and the returned error says |
| -// so, because a database is not safe to delete out from under a process |
| -// that might still be writing it. |
| +// empty (nestedSeedGone) BEFORE any database cleanup runs, never after. |
| +// Running the cleanup on the bare result instead could race a seed process |
| +// SIGKILL hadn't actually reaped yet, exactly the class of bug Stop's own |
| +// ErrStillRunning discipline exists to rule out elsewhere. |
| +// The database goes only when seedOwnsDB AND the proof held, and then the |
| +// obligation (dischargeSeedObligation); when the proof failed, nothing is |
| +// removed, the obligation stays and the returned error says so, because a |
| +// database is not safe to delete out from under a process that might still |
| +// be writing it. |
| func (b *ExecBackend) seedUnitFailed(ctx context.Context, rt registry.Route, dbPath string, seedOwnsDB bool, cause error) error { |
| - host := rt.Host |
| - // Detached from ctx's own cancellation (it may already be why the seed |
| - // failed) but still bounded, exactly like Stop's own stop call. |
| - sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), instancectl.DeadlineControl) |
| - if err := b.Control.Stop(sctx, host, instancectl.RoleSeed); err != nil { |
| - log.Printf("activator: %s: stop seed unit after failure: %v", host, err) |
| - } |
| - cancel() |
| - wctx, wcancel := context.WithTimeout(context.WithoutCancel(ctx), actStopTimeout) |
| - emptyErr := b.waitSeedCgroupEmpty(wctx, rt) |
| - wcancel() |
| - if emptyErr != nil { |
| + if emptyErr := b.nestedSeedGone(ctx, rt); emptyErr != nil { |
| return fmt.Errorf("%w: %w (the seed's processes remain after stopping it; its database was left in place: %v)", cause, ErrSeedUnproven, emptyErr) |
| } |
| - if seedOwnsDB { |
| - removeSeedDB(dbPath) |
| + if derr := dischargeSeedObligation(dbPath, seedOwnsDB); derr != nil { |
| + return fmt.Errorf("%w; and %w", cause, derr) |
| } |
| return cause |
| } |
amadan