amadan
zdd-readiness: F6 proves the host cgroup empty for a retained flag-off wake; /run exe leftovers are swept
Task 15. Astra IMPL-3 F6: a retained failed wake whose generation has no child of its own (KillGen answers errNoGenContainment, no generation kill, or a flag-off worker with no number) is proven stopped only once KillHost empties carlos-exec@<host>; nothing of another entry can be there while the wake is retained. A box with no delegated cgroup keeps the leader's exit as its proof. IMPL-2-R2 residual 4: sweepGenLeftovers also walks ExeRoot for <iid>@<n> directories with no drop-in and removes them under genMu with the same inactive, empty and not-starting checks; host directories are never touched.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
6 files changed,
+403
−24
internal/activator/opset.go+84 −13internal/activator/opset_test.go+149 −0internal/instancectl/gen.go+57 −10internal/instancectl/names.go+19 −1internal/instancectl/names_gen_test.go+23 −0internal/instancectl/server_gen_test.go+71 −0
diff --git a/internal/activator/opset.go b/internal/activator/opset.go| index d0fb79bf5..02fcb97aa 100644 |
| --- a/internal/activator/opset.go |
| +++ b/internal/activator/opset.go |
| @@ -244,8 +244,21 @@ func (a *Service) genTerminated(rt registry.Route, g Gen, p Proc) bool { |
| // errNoGenContainment, has no generation children: the generation ran in |
| // the host cgroup, and the terminal stop's own stop evidence on that cgroup |
| // is the proof (plan 2, "stop evidence on the parent"). Any other error is |
| -// an unproven termination. |
| +// an unproven termination. A retained failed wake has no such terminal stop |
| +// behind it, so failedWakeStopped asks for the host cgroup itself |
| +// (genChildProof's uncontained). |
| func (a *Service) genChildEmpty(rt registry.Route, m *genMember) bool { |
| + proven, _ := a.genChildProof(rt, m) |
| + return proven |
| +} |
| + |
| +// genChildProof is genChildEmpty's proof. uncontained reports that it |
| +// proved nothing beyond the leader's exit, because the generation has no |
| +// child of its own: the backend has no generation kill, KillGen answered |
| +// errNoGenContainment (the flag-off ExecBackend), or it has no number (a |
| +// flag-off worker). Such a generation ran in carlos-exec@<host>, where a |
| +// process its leader forked can outlive it (Task 15, Astra IMPL-3 F6). |
| +func (a *Service) genChildProof(rt registry.Route, m *genMember) (proven, uncontained bool) { |
| if cp, ok := m.proc.(*claimProcess); ok { |
| // A claims-on generation (Task 10): its own Stop is the proof, and |
| // only a Stop that completed (KillAndEmpty, the exit, the settle and |
| @@ -253,34 +266,78 @@ func (a *Service) genChildEmpty(rt registry.Route, m *genMember) bool { |
| // is not terminated. |
| ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| defer cancel() |
| - return cp.provenStopped(ctx) |
| + return cp.provenStopped(ctx), false |
| } |
| if m.gen.Worker { |
| // A worker runs in its host cgroup (plan 2 D4): that cgroup, not a |
| // g<n> it never ran in, is what must be empty (spec §7.4, §7.9). |
| hk, ok := a.backend.(hostKiller) |
| if !ok { |
| - return true // a backend with no host cgroup: the leader's exit is all there is |
| + return true, false // a backend with no host cgroup: the leader's exit is all there is |
| } |
| ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| defer cancel() |
| if err := hk.KillHost(ctx, rt); err != nil { |
| log.Printf("activator: %s: worker generation %d's host cgroup is not proven empty: %v", rt.Host, m.gen.N, err) |
| - return false |
| + return false, false |
| } |
| - return true |
| + return true, false |
| } |
| k, ok := a.backend.(genKiller) |
| if !ok || m.gen.N == 0 { |
| - return true |
| + return true, true |
| } |
| ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| defer cancel() |
| err := k.KillGen(ctx, rt, m.gen.N) |
| - if err == nil || errors.Is(err, errNoGenContainment) { |
| - return true |
| + if err == nil { |
| + return true, false |
| + } |
| + if errors.Is(err, errNoGenContainment) { |
| + return true, true |
| } |
| log.Printf("activator: %s: generation %d's child is not proven empty: %v", rt.Host, m.gen.N, err) |
| + return false, false |
| +} |
| + |
| +// uncontainedHostEmpty proves a retained failed wake's host cgroup empty |
| +// when its generation had no child of its own (Task 15, Astra IMPL-3 F6, |
| +// flag off): KillHost kills whatever is left in carlos-exec@<host> and |
| +// waits it empty, as the worker proof does. An absent cgroup is empty. |
| +// |
| +// Killing is safe here because nothing of another entry can be in that |
| +// cgroup while the wake is retained. The host's next wake, request or |
| +// claim, is refused (errWakeUnproven) until this proof holds; the failed |
| +// wake's own entry is gone, and it never reached the legacy sidecar's |
| +// co-spawn, which comes after every failure that retains a wake; sidecar |
| +// and seed starts run only under the host's live entry; workers and |
| +// sidecar members are rows of their own, each with its own |
| +// carlos-exec@<host>. What can be there is the wake's own leftovers, or a |
| +// stray sidecar of an earlier entry whose stop was not proven, which is |
| +// meant to be gone too. The killed cgroup is recreated by the next start |
| +// into it (Task 19). |
| +// |
| +// A box with no delegated cgroup (cgroup.ErrNoDelegation) ran the wake |
| +// unaccounted, with nothing that could reach a descendant: the leader's |
| +// exit is all there is, as before. Keeping the lease there would refuse |
| +// the host on this box until the agent restarts while proving nothing. |
| +// Any other failure keeps the wake retained, and the sweep asks again. |
| +func (a *Service) uncontainedHostEmpty(rt registry.Route, g Gen) bool { |
| + hk, ok := a.backend.(hostKiller) |
| + if !ok { |
| + return true // a backend with no host cgroup: the leader's exit is all there is |
| + } |
| + ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| + defer cancel() |
| + err := hk.KillHost(ctx, rt) |
| + if err == nil { |
| + return true |
| + } |
| + if errors.Is(err, cgroup.ErrNoDelegation) { |
| + log.Printf("activator: %s: failed wake's generation %d ran with no delegated cgroup; its leader's exit is the only proof there is", rt.Host, g.N) |
| + return true |
| + } |
| + log.Printf("activator: %s: failed wake's generation %d had no cgroup of its own, and its host cgroup is not proven empty: %v", rt.Host, g.N, err) |
| return false |
| } |
| @@ -550,15 +607,29 @@ func genStartUnproven(err error) bool { |
| // generation's own child, proven empty through KillGen, and never the |
| // missing handle: a retained wake with no number cannot be proven at all |
| // and stays retained. |
| +// |
| +// Either way, a generation with no child of its own (flag off) is proven |
| +// only once its host cgroup is empty too (uncontainedHostEmpty; Task 15, |
| +// Astra IMPL-3 F6). Its leader's exit, all genTerminated can see there, |
| +// leaves whatever the leader forked running in carlos-exec@<host>. |
| func (a *Service) failedWakeStopped(rt registry.Route, g Gen, p Proc) bool { |
| - if p != nil { |
| - return a.genTerminated(rt, g, p) |
| - } |
| - if g.N == 0 { |
| + if p == nil && g.N == 0 { |
| log.Printf("activator: %s: failed wake left no process handle and no generation number; it cannot be proven stopped, so its lease stays held", rt.Host) |
| return false |
| } |
| - return a.handlelessGenEmpty(rt, g) |
| + // genTerminated's proof (handlelessGenEmpty's without a process), |
| + // spelled out so the proof can say whether it reached a child. |
| + if p != nil && !p.Exited() { |
| + return false |
| + } |
| + proven, uncontained := a.genChildProof(rt, &genMember{gen: g, proc: p}) |
| + if !proven { |
| + return false |
| + } |
| + if uncontained { |
| + return a.uncontainedHostEmpty(rt, g) |
| + } |
| + return true |
| } |
| // reapUnprovenWakesLocked retries each unproven failed wake's proof and, |
diff --git a/internal/activator/opset_test.go b/internal/activator/opset_test.go| index c4cebf1e7..7a2b76750 100644 |
| --- a/internal/activator/opset_test.go |
| +++ b/internal/activator/opset_test.go |
| @@ -14,6 +14,7 @@ import ( |
| "testing" |
| "time" |
| + "github.com/carlosframework/platform/internal/cgroup" |
| "github.com/carlosframework/platform/internal/instancectl" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -1113,3 +1114,151 @@ func TestSwapWitnessAFencedWakeKeepsItsLeaseWhileItsProcessRuns(t *testing.T) { |
| }) |
| } |
| } |
| + |
| +// uncontainedStartBackend is the flag-off exec backend's shape for a |
| +// retained failed wake (Task 15, Astra IMPL-3 F6): KillGen answers |
| +// errNoGenContainment, as ExecBackend's does with GenContainment off, and |
| +// KillHost proves carlos-exec@<host> empty, failing while populated. |
| +type uncontainedStartBackend struct { |
| + *unprovenStartBackend |
| + hmu sync.Mutex |
| + populated bool |
| + err error // when set, every KillHost answers it |
| + hostKills int |
| +} |
| + |
| +func (h *uncontainedStartBackend) KillHost(context.Context, registry.Route) error { |
| + h.hmu.Lock() |
| + defer h.hmu.Unlock() |
| + h.hostKills++ |
| + if h.err != nil { |
| + return h.err |
| + } |
| + if h.populated { |
| + return errors.New("carlos-exec@<host> still holds a process") |
| + } |
| + return nil |
| +} |
| + |
| +func (h *uncontainedStartBackend) setPopulated(on bool) { |
| + h.hmu.Lock() |
| + h.populated = on |
| + h.hmu.Unlock() |
| +} |
| + |
| +func (h *uncontainedStartBackend) hostKillCount() int { |
| + h.hmu.Lock() |
| + defer h.hmu.Unlock() |
| + return h.hostKills |
| +} |
| + |
| +var _ hostKiller = (*uncontainedStartBackend)(nil) |
| + |
| +// Task 15, Astra IMPL-3 F6: with the flag off a retained failed wake was |
| +// proven stopped by its leader's exit alone. KillGen answers |
| +// errNoGenContainment (there is no g<n>), and genChildEmpty took that as |
| +// proof, so the lease went back while a process the leader forked could |
| +// still run in carlos-exec@<host>. The proof now also asks KillHost to |
| +// empty the host cgroup: while it cannot, the wake stays retained with its |
| +// lease, and once it can the lease goes back. |
| +func TestARetainedFlagOffWakeProvesItsHostCgroupEmpty(t *testing.T) { |
| + a, b, _, _ := testActivator(t, time.Hour) |
| + a.gens = false |
| + leader := &sigtermIgnorer{} |
| + kb := &killGenBackend{fakeBackend: b, err: errNoGenContainment} |
| + ub := &unprovenStartBackend{killGenBackend: kb, fail: func(uint64) (Proc, error) { |
| + return leader, errors.New("health: no answer within 30s") |
| + }} |
| + hb := &uncontainedStartBackend{unprovenStartBackend: ub, populated: true} |
| + a.backend = hb |
| + host := "forked.example.com" |
| + rt := testRoute(host) |
| + retained := func() bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + return a.unprovenWakes[host] != nil |
| + } |
| + settled := func() bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + u := a.unprovenWakes[host] |
| + return u == nil || !u.busy |
| + } |
| + if _, err := a.beginRequest(context.Background(), rt); err == nil { |
| + t.Fatal("the wake should have failed") |
| + } |
| + if !retained() || b.releaseCount() != 0 { |
| + t.Fatalf("retained %v, lease releases %d: want the wake whose leader lives retained with its lease", retained(), b.releaseCount()) |
| + } |
| + |
| + // The leader exits, but a descendant it forked is still in the host |
| + // cgroup: the lease stays and the wake stays retained. |
| + leader.exited.Store(true) |
| + a.sweepOnce() |
| + waitFor(t, "the retained wake's retry", settled) |
| + if got := b.releaseCount(); got != 0 || !retained() { |
| + t.Fatalf("lease releases = %d with the leader gone and carlos-exec@<host> populated, want 0 and the wake retained", got) |
| + } |
| + if hb.hostKillCount() == 0 { |
| + t.Fatal("the proof never asked KillHost about the host cgroup") |
| + } |
| + if k := kb.killed(); !slices.Contains(k, ub.firstGen()) { |
| + t.Fatalf("KillGen calls = %v, want generation %d's (answered errNoGenContainment)", k, ub.firstGen()) |
| + } |
| + if _, err := a.beginRequest(context.Background(), rt); !errors.Is(err, errWakeUnproven) { |
| + t.Fatalf("second wake: err = %v, want errWakeUnproven while the host cgroup is unproven", err) |
| + } |
| + |
| + // The host cgroup empties: the next sweep proves the wake stopped. |
| + hb.setPopulated(false) |
| + a.sweepOnce() |
| + waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 }) |
| + waitFor(t, "the retained wake dropped", func() bool { return !retained() }) |
| +} |
| + |
| +// The same proof for a flag-off worker's failed wake: it is never numbered |
| +// with generations off (Gen{}), so genChildEmpty's "no number, nothing to |
| +// prove" answered for it. A worker runs in carlos-exec@<host> as well, so |
| +// its leader's exit is not enough either. A backend with no host kill and |
| +// a contained generation are as before, and so is a box with no delegated |
| +// cgroup, where the wake ran unaccounted and nothing could be proven. |
| +func TestFailedWakeStoppedProvesTheHostCgroupWithoutContainment(t *testing.T) { |
| + exited := &sigtermIgnorer{} |
| + exited.exited.Store(true) |
| + for _, row := range []struct { |
| + name string |
| + gen Gen |
| + killGen error // nil: a contained generation whose child KillGen proved empty |
| + hostKill bool // the backend has KillHost |
| + populated bool |
| + hostErr error // what KillHost answers, when set |
| + want bool |
| + wantHost bool // KillHost asked |
| + }{ |
| + {"uncontained instance, host populated", Gen{N: 4}, errNoGenContainment, true, true, nil, false, true}, |
| + {"uncontained instance, host empty", Gen{N: 4}, errNoGenContainment, true, false, nil, true, true}, |
| + {"flag-off worker, host populated", Gen{}, errNoGenContainment, true, true, nil, false, true}, |
| + {"flag-off worker, host empty", Gen{}, errNoGenContainment, true, false, nil, true, true}, |
| + {"contained generation", Gen{N: 4}, nil, true, true, nil, true, false}, |
| + {"no host kill", Gen{N: 4}, errNoGenContainment, false, false, nil, true, false}, |
| + {"no delegated cgroup", Gen{N: 4}, errNoGenContainment, true, false, cgroup.ErrNoDelegation, true, true}, |
| + } { |
| + t.Run(row.name, func(t *testing.T) { |
| + a, b, _, _ := testActivator(t, time.Hour) |
| + kb := &killGenBackend{fakeBackend: b, err: row.killGen} |
| + ub := &unprovenStartBackend{killGenBackend: kb} |
| + hb := &uncontainedStartBackend{unprovenStartBackend: ub, populated: row.populated, err: row.hostErr} |
| + if row.hostKill { |
| + a.backend = hb |
| + } else { |
| + a.backend = ub |
| + } |
| + if got := a.failedWakeStopped(testRoute("f6.example.com"), row.gen, exited); got != row.want { |
| + t.Fatalf("failedWakeStopped = %v, want %v", got, row.want) |
| + } |
| + if asked := hb.hostKillCount() > 0; asked != row.wantHost { |
| + t.Fatalf("KillHost asked = %v, want %v", asked, row.wantHost) |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/instancectl/gen.go b/internal/instancectl/gen.go| index a870565f6..6be63042a 100644 |
| --- a/internal/instancectl/gen.go |
| +++ b/internal/instancectl/gen.go |
| @@ -256,18 +256,25 @@ func (s *Server) markGenStarting(unit string, on bool) { |
| // inactive or failed AND their cgroup is empty. No daemon-reload follows: |
| // stopGen removes files the same way, and a generation's name is never |
| // started again. |
| +// |
| +// Each generation executable directory under ExeRoot (<iid>@<n>) is |
| +// checked the same way, so a generation that kept only its exe layout is |
| +// found too: its drop-in went and the exe's removal failed, which |
| +// removeGenFiles ignores (Task 15, Astra IMPL-2-R2 residual 4). A host's |
| +// own directory (<iid>, no '@') is never a generation key and is never |
| +// touched. On a box that never ran generations, ExeRoot holds only host |
| +// directories and this finds nothing. |
| func (s *Server) sweepGenLeftovers(ctx context.Context, failed map[string]string) { |
| - if s.DropInDir == "" { |
| - return |
| - } |
| - entries, err := os.ReadDir(s.DropInDir) |
| - if err != nil { |
| - if !errors.Is(err, os.ErrNotExist) { |
| - s.logf("instance-control: sweep: %s: %v", s.DropInDir, err) |
| + var units []string |
| + seen := map[string]bool{} |
| + add := func(unit string) { |
| + if _, stuck := failed[unit]; stuck || seen[unit] { |
| + return |
| } |
| - return |
| + seen[unit] = true |
| + units = append(units, unit) |
| } |
| - for _, e := range entries { |
| + for _, e := range s.readSweepDir(s.DropInDir) { |
| unit, ok := strings.CutSuffix(e.Name(), ".d") |
| if !ok || !e.IsDir() { |
| continue |
| @@ -275,13 +282,53 @@ func (s *Server) sweepGenLeftovers(ctx context.Context, failed map[string]string |
| if _, _, ok := ParseGenUnitName(unit); !ok { |
| continue |
| } |
| - if _, stuck := failed[unit]; stuck { |
| + add(unit) |
| + } |
| + for _, e := range s.readSweepDir(s.ExeRoot) { |
| + if !e.IsDir() { |
| continue |
| } |
| + if unit, ok := genUnitOfExeKey(e.Name()); ok { |
| + add(unit) |
| + } |
| + } |
| + for _, unit := range units { |
| s.sweepGenLeftover(ctx, unit) |
| } |
| } |
| +// readSweepDir lists dir for sweepGenLeftovers. An unset or absent |
| +// directory has nothing to sweep; any other failure is logged. |
| +func (s *Server) readSweepDir(dir string) []os.DirEntry { |
| + if dir == "" { |
| + return nil |
| + } |
| + entries, err := os.ReadDir(dir) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + s.logf("instance-control: sweep: %s: %v", dir, err) |
| + } |
| + return entries |
| +} |
| + |
| +// genUnitOfExeKey names the generation unit whose executable directory is |
| +// key. It answers only when removeGenFiles, given that unit, removes |
| +// exactly key, so the sweep can never remove some other directory. |
| +func genUnitOfExeKey(key string) (string, bool) { |
| + iid, n, ok := ParseGenKey(key) |
| + if !ok { |
| + return "", false |
| + } |
| + unit := GenUnitNameOf(iid, n) |
| + esc, m, ok := ParseGenUnitName(unit) |
| + if !ok || m != n { |
| + return "", false |
| + } |
| + if back, ok := Unescape(esc); !ok || GenKeyOf(back, n) != key { |
| + return "", false |
| + } |
| + return unit, true |
| +} |
| + |
| func (s *Server) sweepGenLeftover(ctx context.Context, unit string) { |
| // Held across the check and the removal, so a start of the same unit |
| // cannot write fresh files in between. |
diff --git a/internal/instancectl/names.go b/internal/instancectl/names.go| index 323407d88..9b3722eb2 100644 |
| --- a/internal/instancectl/names.go |
| +++ b/internal/instancectl/names.go |
| @@ -127,7 +127,13 @@ func ParseGenInstance(s string) (iid string, n uint64, ok bool) { |
| // GenUnitName is carlos-instance-gen@<Escape(GenInstance(host, n))>.service. |
| func GenUnitName(host string, n uint64) string { |
| - return UnitPrefixGen + Escape(GenInstance(host, n)) + ".service" |
| + return GenUnitNameOf(provision.InstanceID(host), n) |
| +} |
| + |
| +// GenUnitNameOf is GenUnitName keyed by the instance ID, for a name read |
| +// back from a generation's files (ParseGenKey), where no host is known. |
| +func GenUnitNameOf(iid string, n uint64) string { |
| + return UnitPrefixGen + Escape(iid+":"+strconv.FormatUint(n, 10)) + ".service" |
| } |
| // ParseGenUnitName recognises a generation unit and returns its ESCAPED |
| @@ -152,6 +158,18 @@ func GenKeyOf(iid string, n uint64) string { return iid + "@" + strconv.FormatUi |
| func GenKey(host string, n uint64) string { return GenKeyOf(provision.InstanceID(host), n) } |
| +// ParseGenKey is GenKeyOf's inverse, under ParseGenInstance's rules: n is |
| +// canonical decimal and never 0, and iid is non-empty and holds no '@', |
| +// ':', '/' or NUL. A host's own directory under the exe root has no '@', |
| +// so it is never a generation key. |
| +func ParseGenKey(key string) (iid string, n uint64, ok bool) { |
| + i := strings.LastIndexByte(key, '@') |
| + if i <= 0 || strings.ContainsRune(key[:i], '@') { |
| + return "", 0, false |
| + } |
| + return ParseGenInstance(key[:i] + ":" + key[i+1:]) |
| +} |
| + |
| // GenLogSocket is the generation's run log socket: |
| // /run/carlos/logs/<GenKey>.run.sock. |
| func GenLogSocket(host string, n uint64) string { |
diff --git a/internal/instancectl/names_gen_test.go b/internal/instancectl/names_gen_test.go| index e968a1f36..3dd270c76 100644 |
| --- a/internal/instancectl/names_gen_test.go |
| +++ b/internal/instancectl/names_gen_test.go |
| @@ -121,3 +121,26 @@ func TestUnescapeInvertsEscape(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// ParseGenKey reads a generation's exe directory name back (Task 15's |
| +// sweep): it inverts GenKey, for a long host's hashed identifier too, and |
| +// refuses a host's own directory and every name GenKey never makes. |
| +func TestParseGenKeyInvertsGenKey(t *testing.T) { |
| + for _, host := range []string{"bill-2.example.com", strings.Repeat("a", 60) + ".example.com"} { |
| + iid, n, ok := ParseGenKey(GenKey(host, 7)) |
| + if !ok || iid != provision.InstanceID(host) || n != 7 { |
| + t.Errorf("ParseGenKey(GenKey(%q, 7)) = %q, %d, %v", host, iid, n, ok) |
| + } |
| + if got := GenUnitNameOf(iid, n); got != GenUnitName(host, 7) { |
| + t.Errorf("GenUnitNameOf = %q, want %q", got, GenUnitName(host, 7)) |
| + } |
| + if unit, ok := genUnitOfExeKey(GenKey(host, 7)); !ok || unit != GenUnitName(host, 7) { |
| + t.Errorf("genUnitOfExeKey(GenKey(%q, 7)) = %q, %v", host, unit, ok) |
| + } |
| + } |
| + for _, bad := range []string{"bill-2.example.com", "@7", "a@0", "a@07", "a@x", "a@", "a@b@7", "a:b@7", "a@7:1", ".@7", "..@7"} { |
| + if iid, n, ok := ParseGenKey(bad); ok { |
| + t.Errorf("ParseGenKey(%q) = %q, %d, true; want refused", bad, iid, n) |
| + } |
| + } |
| +} |
diff --git a/internal/instancectl/server_gen_test.go b/internal/instancectl/server_gen_test.go| index b0fd390cb..263b0cda6 100644 |
| --- a/internal/instancectl/server_gen_test.go |
| +++ b/internal/instancectl/server_gen_test.go |
| @@ -425,6 +425,77 @@ func TestSweepRemovesTheFilesOfAnInactiveGeneration(t *testing.T) { |
| } |
| } |
| +// Task 15 (Astra IMPL-2-R2 residual 4): a generation can leave only its |
| +// executable directory under ExeRoot, when its drop-in went and the exe's |
| +// removal failed (removeGenFiles ignores errors). The sweep listed only |
| +// DropInDir, so that leftover stayed in /run for good. It now walks |
| +// ExeRoot too, with the same checks: the unit inactive or failed, its |
| +// cgroup empty, and no start of it under way. A host's own directory (no |
| +// '@') and a name that is not a generation key are never touched. |
| +func TestSweepRemovesAnExeOnlyGenerationLeftover(t *testing.T) { |
| + ctl := genCtl() |
| + s, sock, dropins := genServer(t, ctl) |
| + if err := s.ensureGenExe(genHost, 7); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.ensureHostExe(genHost); err != nil { |
| + t.Fatal(err) |
| + } |
| + odd := []string{genHost + "@0", genHost + "@x", "@7", genHost + "@07"} |
| + for _, name := range odd { |
| + if err := os.Mkdir(filepath.Join(s.ExeRoot, name), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if _, err := NewClient(sock).Sweep(context.Background()); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !strings.Contains(ctl.joined(), "show -p ActiveState --value "+genUnit) { |
| + t.Errorf("the exe-only leftover's unit state was not checked:\n%s", ctl.joined()) |
| + } |
| + if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe { |
| + t.Errorf("an exe-only leftover of an inactive generation survived the sweep: drop-in %v exe %v", dropIn, exe) |
| + } |
| + if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err == nil { |
| + t.Error("the generation's exe directory is still there") |
| + } |
| + if _, err := os.Lstat(filepath.Join(s.ExeRoot, genHost, "app")); err != nil { |
| + t.Errorf("the host's own exe directory was touched: %v", err) |
| + } |
| + for _, name := range odd { |
| + if _, err := os.Stat(filepath.Join(s.ExeRoot, name)); err != nil { |
| + t.Errorf("%s, not a generation key, was removed: %v", name, err) |
| + } |
| + } |
| +} |
| + |
| +// An exe-only leftover is kept while its unit is not proven gone (active, |
| +// or its cgroup populated) and while a start of that unit is preparing it: |
| +// the start lays the exe out before it writes the drop-in, so for a moment |
| +// a live start looks exactly like an exe-only leftover. |
| +func TestSweepKeepsAnExeOnlyGenerationNotProvenGone(t *testing.T) { |
| + for name, setup := range map[string]func(*fakeCtl, *Server){ |
| + "active": func(c *fakeCtl, _ *Server) { c.answers["show -p ActiveState --value"] = "active\n" }, |
| + "populated": func(_ *fakeCtl, s *Server) { populateGenCgroup(t, s) }, |
| + "starting": func(_ *fakeCtl, s *Server) { s.markGenStarting(genUnit, true) }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + ctl := genCtl() |
| + s, sock, _ := genServer(t, ctl) |
| + setup(ctl, s) |
| + if err := s.ensureGenExe(genHost, 7); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := NewClient(sock).Sweep(context.Background()); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err != nil { |
| + t.Errorf("the exe of a generation not proven gone was removed: %v", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| // Leftovers whose unit is not proven gone stay: one systemd reports |
| // active, and one whose cgroup still holds processes. |
| func TestSweepKeepsTheFilesOfAGenerationNotProvenGone(t *testing.T) { |