amadan

carlos / platform Public

zdd plan 3 task 13: fix round 1: the sweep models unproven terminations; mutate.sh verdicts per run

The simulation now drives the retain-until-proven paths: leaders that
drain slowly, answer ErrStillRunning or a failed kill and exit later;
forked processes that outlive their leader until KillGen; a KillGen that
fails before it heals; starts that fail with errGenUnproven with or
without a handle; seeds with generations off and without containment; a
background sweeper and a re-stop cadence event. The sim backend is a
GenBackend and genKiller like the exec backend. I1-I3 are unchanged; the
tally now carries normalized abandonment reasons.

mutate.sh runs the sweep and each mutant's designated witnesses as
separate go test calls. CAUGHT needs a designated witness's assertion or
an I1-I3 violation; timeouts, panics, liveness-only failures and a sweep
that never ran get their own verdicts and fail the run. The two witness
predicates that dereferenced a missing entry are nil-guarded.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude c25878d98d26b7fc5cabccaf1f918f1c1fd951e3 parent 502eb11
3 files changed, +490 −99
  • internal/activator/swap_review_test.go +8 −2
  • internal/activator/swap_sim_test.go +354 −49
  • internal/activator/testdata/swapsim/mutate.sh +128 −48
diff --git a/internal/activator/swap_review_test.go b/internal/activator/swap_review_test.go
index d189ec2a8..98c74fc34 100644
--- a/internal/activator/swap_review_test.go
+++ b/internal/activator/swap_review_test.go
@@ -69,7 +69,10 @@ func TestSwapWitnessAgentRestartMidSwap(t *testing.T) {
waitFor(t, "Shutdown to reach the entry", func() bool {
f.a.mu.Lock()
defer f.a.mu.Unlock()
- return f.a.entries[f.host].state == actStopping
+ // A missing entry means the stop already finished: the checks
+ // below report that, rather than a nil dereference here.
+ e := f.a.entries[f.host]
+ return e == nil || e.state == actStopping
})
// The barrier holds the stop back while the spawn is in flight. As in
// the draining case nothing signals the barrier itself, so the
@@ -134,7 +137,10 @@ func TestSwapWitnessAgentRestartMidSwap(t *testing.T) {
waitFor(t, "Shutdown to reach the entry", func() bool {
f.a.mu.Lock()
defer f.a.mu.Unlock()
- return f.a.entries[f.host].state == actStopping
+ // A missing entry means the stop already finished: the checks
+ // below report that, rather than a nil dereference here.
+ e := f.a.entries[f.host]
+ return e == nil || e.state == actStopping
})
waitFor(t, "the retire of the incumbent", func() bool { return incumbent.stopEntered() >= 1 })
// The barrier holds Shutdown's own stop of the entry back until the
diff --git a/internal/activator/swap_sim_test.go b/internal/activator/swap_sim_test.go
index e3a89a27b..d9ead5be6 100644
--- a/internal/activator/swap_sim_test.go
+++ b/internal/activator/swap_sim_test.go
@@ -10,6 +10,8 @@ import (
"math/rand"
"os"
"path/filepath"
+ "regexp"
+ "sort"
"strconv"
"strings"
"sync"
@@ -23,9 +25,18 @@ import (
// TestSwapInvariants drives random schedules of requests, overlap and hold
// swaps, hibernation, lease loss, restarts, maintenance fences, root
-// repoints, supersession, crashes and a final Shutdown against the real
-// Service and a simulated backend. It checks I1, I2 and I3 (plan 3 task 13;
-// spec §12).
+// repoints, supersession, crashes, re-stop cadences and a final Shutdown
+// against the real Service and a simulated backend. It checks I1, I2 and I3
+// (plan 3 task 13; spec §12).
+//
+// The simulated processes do not all stop when asked. Some drain slowly,
+// some answer ErrStillRunning or a failed kill and exit on their own later,
+// some generations fork a process that outlives its leader, KillGen can
+// fail before it heals, and a start can fail with its kill unproven
+// (errGenUnproven), with or without a process handle. So the
+// retain-until-proven paths (pending stops, the member and wake reapers)
+// run under the sweep too. Each seed also picks whether generations are on
+// (the dormant switch) and whether generation containment is.
//
// Seeds: SWAP_SIM_SEEDS (default 100). SWAP_SIM_SEED=<n> replays one seed.
// Goroutine interleaving is not deterministic, so a replay is best effort.
@@ -65,7 +76,16 @@ func TestSwapInvariants(t *testing.T) {
failed := 0
tally := map[string]int{}
t.Cleanup(func() {
- t.Logf("swap outcomes over seeds %d..%d (mode/outcome, as Swap reported them): %v", first, last-1, tally)
+ keys := make([]string, 0, len(tally))
+ for k := range tally {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ lines := make([]string, len(keys))
+ for i, k := range keys {
+ lines[i] = fmt.Sprintf("%6d %s", tally[k], k)
+ }
+ t.Logf("seeds %d..%d: seed settings and Swap results (mode/outcome: reason, normalized):\n%s", first, last-1, strings.Join(lines, "\n"))
})
for seed := first; seed < last; seed++ {
violations, events, outcomes := simulateSwapOnce(seed, staged, sockDir, sink)
@@ -79,7 +99,15 @@ func TestSwapInvariants(t *testing.T) {
if len(events) > 200 {
events = events[len(events)-200:]
}
- t.Errorf("invariant violated at seed %d: %s\n--- last events ---\n%s", seed, strings.Join(violations, "; "), strings.Join(events, "\n"))
+ // A seed whose only failure is liveness (a Swap that never returned)
+ // says so: it is not an invariant violation.
+ kind := "liveness failed"
+ for _, v := range violations {
+ if !strings.HasPrefix(v, "liveness:") {
+ kind = "invariant violated"
+ }
+ }
+ t.Errorf("%s at seed %d (replay: SWAP_SIM_SEED=%d): %s\n--- last events ---\n%s", kind, seed, seed, strings.Join(violations, "; "), strings.Join(events, "\n"))
if failed >= 3 {
t.FailNow()
}
@@ -133,11 +161,21 @@ type simWorld struct {
violations []string
seen map[string]bool
nextID int
+ hosts []string
+ // gensOn is this seed's dormant switch (Service.gens). contained is
+ // generation containment: with it, each generation has its own child
+ // that KillGen empties, and a start can fail with its kill unproven;
+ // without it, KillGen answers errNoGenContainment, the parent cannot be
+ // raised, and a generation's leader is all there is of it.
+ gensOn, contained bool
+ // killFails is how many more KillGen calls fail for host#n before the
+ // kill heals.
+ killFails map[string]int
// shutdown is set once Service.Shutdown has returned: nothing may spawn
// after it (I2).
shutdown bool
- // outcomes counts Swap results by the mode they ran in and their
- // outcome, so a sweep shows it reached switches and holds.
+ // outcomes counts the seed's settings and its Swap results by mode,
+ // outcome and reason, so a sweep shows what it reached.
outcomes map[string]int
}
@@ -171,16 +209,41 @@ func (w *simWorld) jitter() {
}
}
-func (w *simWorld) spawn(host, build string, n uint64) *simProc {
+// spawn starts one simulated process of generation n. A leader gets a
+// random stop behaviour; a descendant (a process the app forked) stops
+// only when killed.
+func (w *simWorld) spawn(host, build string, n uint64, leader bool) *simProc {
w.mu.Lock()
w.nextID++
- p := &simProc{w: w, id: w.nextID, host: host, build: build, n: n}
+ p := &simProc{w: w, id: w.nextID, host: host, build: build, n: n, leader: leader}
+ if leader {
+ switch r := w.rng.Intn(20); {
+ case r < 2:
+ p.mode = procStubborn
+ case r < 3:
+ // A failed kill that is not ErrStillRunning is a claims-on
+ // process's answer (KillAndEmpty), and the generations-on stop
+ // handles it through releasable. With generations off the exec
+ // backends' Stop (childProc, unitProc) answers ErrStillRunning
+ // for a process that may remain, so that is what it gets.
+ p.mode = procStubborn
+ if w.gensOn {
+ p.mode = procKillFails
+ }
+ case r < 7:
+ p.mode = procSlow
+ }
+ }
if w.running[host] == nil {
w.running[host] = map[*simProc]bool{}
}
w.running[host][p] = true
- w.events = append(w.events, fmt.Sprintf("spawn p%d %s build=%s gen=%d", p.id, host, build, n))
- after := w.shutdown
+ what := "spawn"
+ if !leader {
+ what = "fork"
+ }
+ w.events = append(w.events, fmt.Sprintf("%s p%d %s build=%s gen=%d mode=%s", what, p.id, host, build, n, p.mode))
+ after := w.shutdown && w.gensOn
w.mu.Unlock()
if after {
w.violatef("I2: p%d (%s build %s) spawned after Shutdown returned", p.id, host, build)
@@ -188,6 +251,17 @@ func (w *simWorld) spawn(host, build string, n uint64) *simProc {
return p
}
+// spawnGen starts generation n's leader. With generations and containment
+// on, one generation in four also forks a process that outlives its
+// leader, which only KillGen (or a crash) ends.
+func (w *simWorld) spawnGen(host, build string, n uint64) *simProc {
+ p := w.spawn(host, build, n, true)
+ if w.gensOn && w.contained && n != 0 && w.intn(4) == 0 {
+ w.spawn(host, build, n, false)
+ }
+ return p
+}
+
func (w *simWorld) runningOn(host string) int {
w.mu.Lock()
defer w.mu.Unlock()
@@ -206,7 +280,7 @@ func (w *simWorld) runningCount() int {
func (w *simWorld) leaseFreed(host, by string) {
if n := w.runningOn(host); n > 0 {
- w.violatef("I2: lease of %s released by %s while %d generation(s) run", host, by, n)
+ w.violatef("I2: lease of %s released by %s while %d generation process(es) run", host, by, n)
}
w.logf("lease %s released by %s", host, by)
}
@@ -224,6 +298,8 @@ func (w *simWorld) crashOne(host string) {
for p := range w.running[host] {
victims = append(victims, p)
}
+ // Map order is random; sort so the seed picks the victim.
+ sort.Slice(victims, func(i, j int) bool { return victims[i].id < victims[j].id })
var p *simProc
if len(victims) > 0 {
p = victims[w.rng.Intn(len(victims))]
@@ -234,6 +310,28 @@ func (w *simWorld) crashOne(host string) {
}
}
+var simDigits = regexp.MustCompile(`[0-9]+`)
+
+// tally counts one Swap result. The reason is normalized (host names and
+// numbers out, length capped) so that like results group.
+func (w *simWorld) tally(res SwapResult) {
+ key := string(res.Mode) + "/" + string(res.Outcome)
+ if res.Outcome != SwapDone && res.Reason != "" {
+ r := res.Reason
+ for _, h := range w.hosts {
+ r = strings.ReplaceAll(r, h, "<host>")
+ }
+ r = simDigits.ReplaceAllString(r, "N")
+ if len(r) > 110 {
+ r = r[:110] + "..."
+ }
+ key += ": " + r
+ }
+ w.mu.Lock()
+ w.outcomes[key]++
+ w.mu.Unlock()
+}
+
func (w *simWorld) snapshot() ([]string, []string, map[string]int) {
w.mu.Lock()
defer w.mu.Unlock()
@@ -244,20 +342,77 @@ func (w *simWorld) snapshot() ([]string, []string, map[string]int) {
return append([]string(nil), w.violations...), append([]string(nil), w.events...), outcomes
}
-// simProc is one simulated generation process. Stop always ends it: a
-// termination the sim never leaves unproven, so every retained lease the
-// sim sees is a retention the code chose.
+// simProcMode is how a simulated leader answers Stop.
+type simProcMode int
+
+const (
+ // procPrompt ends at once.
+ procPrompt simProcMode = iota
+ // procSlow drains for a few milliseconds, then ends.
+ procSlow
+ // procStubborn answers ErrStillRunning: SIGKILL did not prove it gone.
+ // It exits on its own a few milliseconds after it was first asked.
+ procStubborn
+ // procKillFails answers a kill error that is not ErrStillRunning (a
+ // claims-on KillAndEmpty failure), and exits on its own later, as
+ // procStubborn does. Generations-on seeds only (see spawn).
+ procKillFails
+)
+
+func (m simProcMode) String() string {
+ return [...]string{"prompt", "slow", "stubborn", "killfails"}[m]
+}
+
+var errSimKillFailed = errors.New("sim: kill failed")
+
+// simProc is one simulated process of a generation: its leader, or a
+// process the leader forked (leader false), which nothing but KillGen or a
+// crash ends.
type simProc struct {
- w *simWorld
- id int
- host string
- build string
- n uint64
- mu sync.Mutex
- dead bool
+ w *simWorld
+ id int
+ host string
+ build string
+ n uint64
+ leader bool
+ mode simProcMode
+ mu sync.Mutex
+ dead bool
+ later bool
+}
+
+func (p *simProc) Stop(context.Context) error {
+ p.w.jitter()
+ if p.Exited() {
+ return nil
+ }
+ switch p.mode {
+ case procSlow:
+ time.Sleep(time.Duration(1+p.w.intn(6)) * time.Millisecond)
+ case procStubborn:
+ p.exitLater()
+ return ErrStillRunning
+ case procKillFails:
+ p.exitLater()
+ return errSimKillFailed
+ }
+ p.die("stop")
+ return nil
}
-func (p *simProc) Stop(context.Context) error { p.w.jitter(); p.die("stop"); return nil }
+// exitLater arms the one delayed exit of a process that did not stop when
+// asked.
+func (p *simProc) exitLater() {
+ p.mu.Lock()
+ armed := p.later
+ p.later = true
+ p.mu.Unlock()
+ if armed {
+ return
+ }
+ d := time.Duration(1+p.w.intn(15)) * time.Millisecond
+ time.AfterFunc(d, func() { p.die("late exit") })
+}
func (p *simProc) Exited() bool {
p.mu.Lock()
@@ -316,11 +471,10 @@ func (l *simLease) Release(context.Context) error {
return nil
}
-// simBackend is Backend plus SwapBackend. It is not a GenBackend, so a
-// wake of the live link goes through Start and gets canonicalGen (a fresh
-// number on the canonical socket), as every pre-generations test double
-// does. It has no genKiller, hostKiller or sidecarKiller: a generation's
-// leader exit is its proof (genChildEmpty, sidecarTerminated).
+// simBackend is Backend, GenBackend, SwapBackend and genKiller, as the
+// exec backend is. It has no hostKiller or sidecarKiller (no workers, no
+// sidecars). The live-link wake goes through StartGen, so every generation
+// has a number KillGen can name.
type simBackend struct {
w *simWorld
dir string
@@ -330,7 +484,9 @@ type simBackend struct {
var (
_ Backend = (*simBackend)(nil)
+ _ GenBackend = (*simBackend)(nil)
_ SwapBackend = (*simBackend)(nil)
+ _ genKiller = (*simBackend)(nil)
_ DueStore = simDue{}
)
@@ -348,7 +504,7 @@ func (b *simBackend) loseLease(host string) {
func (b *simBackend) Acquire(_ context.Context, rt registry.Route) (Lease, error) {
b.w.jitter()
if n := b.w.runningOn(rt.Host); n > 0 {
- b.w.violatef("I2: lease of %s acquired while %d generation(s) of the previous holder run", rt.Host, n)
+ b.w.violatef("I2: lease of %s acquired while %d process(es) of the previous holder's generations run", rt.Host, n)
}
return &simLease{b: b, host: rt.Host}, nil
}
@@ -357,12 +513,57 @@ func (b *simBackend) Restore(context.Context, registry.Route) (bool, error) {
return false, nil
}
func (b *simBackend) Seed(context.Context, registry.Route) error { return nil }
-func (b *simBackend) Start(_ context.Context, rt registry.Route) (Proc, string, error) {
+
+// Start is the pre-generations spawn. The Service never calls it on a
+// GenBackend; it is here for the Backend interface.
+func (b *simBackend) Start(ctx context.Context, rt registry.Route) (Proc, string, error) {
+ p, g, err := b.StartGen(ctx, rt, 0, false)
+ return p, g.Build, err
+}
+
+// StartGen is the wake's spawn of the live link (which only ever names v1
+// or v2).
+func (b *simBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, perGen bool) (Proc, Gen, error) {
b.w.jitter()
b.w.mu.Lock()
build := b.w.live[rt.Host]
b.w.mu.Unlock()
- return b.w.spawn(rt.Host, build, 0), build, nil
+ g := canonicalGen(rt, n, build)
+ if perGen {
+ g.Socket = filepath.Join(b.dir, fmt.Sprintf("%s.%d.sock", rt.Host, n))
+ g.Control = g.Socket + ".control"
+ }
+ if err := ctx.Err(); err != nil {
+ return nil, g, err
+ }
+ p, err := b.startOutcome(rt.Host, build, n)
+ return p, g, err
+}
+
+// startOutcome is a spawn's result. Most run. Some fail before the exec,
+// with nothing running. With containment, some fail after it with their
+// kill unproven (errGenUnproven, as plan 2's startUnitAt answers): the
+// generation runs on, with its handle returned or not.
+func (b *simBackend) startOutcome(host, build string, n uint64) (Proc, error) {
+ w := b.w
+ r := w.intn(30)
+ switch {
+ case r == 0:
+ return nil, errors.New("sim: exec failed")
+ case r <= 2 && w.contained:
+ p := w.spawnGen(host, build, n)
+ err := fmt.Errorf("sim: health wait failed; %w: kill unproven", errGenUnproven)
+ w.logf("start of %s gen %d failed with its kill unproven (handle returned: %v)", host, n, r == 1)
+ if r == 1 {
+ return p, err
+ }
+ return nil, err
+ }
+ return w.spawnGen(host, build, n), nil
+}
+
+func (b *simBackend) NextDueVia(context.Context, registry.Route, string) (int64, error) {
+ return 0, nil
}
func (b *simBackend) NextDue(context.Context, registry.Route) (int64, error) { return 0, nil }
func (b *simBackend) HasSidecar(registry.Route) bool { return false }
@@ -388,16 +589,68 @@ func (b *simBackend) StartStaged(ctx context.Context, rt registry.Route, g Gen)
if err := ctx.Err(); err != nil {
return g, nil, err
}
- if b.w.intn(20) == 0 {
- return g, nil, errors.New("sim: exec failed")
- }
g.Socket = filepath.Join(b.dir, fmt.Sprintf("%s.%d.sock", rt.Host, g.N))
g.Control = g.Socket + ".control"
- return g, b.w.spawn(rt.Host, g.Build, g.N), nil
+ p, err := b.startOutcome(rt.Host, g.Build, g.N)
+ return g, p, err
+}
+
+// KillGen is plan 2's generation kill: it empties generation n's child
+// and returns once the child is empty. One generation in six fails its
+// first one or two kills, then heals. A process that does not die when
+// killed (procStubborn, procKillFails) keeps the child from emptying until
+// it exits on its own. Without containment it answers errNoGenContainment,
+// as ExecBackend.KillGen does.
+func (b *simBackend) KillGen(_ context.Context, rt registry.Route, n uint64) error {
+ w := b.w
+ if !w.contained {
+ return fmt.Errorf("sim: %w", errNoGenContainment)
+ }
+ w.jitter()
+ key := fmt.Sprintf("%s#%d", rt.Host, n)
+ w.mu.Lock()
+ fails, seen := w.killFails[key]
+ if !seen && w.rng.Intn(6) == 0 {
+ fails = 1 + w.rng.Intn(2)
+ }
+ var procs []*simProc
+ for p := range w.running[rt.Host] {
+ if p.n == n {
+ procs = append(procs, p)
+ }
+ }
+ if fails > 0 {
+ w.killFails[key] = fails - 1
+ w.events = append(w.events, fmt.Sprintf("KillGen %s gen %d fails (%d more)", rt.Host, n, fails-1))
+ w.mu.Unlock()
+ return errors.New("sim: cgroup.kill: the child is not empty")
+ }
+ w.killFails[key] = 0
+ w.mu.Unlock()
+ stuck := 0
+ for _, p := range procs {
+ if p.mode == procStubborn || p.mode == procKillFails {
+ p.exitLater()
+ stuck++
+ continue
+ }
+ p.die("cgroup.kill")
+ }
+ if stuck > 0 {
+ return fmt.Errorf("sim: g%d still holds %d process(es)", n, stuck)
+ }
+ return nil
}
+
func (b *simBackend) GenTier(registry.Route, string) cgroup.Limit { return cgroup.TierDefault }
func (b *simBackend) GenReady(registry.Route, Gen, Proc) {}
+
+// RaiseGenParent refuses without containment, as plan 2's ParentLimit
+// does, so such a seed never overlaps.
func (b *simBackend) RaiseGenParent(context.Context, registry.Route, cgroup.Limit, cgroup.Limit) error {
+ if !b.w.contained {
+ return fmt.Errorf("sim: %w", errNoGenContainment)
+ }
return nil
}
func (b *simBackend) RestoreGenParent(context.Context, registry.Route, cgroup.Limit) error {
@@ -422,9 +675,7 @@ func (simDue) EarliestTimers() (map[string]int64, error) { return nil, nil }
// target v3 or v4, so a swap that ran the live link is a visible mismatch.
func (w *simWorld) checkSwap(at Attempt, res SwapResult) {
w.logf("swap %s %s -> %s: %s %s", at.Host, at.Mode, at.Build, res.Outcome, res.Reason)
- w.mu.Lock()
- w.outcomes[string(res.Mode)+"/"+string(res.Outcome)]++
- w.mu.Unlock()
+ w.tally(res)
if res.Outcome != SwapDone {
return
}
@@ -469,22 +720,36 @@ func (w *simWorld) checkGenerations(a *Service, hosts []string, stop <-chan stru
}
}
+func onOff(b bool) string {
+ if b {
+ return "on"
+ }
+ return "off"
+}
+
func simulateSwapOnce(seed int, staged map[string]simArtifact, sockDir string, sink *simLogSink) ([]string, []string, map[string]int) {
+ hosts := []string{"one.sim.example.com", "two.sim.example.com"}
w := &simWorld{rng: rand.New(rand.NewSource(int64(seed))), running: map[string]map[*simProc]bool{},
- live: map[string]string{}, staged: map[string]string{}, seen: map[string]bool{}, outcomes: map[string]int{}}
+ live: map[string]string{}, staged: map[string]string{}, seen: map[string]bool{}, outcomes: map[string]int{},
+ killFails: map[string]int{}, hosts: hosts}
+ // One seed in five runs with the dormant switch off (today's paths,
+ // where stop()'s own ErrStillRunning check is the only retention), and
+ // one in five without generation containment.
+ w.gensOn, w.contained = w.rng.Intn(5) != 0, w.rng.Intn(5) != 0
+ w.outcomes[fmt.Sprintf("seeds with generations %s, containment %s", onOff(w.gensOn), onOff(w.contained))]++
+ w.events = append(w.events, fmt.Sprintf("seed %d: generations %s, containment %s", seed, onOff(w.gensOn), onOff(w.contained)))
sink.w.Store(w)
defer sink.w.Store(nil)
for b, art := range staged {
w.staged[b] = art.path
}
- hosts := []string{"one.sim.example.com", "two.sim.example.com"}
for _, h := range hosts {
w.live[h] = "v1"
}
sb := &simBackend{w: w, dir: sockDir, reps: map[string]*simRep{}}
clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}
a := New(Config{Bucket: "sim", Idle: time.Hour}, sb, simDue{})
- a.gens = true
+ a.gens = w.gensOn
a.now = clk.now
a.genHealth = func(ctx context.Context, _ Gen) error {
w.jitter()
@@ -507,11 +772,37 @@ func simulateSwapOnce(seed int, staged map[string]simArtifact, sockDir string, s
var reqs, swaps sync.WaitGroup
var fenceGen atomic.Uint64
+ // The sweeper. The real Service ticks it on its own goroutine, so a
+ // pending stop or an unproven wake is retried without an event to
+ // trigger it. Here it ticks every 5ms of wall time (the fake clock does
+ // not move, so nothing goes idle by it) and is serialized with the
+ // events' own sweeps: there is one sweeper. It stops before Shutdown, as
+ // the real one does.
+ var sweepMu sync.Mutex
+ sweep := func() {
+ sweepMu.Lock()
+ a.sweepOnce()
+ sweepMu.Unlock()
+ }
+ stopSweeper, sweeperDone := make(chan struct{}), make(chan struct{})
+ go func() {
+ defer close(sweeperDone)
+ tick := time.NewTicker(5 * time.Millisecond)
+ defer tick.Stop()
+ for {
+ select {
+ case <-stopSweeper:
+ return
+ case <-tick.C:
+ sweep()
+ }
+ }
+ }()
for i := 0; i < 25; i++ {
host := hosts[w.intn(len(hosts))]
rt := registry.Route{Host: host, Kind: "instance", Socket: filepath.Join(sockDir, host+".sock"),
DBPath: filepath.Join(sockDir, host+".db"), Hibernate: true, Status: "ready"}
- switch w.intn(11) {
+ switch w.intn(12) {
case 0, 1:
reqs.Add(1)
go func() {
@@ -549,10 +840,10 @@ func simulateSwapOnce(seed int, staged map[string]simArtifact, sockDir string, s
}()
case 5:
clk.advance(2 * time.Hour)
- a.sweepOnce()
+ sweep()
case 6:
sb.loseLease(host)
- a.sweepOnce()
+ sweep()
case 7:
a.Bounce(host)
case 8:
@@ -571,11 +862,19 @@ func simulateSwapOnce(seed int, staged map[string]simArtifact, sockDir string, s
revs[host].Add(1) // the scheduler's newer desired build supersedes in-flight attempts
case 10:
w.crashOne(host)
- a.sweepOnce()
+ sweep()
+ case 11:
+ // A stop-retry interval passes: the sweep's re-stop cadences
+ // (pending stops, unproven members) come due, and nothing goes
+ // idle.
+ clk.advance(actStopTimeout + time.Second)
+ sweep()
}
w.jitter()
}
reqs.Wait()
+ close(stopSweeper)
+ <-sweeperDone
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
a.Shutdown(ctx)
cancel()
@@ -583,8 +882,14 @@ func simulateSwapOnce(seed int, staged map[string]simArtifact, sockDir string, s
w.shutdown = true
w.mu.Unlock()
w.logf("Shutdown returned")
- if n := w.runningCount(); n > 0 {
- w.violatef("I2: %d generation(s) still run after Shutdown returned", n)
+ // "Nothing runs after Shutdown returns" is the generations-on Shutdown
+ // (spec §7.4: it cancels and joins waking entries, the swapsim:M6 line).
+ // With the switch off, today's rule holds by design (global constraints:
+ // Shutdown and wake() byte for byte as today): a wake already in flight
+ // resolves on its own and the lease TTL bounds it. Flag-off seeds check
+ // every lease release and acquire, not this clause.
+ if n := w.runningCount(); n > 0 && w.gensOn {
+ w.violatef("I2: %d generation process(es) still run after Shutdown returned", n)
}
done := make(chan struct{})
go func() { swaps.Wait(); close(done) }()
@@ -593,8 +898,8 @@ func simulateSwapOnce(seed int, staged map[string]simArtifact, sockDir string, s
case <-time.After(10 * time.Second):
w.violatef("liveness: a Swap did not return after Shutdown")
}
- if n := w.runningCount(); n > 0 {
- w.violatef("I2: %d generation(s) run once every Swap has returned after Shutdown", n)
+ if n := w.runningCount(); n > 0 && w.gensOn {
+ w.violatef("I2: %d generation process(es) run once every Swap has returned after Shutdown", n)
}
close(stopCheck)
<-checked
diff --git a/internal/activator/testdata/swapsim/mutate.sh b/internal/activator/testdata/swapsim/mutate.sh
index 53ae29059..0506a68a9 100755
--- a/internal/activator/testdata/swapsim/mutate.sh
+++ b/internal/activator/testdata/swapsim/mutate.sh
@@ -1,33 +1,49 @@
#!/bin/bash
# Mutation checks for the swap invariants (zero-downtime plan 3, task 13).
# Each mutant rewrites one marked line (// swapsim:Mn) in a scratch copy of
-# the tree, never in the checkout. It then runs the simulation and the
-# deterministic witnesses (TestSwapWitness*) against that copy.
+# the tree, never in the checkout. Against that copy it then runs, as
+# separate go test calls, the simulation (TestSwapInvariants) and each of
+# the mutant's designated witnesses, so a witness that panics cannot keep
+# the sweep from running, and another witness's failure is not counted.
#
# internal/activator/testdata/swapsim/mutate.sh 300
#
-# The script renices itself to 19 and runs one mutant at a time with
-# GOMAXPROCS=2 (the CPU budget on shared boxes). No witness binds a socket,
-# so TMPDIR stays whatever it is; Go's own scratch goes under the script's.
-#
-# Verdicts, one line per mutant, with the sweep and the witnesses reported
-# separately:
-# CAUGHT a witness failed, or the sweep reported a violated invariant
-# (the sweep stops after three failing seeds, so its count is at
-# most 3)
-# MISSED everything passed under the mutation
-# ERROR the marker is not on exactly one line, the substitution did not
-# apply, or the mutant did not build
-# Exit 1 on any MISSED or ERROR. Scratch: $TMPDIR/zdd3-mut. The tree copy is
-# removed on exit; the logs stay in $TMPDIR/zdd3-mut/logs. Remove them after.
-#
# Names after the seed count run only those mutants (a fix loop's re-check):
# internal/activator/testdata/swapsim/mutate.sh 300 M5 M7
+#
+# The script runs at nice 19 (it re-executes itself under nice if it has
+# to, and warns if it cannot) and one mutant at a time with GOMAXPROCS=2
+# (the CPU budget on shared boxes). No witness binds a socket, so TMPDIR
+# stays whatever it is; Go's own scratch goes under the script's.
+#
+# Each verdict line reports the sweep and every designated witness:
+# sweep: "violated at K seed(s)" (an I1-I3 violation; the sweep stops
+# after three failing seeds, so K is at most 3), "liveness only at K
+# seed(s)", "passed", "TIMEOUT", "PANIC", or "NOT RUN" (no PASS or FAIL
+# line for TestSwapInvariants).
+# witness: FAIL (an assertion failed), PASS, TIMEOUT or PANIC.
+# Verdicts:
+# CAUGHT a designated witness failed by assertion, or the sweep
+# reported an I1-I3 violation
+# TIMEOUT not caught, and a run timed out
+# PANIC not caught, and a run panicked
+# LIVENESS not caught; the sweep's only failure is a Swap that never
+# returned
+# MISSED everything passed under the mutation
+# ERROR the marker is not on exactly one line, the substitution did
+# not apply, or the mutant did not build
+# Exit 1 on anything but CAUGHT. Scratch: $TMPDIR/zdd3-mut. The tree copy is
+# removed on exit; the logs stay in $TMPDIR/zdd3-mut/logs. Remove them after.
set -u
+if [ "$(nice)" -lt 19 ] && [ -z "${SWAPSIM_RENICED:-}" ]; then
+ SWAPSIM_RENICED=1 exec nice -n 19 "$0" "$@"
+fi
+if [ "$(nice)" -lt 19 ]; then
+ echo "warning: running at niceness $(nice), not 19" >&2
+fi
SEEDS=${1:-300}
shift $(($# > 0 ? 1 : 0))
ONLY=" $* "
-renice -n 19 -p $$ >/dev/null 2>&1 || true
export GOMAXPROCS=2
ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) || exit 2
SCRATCH=${TMPDIR:?}/zdd3-mut
@@ -41,19 +57,40 @@ mkdir -p "$SCRATCH/tree" "$SCRATCH/logs" "$SCRATCH/gowork" || exit 2
trap 'rm -rf "$SCRATCH/tree" "$SCRATCH/gowork"' EXIT
export GOTMPDIR=$SCRATCH/gowork
cp -a "$ROOT/go.mod" "$ROOT/go.sum" "$ROOT/internal" "$ROOT/launchctx" "$SCRATCH/tree/" || exit 2
-# name|file|marker|perl substitution applied to the marked line only. The
-# marker match is anchored (\b), so swapsim:M1 never matches swapsim:M10.
-MUTANTS='M1|activator.go|swapsim:M1|s/a\.joinOps\(e\)/true/
-M2|swap.go|swapsim:M2|s/a\.addMemberLocked\(e, retiring\)/_ = retiring/
-M3|activator.go|swapsim:M3|s/e\.pin != nil/false/
-M4|opset.go|swapsim:M4|s/proven := a\.stopMember\(e, m\)/proven := true; go a.stopMember(e, m)/
-M5|swapadmit.go|swapsim:M5|s/!waking \&\& e\.state != actAwake/false/
-M6|activator.go|swapsim:M6|s/if a\.gens \{/if false {/
-M7|opset.go|swapsim:M7|s/return a\.genTerminated\(rt, m\.gen, m\.proc\)/return true || a.genTerminated(rt, m.gen, m.proc)/
-M8|activator.go|swapsim:M8|s/if a\.gens \&\& !a\.releasable\(e\)/if false \&\& !a.releasable(e)/
-M9|activator.go|swapsim:M9|s/w\.genUnproven = genStartUnproven\(err\)/w.genUnproven = false/'
+
+# name|file|marker|designated witnesses (comma-separated)|perl substitution
+# applied to the marked line only. The marker match is anchored (\b), so
+# swapsim:M1 never matches swapsim:M10. The substitution is the last field,
+# so it may contain "|".
+MUTANTS='M1|activator.go|swapsim:M1|TestSwapWitnessBarrierJoinsBeforeTheLease,TestSwapWitnessAgentRestartMidSwap|s/a\.joinOps\(e\)/true/
+M2|swap.go|swapsim:M2|TestSwapWitnessAgentRestartMidSwap|s/a\.addMemberLocked\(e, retiring\)/_ = retiring/
+M3|activator.go|swapsim:M3|TestSwapWitnessHoldExecsTheStagedBuild|s/e\.pin != nil/false/
+M4|opset.go|swapsim:M4|TestSwapWitnessBarrierJoinsBeforeTheLease,TestSwapWitnessAgentRestartMidSwap|s/proven := a\.stopMember\(e, m\)/proven := true; go a.stopMember(e, m)/
+M5|swapadmit.go|swapsim:M5|TestSwapWitnessNoAdmissionOntoAStoppingEntry|s/!waking \&\& e\.state != actAwake/false/
+M6|activator.go|swapsim:M6|TestSwapWitnessShutdownJoinsAWakingEntry|s/if a\.gens \{/if false {/
+M7|opset.go|swapsim:M7|TestSwapWitnessMemberStopProvesTheGenerationChildEmpty|s/return a\.genTerminated\(rt, m\.gen, m\.proc\)/return true || a.genTerminated(rt, m.gen, m.proc)/
+M8|activator.go|swapsim:M8|TestSwapWitnessStopProvesTheCurrentGenerationEmpty|s/if a\.gens \&\& !a\.releasable\(e\)/if false \&\& !a.releasable(e)/
+M9|activator.go|swapsim:M9|TestSwapWitnessUnprovenWakeKeepsTheLease|s/w\.genUnproven = genStartUnproven\(err\)/w.genUnproven = false/'
+
+# runtest LOG TIMEOUT RUN [ENV...]: one go test call against the scratch
+# tree, verbose so that a test that ran prints its --- PASS or --- FAIL.
+runtest() {
+ local log=$1 timeout=$2 run=$3
+ shift 3
+ (cd "$SCRATCH/tree" && env "$@" go test -v -p 1 -count=1 -timeout "$timeout" -run "$run" ./$A/) >"$log" 2>&1
+}
+
+# crashed LOG: TIMEOUT or PANIC if the test binary died, else empty.
+crashed() {
+ if grep -q '^panic: test timed out' "$1"; then
+ echo TIMEOUT
+ elif grep -qE '^(panic: |fatal error: )' "$1"; then
+ echo PANIC
+ fi
+}
+
fail=0
-while IFS='|' read -r name file marker sub; do
+while IFS='|' read -r name file marker witnesses sub; do
if [ "$ONLY" != " " ] && [[ "$ONLY" != *" $name "* ]]; then
continue
fi
@@ -61,37 +98,80 @@ while IFS='|' read -r name file marker sub; do
# Each marker sits on exactly one line of the package's Go sources.
lines=$(cat "$ROOT/$A"/*.go | grep -cE "$marker\\b")
if [ "$lines" -ne 1 ]; then
- echo "ERROR $name: $marker is on $lines lines of $A/*.go, not one"
+ echo "ERROR $name: $marker is on $lines lines of $A/*.go, not one"
fail=1
continue
fi
cp "$ROOT/$A/$file" "$src"
perl -pi -e "if (/\\Q$marker\\E\\b/) { $sub }" "$src"
if cmp -s "$ROOT/$A/$file" "$src"; then
- echo "ERROR $name: the substitution did not apply to $file"
+ echo "ERROR $name: the substitution did not apply to $file"
fail=1
continue
fi
- log="$SCRATCH/logs/$name.log"
- (cd "$SCRATCH/tree" && SWAP_SIM_SEEDS=$SEEDS go test -p 1 -count=1 -timeout 20m \
- -run 'TestSwapInvariants|TestSwapWitness' ./$A/) >"$log" 2>&1
- rc=$?
+
+ # The sweep, on its own.
+ slog="$SCRATCH/logs/$name.sweep.log"
+ runtest "$slog" 20m '^TestSwapInvariants$' SWAP_SIM_SEEDS="$SEEDS"
+ built=1
+ grep -q '\[build failed\]\|\[setup failed\]' "$slog" && built=0
+ inv=$(grep -c 'invariant violated at seed' "$slog")
+ live=$(grep -c 'liveness failed at seed' "$slog")
+ scrash=$(crashed "$slog")
+ if [ "$inv" -gt 0 ]; then
+ sweep="violated at $inv seed(s)"
+ elif [ -n "$scrash" ]; then
+ sweep=$scrash
+ elif ! grep -qE -- '--- (PASS|FAIL): TestSwapInvariants' "$slog"; then
+ sweep="NOT RUN"
+ elif [ "$live" -gt 0 ]; then
+ sweep="liveness only at $live seed(s)"
+ else
+ sweep=passed
+ fi
+
+ # Each designated witness, on its own.
+ wres=""
+ wcaught=0
+ crash=$scrash
+ for w in ${witnesses//,/ }; do
+ wlog="$SCRATCH/logs/$name.$w.log"
+ runtest "$wlog" 5m "^$w\$"
+ grep -q '\[build failed\]\|\[setup failed\]' "$wlog" && built=0
+ wcrash=$(crashed "$wlog")
+ if [ -n "$wcrash" ]; then
+ r=$wcrash
+ # A timeout outranks a panic in the verdict.
+ if [ "$wcrash" = TIMEOUT ] || [ -z "$crash" ]; then
+ crash=$wcrash
+ fi
+ elif grep -q -- "--- FAIL: $w " "$wlog"; then
+ r=FAIL
+ wcaught=1
+ elif grep -q -- "--- PASS: $w " "$wlog"; then
+ r=PASS
+ else
+ r="NOT RUN"
+ fi
+ wres="$wres $w=$r"
+ done
cp "$ROOT/$A/$file" "$src"
- if grep -q '\[build failed\]\|\[setup failed\]' "$log"; then
- echo "ERROR $name: the mutant did not build; see $log"
+
+ detail="sweep $sweep; witnesses${wres}"
+ if [ $built -eq 0 ]; then
+ echo "ERROR $name: the mutant did not build; see $SCRATCH/logs/$name.*"
fail=1
- elif [ $rc -eq 0 ]; then
- echo "MISSED $name"
+ elif [ $wcaught -eq 1 ] || [ "$inv" -gt 0 ]; then
+ echo "CAUGHT $name: $detail"
+ elif [ -n "$crash" ]; then
+ printf '%-9s %s: %s; see %s\n' "$crash" "$name" "$detail" "$SCRATCH/logs/$name.*"
+ fail=1
+ elif [ "$live" -gt 0 ]; then
+ echo "LIVENESS $name: $detail"
fail=1
else
- sweep=$(grep -c 'invariant violated at seed' "$log")
- witness=$(grep -o -- '--- FAIL: TestSwapWitness[A-Za-z]*' "$log" | sed 's/--- FAIL: //' | sort -u | tr '\n' ' ')
- other=""
- if [ "$sweep" -eq 0 ] && [ -z "$witness" ]; then
- # Neither a violation nor a witness: say what did fail.
- other="; other failure: $(grep -m1 -E 'panic: |FAIL' "$log"), see $log"
- fi
- echo "CAUGHT $name: sweep seeds failing ${sweep}; witnesses ${witness:-none}${other}"
+ echo "MISSED $name: $detail"
+ fail=1
fi
done <<<"$MUTANTS"
exit $fail