amadan
zdd plan 3 final fix: WIP mutate.sh integrity and scratch isolation (F3/F4, partial, untested; fixer cut off by a usage limit)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 file changed,
+45
−22
internal/activator/testdata/swapsim/mutate.sh+45 −22
diff --git a/internal/activator/testdata/swapsim/mutate.sh b/internal/activator/testdata/swapsim/mutate.sh| index 0506a68a9..5011d2e10 100755 |
| --- a/internal/activator/testdata/swapsim/mutate.sh |
| +++ b/internal/activator/testdata/swapsim/mutate.sh |
| @@ -21,19 +21,26 @@ |
| # after three failing seeds, so K is at most 3), "liveness only at K |
| # seed(s)", "passed", "TIMEOUT", "PANIC", or "NOT RUN" (no PASS or FAIL |
| # line for TestSwapInvariants). |
| -# witness: FAIL (an assertion failed), PASS, TIMEOUT or PANIC. |
| +# witness: FAIL (an assertion failed), PASS, TIMEOUT, PANIC or NOT RUN. |
| +# Execution integrity is recorded apart from detection. Every run the |
| +# mutant needs (the sweep and each designated witness) must build, start, |
| +# and end in its own PASS or FAIL line. A run that panicked, timed out, never |
| +# ran, or did not build fails the mutant's gate even when another run caught |
| +# the mutation; that kill is still printed, as a diagnostic. |
| # Verdicts: |
| -# CAUGHT a designated witness failed by assertion, or the sweep |
| -# reported an I1-I3 violation |
| -# TIMEOUT not caught, and a run timed out |
| -# PANIC not caught, and a run panicked |
| +# CAUGHT every run executed intact, and a designated witness failed by |
| +# assertion or the sweep reported an I1-I3 violation |
| +# BROKEN a run panicked, timed out or never ran (the detail names it); |
| +# "caught" in the detail is the diagnostic kill, not a pass |
| # LIVENESS not caught; the sweep's only failure is a Swap that never |
| # returned |
| # MISSED everything passed under the mutation |
| # ERROR the marker is not on exactly one line, the substitution did |
| -# not apply, or the mutant did not build |
| -# Exit 1 on anything but CAUGHT. Scratch: $TMPDIR/zdd3-mut. The tree copy is |
| -# removed on exit; the logs stay in $TMPDIR/zdd3-mut/logs. Remove them after. |
| +# not apply, or the mutant did not build or set up |
| +# Exit 1 on anything but CAUGHT. Scratch: a fresh directory per run, made |
| +# with mktemp -d under $TMPDIR (named zdd3-mut.XXXXXX) and printed first. |
| +# Its tree copy is removed on exit; its logs stay. Remove that directory |
| +# after. The script never touches any other path under $TMPDIR. |
| set -u |
| if [ "$(nice)" -lt 19 ] && [ -z "${SWAPSIM_RENICED:-}" ]; then |
| SWAPSIM_RENICED=1 exec nice -n 19 "$0" "$@" |
| @@ -46,13 +53,15 @@ shift $(($# > 0 ? 1 : 0)) |
| ONLY=" $* " |
| export GOMAXPROCS=2 |
| ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) || exit 2 |
| -SCRATCH=${TMPDIR:?}/zdd3-mut |
| +# This run's own scratch: never a fixed name, so two runs sharing TMPDIR |
| +# cannot delete each other's trees or logs. |
| +SCRATCH=$(mktemp -d "${TMPDIR:?}/zdd3-mut.XXXXXX") || exit 2 |
| +echo "scratch: $SCRATCH" |
| A=internal/activator |
| # The default build cache is read-only inside a sandbox. |
| if [ -z "${GOCACHE:-}" ] && [ ! -w "$(go env GOCACHE)" ]; then |
| export GOCACHE=$TMPDIR/gocache |
| fi |
| -rm -rf "$SCRATCH" |
| mkdir -p "$SCRATCH/tree" "$SCRATCH/logs" "$SCRATCH/gowork" || exit 2 |
| trap 'rm -rf "$SCRATCH/tree" "$SCRATCH/gowork"' EXIT |
| export GOTMPDIR=$SCRATCH/gowork |
| @@ -110,14 +119,22 @@ while IFS='|' read -r name file marker witnesses sub; do |
| continue |
| fi |
| - # The sweep, on its own. |
| + # The sweep, on its own. Its integrity is judged apart from what it |
| + # found: a crash or a missing result line breaks the run even when the |
| + # log also reports a violation. |
| slog="$SCRATCH/logs/$name.sweep.log" |
| runtest "$slog" 20m '^TestSwapInvariants$' SWAP_SIM_SEEDS="$SEEDS" |
| built=1 |
| grep -q '\[build failed\]\|\[setup failed\]' "$slog" && built=0 |
| inv=$(grep -c 'invariant violated at seed' "$slog") |
| live=$(grep -c 'liveness failed at seed' "$slog") |
| + broken="" |
| scrash=$(crashed "$slog") |
| + if [ -n "$scrash" ]; then |
| + broken="$broken sweep=$scrash" |
| + elif ! grep -qE -- '--- (PASS|FAIL): TestSwapInvariants' "$slog"; then |
| + broken="$broken sweep=NOT-RUN" |
| + fi |
| if [ "$inv" -gt 0 ]; then |
| sweep="violated at $inv seed(s)" |
| elif [ -n "$scrash" ]; then |
| @@ -130,10 +147,10 @@ while IFS='|' read -r name file marker witnesses sub; do |
| sweep=passed |
| fi |
| - # Each designated witness, on its own. |
| + # Each designated witness, on its own. A witness that crashed or never |
| + # reported is broken even if its log also holds a FAIL line. |
| wres="" |
| wcaught=0 |
| - crash=$scrash |
| for w in ${witnesses//,/ }; do |
| wlog="$SCRATCH/logs/$name.$w.log" |
| runtest "$wlog" 5m "^$w\$" |
| @@ -141,10 +158,7 @@ while IFS='|' read -r name file marker witnesses sub; do |
| wcrash=$(crashed "$wlog") |
| if [ -n "$wcrash" ]; then |
| r=$wcrash |
| - # A timeout outranks a panic in the verdict. |
| - if [ "$wcrash" = TIMEOUT ] || [ -z "$crash" ]; then |
| - crash=$wcrash |
| - fi |
| + broken="$broken $w=$wcrash" |
| elif grep -q -- "--- FAIL: $w " "$wlog"; then |
| r=FAIL |
| wcaught=1 |
| @@ -152,20 +166,29 @@ while IFS='|' read -r name file marker witnesses sub; do |
| r=PASS |
| else |
| r="NOT RUN" |
| + broken="$broken $w=NOT-RUN" |
| fi |
| wres="$wres $w=$r" |
| done |
| cp "$ROOT/$A/$file" "$src" |
| + caught=0 |
| + if [ $wcaught -eq 1 ] || [ "$inv" -gt 0 ]; then |
| + caught=1 |
| + fi |
| detail="sweep $sweep; witnesses${wres}" |
| if [ $built -eq 0 ]; then |
| - echo "ERROR $name: the mutant did not build; see $SCRATCH/logs/$name.*" |
| + echo "ERROR $name: the mutant did not build or set up; see $SCRATCH/logs/$name.*" |
| fail=1 |
| - elif [ $wcaught -eq 1 ] || [ "$inv" -gt 0 ]; then |
| - echo "CAUGHT $name: $detail" |
| - elif [ -n "$crash" ]; then |
| - printf '%-9s %s: %s; see %s\n' "$crash" "$name" "$detail" "$SCRATCH/logs/$name.*" |
| + elif [ -n "$broken" ]; then |
| + kill="not caught" |
| + if [ $caught -eq 1 ]; then |
| + kill="caught (diagnostic only)" |
| + fi |
| + printf 'BROKEN %s: execution integrity failed:%s; %s; %s; see %s\n' "$name" "$broken" "$kill" "$detail" "$SCRATCH/logs/$name.*" |
| fail=1 |
| + elif [ $caught -eq 1 ]; then |
| + echo "CAUGHT $name: $detail" |
| elif [ "$live" -gt 0 ]; then |
| echo "LIVENESS $name: $detail" |
| fail=1 |