| 1 | #!/bin/bash |
| 2 | # Mutation checks for the swap invariants (zero-downtime plan 3, task 13). |
| 3 | # Each mutant rewrites one marked line (// swapsim:Mn) in a scratch copy of |
| 4 | # the tree, never in the checkout. Against that copy it then runs, as |
| 5 | # separate go test calls, the simulation (TestSwapInvariants) and each of |
| 6 | # the mutant's designated witnesses, so a witness that panics cannot keep |
| 7 | # the sweep from running, and another witness's failure is not counted. |
| 8 | # |
| 9 | # internal/activator/testdata/swapsim/mutate.sh 300 |
| 10 | # |
| 11 | # Names after the seed count run only those mutants (a fix loop's re-check): |
| 12 | # internal/activator/testdata/swapsim/mutate.sh 300 M5 M7 |
| 13 | # |
| 14 | # The script runs at nice 19 (it re-executes itself under nice if it has |
| 15 | # to, and warns if it cannot) and one mutant at a time with GOMAXPROCS=2 |
| 16 | # (the CPU budget on shared boxes). No witness binds a socket, so TMPDIR |
| 17 | # stays whatever it is; Go's own scratch goes under the script's. |
| 18 | # |
| 19 | # Each verdict line reports the sweep and every designated witness: |
| 20 | # sweep: "violated at K seed(s)" (an I1-I3 violation; the sweep stops |
| 21 | # after three failing seeds, so K is at most 3), "liveness only at K |
| 22 | # seed(s)", "passed", "TIMEOUT", "PANIC", or "NOT RUN" (no PASS or FAIL |
| 23 | # line for TestSwapInvariants). |
| 24 | # witness: FAIL (an assertion failed), PASS, TIMEOUT, PANIC or NOT RUN. |
| 25 | # Execution integrity is recorded apart from detection. Every run the |
| 26 | # mutant needs (the sweep and each designated witness) must build, start, |
| 27 | # and end in its own PASS or FAIL line. A run that panicked, timed out, never |
| 28 | # ran, or did not build fails the mutant's gate even when another run caught |
| 29 | # the mutation; that kill is still printed, as a diagnostic. |
| 30 | # Verdicts: |
| 31 | # CAUGHT every run executed intact, and a designated witness failed by |
| 32 | # assertion or the sweep reported an I1-I3 violation |
| 33 | # BROKEN a run panicked, timed out or never ran (the detail names it); |
| 34 | # "caught" in the detail is the diagnostic kill, not a pass |
| 35 | # LIVENESS not caught; the sweep's only failure is a Swap that never |
| 36 | # returned |
| 37 | # MISSED everything passed under the mutation |
| 38 | # ERROR the marker is not on exactly one line, the substitution did |
| 39 | # not apply, or the mutant did not build or set up |
| 40 | # Exit 1 on anything but CAUGHT. Scratch: a fresh directory per run, made |
| 41 | # with mktemp -d under $TMPDIR (named zdd3-mut.XXXXXX) and printed first. |
| 42 | # Its tree copy is removed on exit; its logs stay. Remove that directory |
| 43 | # after. The script never touches any other path under $TMPDIR. |
| 44 | set -u |
| 45 | if [ "$(nice)" -lt 19 ] && [ -z "${SWAPSIM_RENICED:-}" ]; then |
| 46 | SWAPSIM_RENICED=1 exec nice -n 19 "$0" "$@" |
| 47 | fi |
| 48 | if [ "$(nice)" -lt 19 ]; then |
| 49 | echo "warning: running at niceness $(nice), not 19" >&2 |
| 50 | fi |
| 51 | SEEDS=${1:-300} |
| 52 | shift $(($# > 0 ? 1 : 0)) |
| 53 | ONLY=" $* " |
| 54 | export GOMAXPROCS=2 |
| 55 | ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) || exit 2 |
| 56 | # This run's own scratch: never a fixed name, so two runs sharing TMPDIR |
| 57 | # cannot delete each other's trees or logs. |
| 58 | SCRATCH=$(mktemp -d "${TMPDIR:?}/zdd3-mut.XXXXXX") || exit 2 |
| 59 | echo "scratch: $SCRATCH" |
| 60 | A=internal/activator |
| 61 | # The default build cache is read-only inside a sandbox. |
| 62 | if [ -z "${GOCACHE:-}" ] && [ ! -w "$(go env GOCACHE)" ]; then |
| 63 | export GOCACHE=$TMPDIR/gocache |
| 64 | fi |
| 65 | mkdir -p "$SCRATCH/tree" "$SCRATCH/logs" "$SCRATCH/gowork" || exit 2 |
| 66 | trap 'rm -rf "$SCRATCH/tree" "$SCRATCH/gowork"' EXIT |
| 67 | export GOTMPDIR=$SCRATCH/gowork |
| 68 | cp -a "$ROOT/go.mod" "$ROOT/go.sum" "$ROOT/internal" "$ROOT/launchctx" "$SCRATCH/tree/" || exit 2 |
| 69 | |
| 70 | # name|file|marker|designated witnesses (comma-separated)|perl substitution |
| 71 | # applied to the marked line only. The marker match is anchored (\b), so |
| 72 | # swapsim:M1 never matches swapsim:M10. The substitution is the last field, |
| 73 | # so it may contain "|". |
| 74 | MUTANTS='M1|activator.go|swapsim:M1|TestSwapWitnessBarrierJoinsBeforeTheLease,TestSwapWitnessAgentRestartMidSwap|s/a\.joinOps\(e\)/true/ |
| 75 | M2|swap.go|swapsim:M2|TestSwapWitnessAgentRestartMidSwap|s/a\.addMemberLocked\(e, retiring\)/_ = retiring/ |
| 76 | M3|activator.go|swapsim:M3|TestSwapWitnessHoldExecsTheStagedBuild|s/e\.pin != nil/false/ |
| 77 | M4|opset.go|swapsim:M4|TestSwapWitnessBarrierJoinsBeforeTheLease,TestSwapWitnessAgentRestartMidSwap|s/proven := a\.stopMember\(e, m\)/proven := true; go a.stopMember(e, m)/ |
| 78 | M5|swapadmit.go|swapsim:M5|TestSwapWitnessNoAdmissionOntoAStoppingEntry|s/!waking \&\& e\.state != actAwake/false/ |
| 79 | M6|activator.go|swapsim:M6|TestSwapWitnessShutdownJoinsAWakingEntry,TestShutdownJoinsAWakingEntryWithSwapOff|s/waking = append\(waking, e\)/_ = waking/ |
| 80 | M7|opset.go|swapsim:M7|TestSwapWitnessMemberStopProvesTheGenerationChildEmpty|s/return a\.genTerminated\(rt, m\.gen, m\.proc\)/return true || a.genTerminated(rt, m.gen, m.proc)/ |
| 81 | M8|activator.go|swapsim:M8|TestSwapWitnessStopProvesTheCurrentGenerationEmpty|s/if a\.gens \&\& !a\.releasable\(e\)/if false \&\& !a.releasable(e)/ |
| 82 | M9|activator.go|swapsim:M9|TestSwapWitnessUnprovenWakeKeepsTheLease|s/w\.genUnproven = genStartUnproven\(err\)/w.genUnproven = false/' |
| 83 | |
| 84 | # runtest LOG TIMEOUT RUN [ENV...]: one go test call against the scratch |
| 85 | # tree, verbose so that a test that ran prints its --- PASS or --- FAIL. |
| 86 | runtest() { |
| 87 | local log=$1 timeout=$2 run=$3 |
| 88 | shift 3 |
| 89 | (cd "$SCRATCH/tree" && env "$@" go test -v -p 1 -count=1 -timeout "$timeout" -run "$run" ./$A/) >"$log" 2>&1 |
| 90 | } |
| 91 | |
| 92 | # crashed LOG: TIMEOUT or PANIC if the test binary died, else empty. |
| 93 | crashed() { |
| 94 | if grep -q '^panic: test timed out' "$1"; then |
| 95 | echo TIMEOUT |
| 96 | elif grep -qE '^(panic: |fatal error: )' "$1"; then |
| 97 | echo PANIC |
| 98 | fi |
| 99 | } |
| 100 | |
| 101 | fail=0 |
| 102 | while IFS='|' read -r name file marker witnesses sub; do |
| 103 | if [ "$ONLY" != " " ] && [[ "$ONLY" != *" $name "* ]]; then |
| 104 | continue |
| 105 | fi |
| 106 | src="$SCRATCH/tree/$A/$file" |
| 107 | # Each marker sits on exactly one line of the package's Go sources. |
| 108 | lines=$(cat "$ROOT/$A"/*.go | grep -cE "$marker\\b") |
| 109 | if [ "$lines" -ne 1 ]; then |
| 110 | echo "ERROR $name: $marker is on $lines lines of $A/*.go, not one" |
| 111 | fail=1 |
| 112 | continue |
| 113 | fi |
| 114 | cp "$ROOT/$A/$file" "$src" |
| 115 | perl -pi -e "if (/\\Q$marker\\E\\b/) { $sub }" "$src" |
| 116 | if cmp -s "$ROOT/$A/$file" "$src"; then |
| 117 | echo "ERROR $name: the substitution did not apply to $file" |
| 118 | fail=1 |
| 119 | continue |
| 120 | fi |
| 121 | |
| 122 | # The sweep, on its own. Its integrity is judged apart from what it |
| 123 | # found: a crash or a missing result line breaks the run even when the |
| 124 | # log also reports a violation. |
| 125 | slog="$SCRATCH/logs/$name.sweep.log" |
| 126 | runtest "$slog" 20m '^TestSwapInvariants$' SWAP_SIM_SEEDS="$SEEDS" |
| 127 | built=1 |
| 128 | grep -q '\[build failed\]\|\[setup failed\]' "$slog" && built=0 |
| 129 | inv=$(grep -c 'invariant violated at seed' "$slog") |
| 130 | live=$(grep -c 'liveness failed at seed' "$slog") |
| 131 | broken="" |
| 132 | scrash=$(crashed "$slog") |
| 133 | if [ -n "$scrash" ]; then |
| 134 | broken="$broken sweep=$scrash" |
| 135 | elif ! grep -qE -- '--- (PASS|FAIL): TestSwapInvariants' "$slog"; then |
| 136 | broken="$broken sweep=NOT-RUN" |
| 137 | fi |
| 138 | if [ "$inv" -gt 0 ]; then |
| 139 | sweep="violated at $inv seed(s)" |
| 140 | elif [ -n "$scrash" ]; then |
| 141 | sweep=$scrash |
| 142 | elif ! grep -qE -- '--- (PASS|FAIL): TestSwapInvariants' "$slog"; then |
| 143 | sweep="NOT RUN" |
| 144 | elif [ "$live" -gt 0 ]; then |
| 145 | sweep="liveness only at $live seed(s)" |
| 146 | else |
| 147 | sweep=passed |
| 148 | fi |
| 149 | |
| 150 | # Each designated witness, on its own. A witness that crashed or never |
| 151 | # reported is broken even if its log also holds a FAIL line. |
| 152 | wres="" |
| 153 | wcaught=0 |
| 154 | for w in ${witnesses//,/ }; do |
| 155 | wlog="$SCRATCH/logs/$name.$w.log" |
| 156 | runtest "$wlog" 5m "^$w\$" |
| 157 | grep -q '\[build failed\]\|\[setup failed\]' "$wlog" && built=0 |
| 158 | wcrash=$(crashed "$wlog") |
| 159 | if [ -n "$wcrash" ]; then |
| 160 | r=$wcrash |
| 161 | broken="$broken $w=$wcrash" |
| 162 | elif grep -q -- "--- FAIL: $w " "$wlog"; then |
| 163 | r=FAIL |
| 164 | wcaught=1 |
| 165 | elif grep -q -- "--- PASS: $w " "$wlog"; then |
| 166 | r=PASS |
| 167 | else |
| 168 | r="NOT RUN" |
| 169 | broken="$broken $w=NOT-RUN" |
| 170 | fi |
| 171 | wres="$wres $w=$r" |
| 172 | done |
| 173 | cp "$ROOT/$A/$file" "$src" |
| 174 | |
| 175 | caught=0 |
| 176 | if [ $wcaught -eq 1 ] || [ "$inv" -gt 0 ]; then |
| 177 | caught=1 |
| 178 | fi |
| 179 | detail="sweep $sweep; witnesses${wres}" |
| 180 | if [ $built -eq 0 ]; then |
| 181 | echo "ERROR $name: the mutant did not build or set up; see $SCRATCH/logs/$name.*" |
| 182 | fail=1 |
| 183 | elif [ -n "$broken" ]; then |
| 184 | kill="not caught" |
| 185 | if [ $caught -eq 1 ]; then |
| 186 | kill="caught (diagnostic only)" |
| 187 | fi |
| 188 | printf 'BROKEN %s: execution integrity failed:%s; %s; %s; see %s\n' "$name" "$broken" "$kill" "$detail" "$SCRATCH/logs/$name.*" |
| 189 | fail=1 |
| 190 | elif [ $caught -eq 1 ]; then |
| 191 | echo "CAUGHT $name: $detail" |
| 192 | elif [ "$live" -gt 0 ]; then |
| 193 | echo "LIVENESS $name: $detail" |
| 194 | fail=1 |
| 195 | else |
| 196 | echo "MISSED $name: $detail" |
| 197 | fail=1 |
| 198 | fi |
| 199 | done <<<"$MUTANTS" |
| 200 | exit $fail |
| 201 | |
amadan