| 1 | #!/bin/bash |
| 2 | # Mutation checks for the engines model (fleet-internal engines spec §8; |
| 3 | # #335 plan 1). Adapted from internal/sim/hostlife/mutate.sh: the same |
| 4 | # build-once, run-per-rule shape and the same verdicts. |
| 5 | # |
| 6 | # Build the test binary once (this is the only step that writes the Go |
| 7 | # build cache), then run the sweep. The sweep never copies or edits the |
| 8 | # source: each rule runs the same binary with ENGINESIM_MUTATE set. |
| 9 | # |
| 10 | # go test -c -o "$TMPDIR/enginesim/engines.test" ./internal/sim/engines/ |
| 11 | # nice -n 19 internal/sim/engines/mutate.sh 3000 # every rule |
| 12 | # nice -n 19 internal/sim/engines/mutate.sh 3000 @all # the same |
| 13 | # nice -n 19 internal/sim/engines/mutate.sh 3000 rule-a rule-b # named rules |
| 14 | # nice -n 19 internal/sim/engines/mutate.sh 3000 rule-a,rule-b # a combination, dropped together |
| 15 | # ENGINESIM_LIST=1 internal/sim/engines/mutate.sh 0 # print the rules a run would check, and run nothing |
| 16 | # |
| 17 | # Each rule gets two runs under its mutation: the random sweep (TestEngines |
| 18 | # over the seeds) and the rule's deterministic witness scenarios |
| 19 | # (TestWitnesses, witness_test.go). The verdict line reports both, |
| 20 | # separately: |
| 21 | # |
| 22 | # CAUGHT rule: sweep N of S (first seed n); witness violated|held|none |
| 23 | # |
| 24 | # Classification: |
| 25 | # |
| 26 | # CAUGHT the sweep completed and reported at least one seed that |
| 27 | # violated an invariant, with the seed number parsed from its |
| 28 | # output (so it can be replayed with ENGINESIM_SEED), OR a |
| 29 | # witness of the rule violated an invariant under the mutation; |
| 30 | # MISSED the sweep completed with every seed passing, and every |
| 31 | # witness held (or the rule has none); |
| 32 | # ERROR anything else: an unknown rule, a stale binary, a panic, a |
| 33 | # timeout, output the harness cannot parse, or a witness run |
| 34 | # that failed (a witness that must violate under this mutation |
| 35 | # and held: the witness is broken). An error is never a |
| 36 | # detection. |
| 37 | # |
| 38 | # Expected outcomes live in expected-outcomes.txt beside this script: a |
| 39 | # rule listed there as `missed` (caught only in a stated combination: its |
| 40 | # witness holds alone) is reported as EXPECTED-MISS. A rule listed as |
| 41 | # `rare <N>` is caught by the random sweep only at N seeds or more: below |
| 42 | # N a miss is EXPECTED-MISS and a catch is CAUGHT; at N or more a miss is |
| 43 | # MISSED. The sweep exits nonzero on any ERROR, on any unexpected MISSED, |
| 44 | # and on any rule listed as `missed` that is CAUGHT (STALE). |
| 45 | # |
| 46 | # The machine is shared: run the sweep once per revision, serially, under |
| 47 | # nice -n 19. ENGINESIM_PAR (default 1) runs that many rules at once. |
| 48 | # |
| 49 | # Scratch (the binary, every run's log, the verdicts) lives under |
| 50 | # ${ENGINESIM_SCRATCH:-$TMPDIR/enginesim}; remove that one directory, by |
| 51 | # its exact path, once, after the whole sweep. |
| 52 | set -u |
| 53 | SEEDS=${1:?seeds} |
| 54 | shift |
| 55 | ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) |
| 56 | PKG="$ROOT/internal/sim/engines" |
| 57 | SCRATCH=${ENGINESIM_SCRATCH:-${TMPDIR:?}/enginesim} |
| 58 | BIN=${ENGINESIM_BIN:-$SCRATCH/engines.test} |
| 59 | RULES="$*" |
| 60 | if [ -z "$RULES" ] || [ "$RULES" = "@all" ]; then |
| 61 | RULES=$(sed -n '/^var allRules/,/^}/p' "$PKG/rules_test.go" | grep -o '^\s*"[a-z0-9-]*"' | tr -d ' "\t') |
| 62 | fi |
| 63 | if [ -n "${ENGINESIM_LIST:-}" ]; then |
| 64 | printf '%s\n' $RULES |
| 65 | exit 0 |
| 66 | fi |
| 67 | mkdir -p "$SCRATCH/logs" "$SCRATCH/verdicts" || { echo "cannot create $SCRATCH" >&2; exit 2; } |
| 68 | EXPECT="$PKG/expected-outcomes.txt" |
| 69 | if [ ! -x "$BIN" ]; then |
| 70 | echo "no test binary at $BIN; build it with: go test -c -o $BIN ./internal/sim/engines/" >&2 |
| 71 | exit 2 |
| 72 | fi |
| 73 | if [ -n "$(find "$PKG" -name '*.go' -newer "$BIN")" ]; then |
| 74 | echo "the test binary $BIN is older than the source; rebuild it" >&2 |
| 75 | exit 2 |
| 76 | fi |
| 77 | # the rules that have witnesses (the witnesses table in witness_test.go): a |
| 78 | # run of one of them with no witness output is an ERROR, never "none" |
| 79 | WRULES=$(sed -n '/^var witnesses/,/^}/p' "$PKG/witness_test.go" | grep -o '^\s*{"[a-z0-9-]*",' | tr -d ' \t{",' | sort -u) |
| 80 | # the verdicts of this run only: remove the files inside, by exact name |
| 81 | for f in "$SCRATCH"/verdicts/*; do |
| 82 | [ -e "$f" ] && { rm -f -- "$f" || exit 2; } |
| 83 | done |
| 84 | |
| 85 | # the unmutated sweep first, at the same seed count: if the model violates |
| 86 | # with every rule in place, no catch below means anything (ENGINESIM_MUTATE |
| 87 | # set empty here, so a value in the caller's environment cannot leak in) |
| 88 | blog="$SCRATCH/logs/unmutated.log" |
| 89 | ENGINESIM_MUTATE= ENGINESIM_SEEDS=$SEEDS "$BIN" -test.run '^TestEngines$' -test.count=1 -test.v -test.timeout 180m >"$blog" 2>&1 |
| 90 | brc=$? |
| 91 | if [ $brc -ne 0 ] || [ "$(grep -c "^ sim_test.go:[0-9]*: $SEEDS seeds, 0 failed, 0 residuals; mutated=\"\"" "$blog")" -ne 1 ]; then |
| 92 | echo "BASELINE unmutated: FAILED (rc=$brc); see $blog" >&2 |
| 93 | exit 1 |
| 94 | fi |
| 95 | echo "BASELINE unmutated: sweep 0 of $SEEDS" |
| 96 | |
| 97 | one() { |
| 98 | r=$1 |
| 99 | tag=$(echo "$r" | tr ',' '+') |
| 100 | log="$SCRATCH/logs/$tag.log" |
| 101 | verdict="$SCRATCH/verdicts/$tag" |
| 102 | for x in $(echo "$r" | tr ',' ' '); do |
| 103 | if ! sed -n '/^var allRules/,/^}/p' "$PKG/rules_test.go" | grep -q "^\s*\"$x\","; then |
| 104 | echo "ERROR $r: no rule named $x" | tee "$verdict" |
| 105 | return |
| 106 | fi |
| 107 | done |
| 108 | # the random sweep |
| 109 | ENGINESIM_MUTATE=$r ENGINESIM_SEEDS=$SEEDS "$BIN" -test.run '^TestEngines$' -test.count=1 -test.v -test.timeout 180m >"$log" 2>&1 |
| 110 | rc=$? |
| 111 | expect=$(awk -v r="$r" '$1==r {print $2}' "$EXPECT" 2>/dev/null) |
| 112 | depth=$(awk -v r="$r" '$1==r {print $3}' "$EXPECT" 2>/dev/null) |
| 113 | nviol=$(grep -o "[0-9]* of $SEEDS seeds violated an invariant" "$log" | head -1 | awk '{print $1}') |
| 114 | seed=$(grep -o "first seed [0-9]*" "$log" | head -1 | awk '{print $3}') |
| 115 | clean=$(grep -c "^ sim_test.go:[0-9]*: $SEEDS seeds, 0 failed.*mutated=\"$r\"" "$log") |
| 116 | sweep="" |
| 117 | if [ $rc -ne 0 ] && [ -n "$nviol" ] && [ -n "$seed" ] && ! grep -q "^panic:\|test timed out" "$log"; then |
| 118 | sweep="sweep $nviol of $SEEDS (first seed $seed)" |
| 119 | elif [ $rc -eq 0 ] && [ "$clean" -eq 1 ]; then |
| 120 | sweep="sweep 0 of $SEEDS" |
| 121 | fi |
| 122 | # the witnesses: the rule's own hand-scheduled scenarios, run under the |
| 123 | # same mutation; "violated" if any of them violates an invariant, "held" |
| 124 | # if they all hold, "none" if the rule has no witness. A witness that |
| 125 | # must violate under this mutation and holds fails the test, which is an |
| 126 | # ERROR here: the witness is broken, not the rule |
| 127 | wlog="$SCRATCH/logs/$tag.witness.log" |
| 128 | ENGINESIM_MUTATE=$r "$BIN" -test.run '^TestWitnesses$' -test.count=1 -test.v -test.timeout 10m >"$wlog" 2>&1 |
| 129 | wrc=$? |
| 130 | wit=none |
| 131 | if grep -q "^ witness_test.go:[0-9]*: witness .*: violated" "$wlog"; then |
| 132 | wit=violated |
| 133 | elif grep -q "^ witness_test.go:[0-9]*: witness .*: held" "$wlog"; then |
| 134 | wit=held |
| 135 | fi |
| 136 | if [ $wrc -ne 0 ] || grep -q "^panic:\|test timed out" "$wlog"; then |
| 137 | echo "ERROR $r: witness run rc=$wrc; see $wlog" | tee "$verdict" |
| 138 | return |
| 139 | fi |
| 140 | if [ "$wit" = none ]; then |
| 141 | for x in $(echo "$r" | tr ',' ' '); do |
| 142 | if echo "$WRULES" | grep -qx "$x"; then |
| 143 | echo "ERROR $r: $x has witnesses but the run produced no witness output; see $wlog" | tee "$verdict" |
| 144 | return |
| 145 | fi |
| 146 | done |
| 147 | fi |
| 148 | if [ -z "$sweep" ]; then |
| 149 | echo "ERROR $r: rc=$rc, no parsable sweep verdict; see $log" | tee "$verdict" |
| 150 | return |
| 151 | fi |
| 152 | caught=0 |
| 153 | if [ "$sweep" != "sweep 0 of $SEEDS" ] || [ "$wit" = violated ]; then |
| 154 | caught=1 |
| 155 | fi |
| 156 | if [ $caught -eq 1 ]; then |
| 157 | if [ "$expect" = "missed" ]; then |
| 158 | echo "STALE $r: $sweep; witness $wit; but expected-outcomes.txt says missed" | tee "$verdict" |
| 159 | else |
| 160 | echo "CAUGHT $r: $sweep; witness $wit" | tee "$verdict" |
| 161 | fi |
| 162 | elif [ "$expect" = "missed" ]; then |
| 163 | echo "EXPECTED-MISS $r: $sweep; witness $wit" | tee "$verdict" |
| 164 | elif [ "$expect" = "rare" ] && [ -n "$depth" ] && [ "$SEEDS" -lt "$depth" ]; then |
| 165 | echo "EXPECTED-MISS $r: $sweep; witness $wit (rare: caught at $depth seeds)" | tee "$verdict" |
| 166 | else |
| 167 | echo "MISSED $r: $sweep; witness $wit" | tee "$verdict" |
| 168 | fi |
| 169 | } |
| 170 | export -f one |
| 171 | export SEEDS PKG SCRATCH BIN WRULES EXPECT |
| 172 | printf '%s\n' $RULES | xargs -P "${ENGINESIM_PAR:-1}" -I{} bash -c 'one "$@"' _ {} |
| 173 | xrc=$? |
| 174 | if [ $xrc -ne 0 ]; then |
| 175 | echo "sweep: orchestration failed (xargs rc=$xrc)" >&2 |
| 176 | exit 2 |
| 177 | fi |
| 178 | # exactly one well-formed verdict per requested mutation |
| 179 | want=$(printf '%s\n' $RULES | wc -l) |
| 180 | have=$(find "$SCRATCH/verdicts" -type f | wc -l) |
| 181 | if [ "$have" -ne "$want" ]; then |
| 182 | echo "sweep: $want mutation(s) requested, $have verdict(s) written" >&2 |
| 183 | exit 2 |
| 184 | fi |
| 185 | for r in $RULES; do |
| 186 | v="$SCRATCH/verdicts/$(echo "$r" | tr ',' '+')" |
| 187 | if ! grep -qE '^(CAUGHT|MISSED|EXPECTED-MISS|STALE|ERROR) ' "$v" 2>/dev/null; then |
| 188 | echo "sweep: no well-formed verdict for $r" >&2 |
| 189 | exit 2 |
| 190 | fi |
| 191 | done |
| 192 | bad=$(cat "$SCRATCH"/verdicts/* | grep -c "^ERROR\|^MISSED\|^STALE") |
| 193 | if [ "$bad" -gt 0 ]; then |
| 194 | echo "sweep: $bad rule(s) ERROR, MISSED or STALE" >&2 |
| 195 | exit 1 |
| 196 | fi |
| 197 | |
amadan