| 1 | #!/bin/bash |
| 2 | # Mutation checks for the hostlife model (move-any-app spec §14.2). |
| 3 | # |
| 4 | # Build the test binary once (this is the only step that writes the Go |
| 5 | # build cache), then run the sweep. The sweep never copies or edits the |
| 6 | # source: each rule runs the same binary with HOSTLIFE_MUTATE set. |
| 7 | # |
| 8 | # go test -c -o "$TMPDIR/hostlife-sim/hostlife.test" ./internal/sim/hostlife/ |
| 9 | # internal/sim/hostlife/mutate.sh 3000 # every rule |
| 10 | # internal/sim/hostlife/mutate.sh 30000 rule-a rule-b # named rules |
| 11 | # internal/sim/hostlife/mutate.sh 30000 rule-a,rule-b # a combination, dropped together |
| 12 | # HOSTLIFE_BOXES=2 internal/sim/hostlife/mutate.sh 3000 # the two-box shape (region HA L18); unset is one box per region |
| 13 | # HOSTLIFE_BOXES=2 internal/sim/hostlife/mutate.sh 3000 @regionha # the region HA rules alone (rules_test.go's region HA block) |
| 14 | # HOSTLIFE_LIST=1 internal/sim/hostlife/mutate.sh 0 @regionha # print the rules a run would check, and run nothing |
| 15 | # |
| 16 | # Each rule gets two runs under its mutation (rev 7.3): the random sweep |
| 17 | # (TestHostLifecycle over the seeds) and the rule's deterministic witness |
| 18 | # scenarios (TestWitnesses, witness_test.go). The verdict line reports |
| 19 | # both, separately: |
| 20 | # |
| 21 | # CAUGHT rule: sweep N of S (first seed n); witness violated|held|none |
| 22 | # |
| 23 | # Classification (joint round 2, R2-10; rev 7.3): |
| 24 | # |
| 25 | # CAUGHT the sweep completed and reported at least one seed that |
| 26 | # violated an invariant, with the seed number parsed from its |
| 27 | # output (so it can be replayed with HOSTLIFE_SEED), OR a |
| 28 | # witness of the rule violated an invariant under the mutation; |
| 29 | # MISSED the sweep completed with every seed passing, and every |
| 30 | # witness held (or the rule has none); |
| 31 | # ERROR anything else: an unknown rule, a stale binary, a panic, a |
| 32 | # timeout, output the harness cannot parse, or a witness run |
| 33 | # that failed (a witness that must violate under this mutation |
| 34 | # and held: the witness is broken). An error is never a |
| 35 | # detection. |
| 36 | # |
| 37 | # Expected outcomes live in expected-outcomes.txt beside this script: a |
| 38 | # rule listed there as `missed` (liveness-only, unreachable, or caught |
| 39 | # only in a stated combination: its witness holds alone) is reported as |
| 40 | # EXPECTED-MISS. A rule listed as `rare <N>` (rev 7.11) has no witness |
| 41 | # and is caught by the random sweep only at N seeds or more: below N a |
| 42 | # miss is EXPECTED-MISS and a catch is CAUGHT; at N or more a miss is |
| 43 | # MISSED. The sweep exits nonzero on any ERROR, on any unexpected MISSED, |
| 44 | # and on any rule listed as `missed` that is CAUGHT (STALE). |
| 45 | # At HOSTLIFE_BOXES=2 expected-outcomes.two-box.txt overrides it, rule by |
| 46 | # rule: a line there replaces the rule's line from the base file, and |
| 47 | # `rule - reason` drops the base line (the rule must then be caught). |
| 48 | # Unset or 1, the base file is read as it is. |
| 49 | # |
| 50 | # Scratch (the binary, every run's log, the verdicts) lives under |
| 51 | # ${HOSTLIFE_SCRATCH:-$TMPDIR/hostlife-sim}; remove it once, after the |
| 52 | # whole sweep. |
| 53 | set -u |
| 54 | SEEDS=${1:?seeds} |
| 55 | shift |
| 56 | ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) |
| 57 | PKG="$ROOT/internal/sim/hostlife" |
| 58 | SCRATCH=${HOSTLIFE_SCRATCH:-${TMPDIR:?}/hostlife-sim} |
| 59 | BIN=${HOSTLIFE_BIN:-$SCRATCH/hostlife.test} |
| 60 | SHAPE=${HOSTLIFE_BOXES:-1} |
| 61 | RULES="$*" |
| 62 | if [ -z "$RULES" ]; then |
| 63 | RULES=$(sed -n '/^var allRules/,/^}/p' "$PKG/rules_test.go" | grep -o '^\s*"[a-z0-9-]*"' | tr -d ' "\t') |
| 64 | elif [ "$RULES" = "@regionha" ]; then |
| 65 | # every rule of the region HA block: one mutation check per guard |
| 66 | RULES=$(sed -n '/^\t\/\/ region HA (two boxes per region, L18)/,/^}/p' "$PKG/rules_test.go" | grep -o '^\s*"[a-z0-9-]*"' | tr -d ' "\t') |
| 67 | fi |
| 68 | if [ -n "${HOSTLIFE_LIST:-}" ]; then |
| 69 | printf '%s\n' $RULES |
| 70 | exit 0 |
| 71 | fi |
| 72 | mkdir -p "$SCRATCH/logs" "$SCRATCH/verdicts" || { echo "cannot create $SCRATCH" >&2; exit 2; } |
| 73 | EXPECT="$PKG/expected-outcomes.txt" |
| 74 | EXPECT_NAME=expected-outcomes.txt # the file the verdicts cite |
| 75 | if [ "$SHAPE" = 2 ] && [ -f "$PKG/expected-outcomes.two-box.txt" ]; then |
| 76 | EXPECT="$SCRATCH/expected-outcomes.merged" |
| 77 | EXPECT_NAME="expected-outcomes.txt merged with expected-outcomes.two-box.txt" |
| 78 | awk 'FNR==NR { if ($0 !~ /^#/ && NF) ov[$1]=$0; next } !($1 in ov)' "$PKG/expected-outcomes.two-box.txt" "$PKG/expected-outcomes.txt" >"$EXPECT" |
| 79 | awk '$0 !~ /^#/ && NF && $2 != "-"' "$PKG/expected-outcomes.two-box.txt" >>"$EXPECT" |
| 80 | fi |
| 81 | if [ ! -x "$BIN" ]; then |
| 82 | echo "no test binary at $BIN; build it with: go test -c -o $BIN ./internal/sim/hostlife/" >&2 |
| 83 | exit 2 |
| 84 | fi |
| 85 | if [ -n "$(find "$PKG" -name '*.go' -newer "$BIN")" ]; then |
| 86 | echo "the test binary $BIN is older than the source; rebuild it" >&2 |
| 87 | exit 2 |
| 88 | fi |
| 89 | # the rules that have witnesses (witness_test.go): a run of one of them |
| 90 | # with no witness output is an ERROR, never "none" (R3-6) |
| 91 | WRULES=$(grep -o '^\s*{"[a-z0-9-]*",' "$PKG/witness_test.go" | tr -d ' \t{",' | sort -u) |
| 92 | rm -f "$SCRATCH"/verdicts/* || exit 2 |
| 93 | |
| 94 | one() { |
| 95 | r=$1 |
| 96 | tag=$(echo "$r" | tr ',' '+') |
| 97 | log="$SCRATCH/logs/$tag.log" |
| 98 | verdict="$SCRATCH/verdicts/$tag" |
| 99 | for x in $(echo "$r" | tr ',' ' '); do |
| 100 | if ! sed -n '/^var allRules/,/^}/p' "$PKG/rules_test.go" | grep -q "^\s*\"$x\","; then |
| 101 | echo "ERROR $r [boxes=$SHAPE]: no rule named $x" | tee "$verdict" |
| 102 | return |
| 103 | fi |
| 104 | done |
| 105 | # the random sweep |
| 106 | HOSTLIFE_MUTATE=$r HOSTLIFE_SEEDS=$SEEDS "$BIN" -test.run '^TestHostLifecycle$' -test.count=1 -test.v -test.timeout 180m >"$log" 2>&1 |
| 107 | rc=$? |
| 108 | expect=$(awk -v r="$r" '$1==r {print $2}' "$EXPECT" 2>/dev/null) |
| 109 | depth=$(awk -v r="$r" '$1==r {print $3}' "$EXPECT" 2>/dev/null) |
| 110 | nviol=$(grep -o "[0-9]* of $SEEDS seeds violated an invariant" "$log" | head -1 | awk '{print $1}') |
| 111 | seed=$(grep -o "first seed [0-9]*" "$log" | head -1 | awk '{print $3}') |
| 112 | clean=$(grep -c "^ sim_test.go:[0-9]*: $SEEDS seeds, 0 failed.*mutated=\"$r\"" "$log") |
| 113 | sweep="" |
| 114 | if [ $rc -ne 0 ] && [ -n "$nviol" ] && [ -n "$seed" ] && ! grep -q "^panic:\|test timed out" "$log"; then |
| 115 | sweep="sweep $nviol of $SEEDS (first seed $seed)" |
| 116 | elif [ $rc -eq 0 ] && [ "$clean" -eq 1 ]; then |
| 117 | sweep="sweep 0 of $SEEDS" |
| 118 | fi |
| 119 | # the witnesses (rev 7.3): the rule's own hand-scheduled scenarios, run |
| 120 | # under the same mutation; "violated" if any of them violates an |
| 121 | # invariant, "held" if they all hold, "none" if the rule has no witness. |
| 122 | # A witness that must violate under this mutation and holds fails the |
| 123 | # test, which is an ERROR here: the witness is broken, not the rule |
| 124 | wlog="$SCRATCH/logs/$tag.witness.log" |
| 125 | HOSTLIFE_MUTATE=$r "$BIN" -test.run '^TestWitnesses$' -test.count=1 -test.v -test.timeout 10m >"$wlog" 2>&1 |
| 126 | wrc=$? |
| 127 | wit=none |
| 128 | if grep -q "^ witness_test.go:[0-9]*: witness .*: violated" "$wlog"; then |
| 129 | wit=violated |
| 130 | elif grep -q "^ witness_test.go:[0-9]*: witness .*: held" "$wlog"; then |
| 131 | wit=held |
| 132 | fi |
| 133 | if [ $wrc -ne 0 ] || grep -q "^panic:\|test timed out" "$wlog"; then |
| 134 | echo "ERROR $r [boxes=$SHAPE]: witness run rc=$wrc; see $wlog" | tee "$verdict" |
| 135 | return |
| 136 | fi |
| 137 | if [ "$wit" = none ]; then |
| 138 | for x in $(echo "$r" | tr ',' ' '); do |
| 139 | if echo "$WRULES" | grep -qx "$x"; then |
| 140 | echo "ERROR $r [boxes=$SHAPE]: $x has witnesses but the run produced no witness output; see $wlog" | tee "$verdict" |
| 141 | return |
| 142 | fi |
| 143 | done |
| 144 | fi |
| 145 | if [ -z "$sweep" ]; then |
| 146 | echo "ERROR $r [boxes=$SHAPE]: rc=$rc, no parsable sweep verdict; see $log" | tee "$verdict" |
| 147 | return |
| 148 | fi |
| 149 | caught=0 |
| 150 | if [ "$sweep" != "sweep 0 of $SEEDS" ] || [ "$wit" = violated ]; then |
| 151 | caught=1 |
| 152 | fi |
| 153 | if [ $caught -eq 1 ]; then |
| 154 | if [ "$expect" = "missed" ]; then |
| 155 | echo "STALE $r [boxes=$SHAPE]: $sweep; witness $wit; but $EXPECT_NAME says missed" | tee "$verdict" |
| 156 | else |
| 157 | echo "CAUGHT $r [boxes=$SHAPE]: $sweep; witness $wit" | tee "$verdict" |
| 158 | fi |
| 159 | elif [ "$expect" = "missed" ]; then |
| 160 | echo "EXPECTED-MISS $r [boxes=$SHAPE]: $sweep; witness $wit" | tee "$verdict" |
| 161 | elif [ "$expect" = "rare" ] && [ -n "$depth" ] && [ "$SEEDS" -lt "$depth" ]; then |
| 162 | echo "EXPECTED-MISS $r [boxes=$SHAPE]: $sweep; witness $wit (rare: caught at $depth seeds)" | tee "$verdict" |
| 163 | else |
| 164 | echo "MISSED $r [boxes=$SHAPE]: $sweep; witness $wit" | tee "$verdict" |
| 165 | fi |
| 166 | } |
| 167 | export -f one |
| 168 | export SEEDS PKG SCRATCH BIN WRULES SHAPE EXPECT EXPECT_NAME |
| 169 | printf '%s\n' $RULES | xargs -P "${HOSTLIFE_PAR:-8}" -I{} bash -c 'one "$@"' _ {} |
| 170 | xrc=$? |
| 171 | if [ $xrc -ne 0 ]; then |
| 172 | echo "sweep: orchestration failed (xargs rc=$xrc)" >&2 |
| 173 | exit 2 |
| 174 | fi |
| 175 | # exactly one well-formed verdict per requested mutation (R3-6) |
| 176 | want=$(printf '%s\n' $RULES | wc -l) |
| 177 | have=$(ls "$SCRATCH"/verdicts 2>/dev/null | wc -l) |
| 178 | if [ "$have" -ne "$want" ]; then |
| 179 | echo "sweep: $want mutation(s) requested, $have verdict(s) written" >&2 |
| 180 | exit 2 |
| 181 | fi |
| 182 | for r in $RULES; do |
| 183 | v="$SCRATCH/verdicts/$(echo "$r" | tr ',' '+')" |
| 184 | if ! grep -qE '^(CAUGHT|MISSED|EXPECTED-MISS|STALE|ERROR) ' "$v" 2>/dev/null; then |
| 185 | echo "sweep: no well-formed verdict for $r" >&2 |
| 186 | exit 2 |
| 187 | fi |
| 188 | done |
| 189 | bad=$(cat "$SCRATCH"/verdicts/* | grep -c "^ERROR\|^MISSED\|^STALE") |
| 190 | if [ "$bad" -gt 0 ]; then |
| 191 | echo "sweep: $bad rule(s) ERROR, MISSED or STALE" >&2 |
| 192 | exit 1 |
| 193 | fi |
| 194 | |
amadan