| index c5379b518..7d037e80d 100755 |
| --- a/internal/rollout/testdata/rolloutsim/mutate.sh |
| +++ b/internal/rollout/testdata/rolloutsim/mutate.sh |
| @@ -18,7 +18,14 @@ |
| # |
| # The script runs at nice 19 (it re-executes itself under nice if it has to, |
| # and warns if it cannot) and one run at a time with GOMAXPROCS=2 (the CPU |
| -# budget on shared boxes). |
| +# budget on shared boxes). Each go test run's timeout grows with the seeds, |
| +# SEEDS * 3 / 100 + 20 minutes (3000 seeds: 110 minutes); the header prints |
| +# it. A fixed 20 minutes was too short for 3000 seeds (round 1). |
| +# |
| +# The simulation's invariants are I1 to I10 (TestSimInvariants' doc). I9 |
| +# and I10 run only on the seeds whose failure book is on disk, which the |
| +# simulation turns off when it runs as root (root writes into a read-only |
| +# directory, so it cannot fail a save): as root, M13 to M15 read MISSED. |
| # |
| # Execution integrity is recorded apart from detection. A run must build, |
| # start, and end in its own --- PASS or --- FAIL line for the test it ran |
| @@ -32,7 +39,7 @@ |
| # Verdicts: |
| # CAUGHT the run executed intact and reported a violation of the |
| # mutant's own invariant ("rolloutsim: I2 violated at seed", |
| -# any of I1 to I8), or, for a unit-test mutant, its own test |
| +# any of I1 to I10), or, for a unit-test mutant, its own test |
| # ended in --- FAIL |
| # BROKEN the run panicked, timed out, crashed or never ran |
| # MISSED the run passed, or reported only other invariants' violations |
| @@ -63,9 +70,11 @@ fi |
| SEEDS=${1:-200} |
| shift $(($# > 0 ? 1 : 0)) |
| ONLY=" $* " |
| +# Each go test run's timeout, scaled with the seeds (3000 seeds: 110m). |
| +TIMEOUT_MIN=$((SEEDS * 3 / 100 + 20)) |
| |
| # name|file|check|perl substitution applied to the marked line only. check |
| -# is the simulation invariant that must catch the mutant (I1 to I8), or |
| +# is the simulation invariant that must catch the mutant (I1 to I10), or |
| # test:TestName for a unit-test mutant. The marker match is anchored (\b), |
| # so rolloutsim:M1 never matches rolloutsim:M10. The substitution is the |
| # last field, so it may contain "|". |
| @@ -89,6 +98,25 @@ ONLY=" $* " |
| # run of X can start inside that window; only an ordinary wake of a |
| # build the host served before, under another build's marker, leaves |
| # it open, and that needs X installed again and adopted on a hold. |
| +# |
| +# M13 to M16 are zero-downtime flag readiness round 2 task 2's (the |
| +# crash-epoch hard limit), each on a seed whose book is on disk: |
| +# M13 PublishedEpoch ignores its dirty check: I10. A change made while |
| +# saves fail moves the generation in memory only; status publishes |
| +# it, and the last landed file holds another. |
| +# M14 markLocked's marker set, renewed or replaced moves no generation: |
| +# I9. A grant issued under X's hold-ran marker, then another hold run |
| +# of X on the host (a first wake under the marker, ended after its |
| +# exec with no verdict) renews the marker at the same generation, and |
| +# the grant, delivered again, clears the newer run's marker. |
| +# M15 the reopen's seq keeps only the file's own seq, halved: I9. The |
| +# reopened book hands out again generations it had published, so a |
| +# grant issued before the restart matches a restriction a failure |
| +# after it made (reused numbers; the saved generations and markers |
| +# that would have repaired the max are not read). |
| +# M16 RetryHostAt skips the book-identity comparison: I9. A box rebuild |
| +# opens a new book whose generations start again from one, so a |
| +# grant for the old book matches a restriction the new one made. |
| MUTANTS='M1|queue.go|I2|s/if q\.running >= q\.concurrency \{/if false \&\& q.running >= q.concurrency {/ |
| M2|queue.go|I3|s/class := classRecover/class := classAdopt/ |
| M3|queue.go|I1|s/res, err = book\.Record\(rec\)/res, err = Recorded, nil/ |
| @@ -100,7 +128,11 @@ M8|failures.go|I5|s/cur\.Seq >= snap \{/cur.Seq > ^uint64(0) {/ |
| M9|failures.go|I7|s/if r\.Seq == 0 \{/if true {/ |
| M10|scheduler.go|I8|s/; err != nil \{/; errors.Is(err, ErrIntentStands) {/ |
| M11|queue.go|I7|s/if res == PreRetry \{/if false \&\& res == PreRetry {/ |
| -M12|failures.go|test:TestAnotherExecutionOfTheMarkedBuildRenewsTheMarker|s/h\.Seq = b\.bumpLocked\(\)/if ok \&\& cur.Build == h.Build { h.Seq = cur.Seq } else { h.Seq = b.bumpLocked() }/' |
| +M12|failures.go|test:TestAnotherExecutionOfTheMarkedBuildRenewsTheMarker|s/h\.Seq = b\.bumpLocked\(\)/if ok \&\& cur.Build == h.Build { h.Seq = cur.Seq } else { h.Seq = b.bumpLocked() }/ |
| +M13|failures.go|I10|s/if b\.dirty \{/if false \&\& b.dirty {/ |
| +M14|failures.go|I9|s/b\.touchLocked\(h\.Account, h\.App\)// |
| +M15|failures.go|I9|s/b\.seq = f\.highestSaved\(\)/b.seq = f.Seq \/ 2/ |
| +M16|failures.go|I9|s/ \|\| book != b\.idLocked\(\)//' |
| |
| # Every name selected must be a mutant in the list, checked before any work. |
| KNOWN=" $(printf '%s\n' "$MUTANTS" | cut -d'|' -f1 | tr '\n' ' ')" |
| @@ -142,6 +174,7 @@ mkdir -p "$SCRATCH/tree" "$SCRATCH/orig" "$SCRATCH/logs" "$SCRATCH/gowork" || ex |
| export GOTMPDIR=$SCRATCH/gowork |
| git -C "$ROOT" archive HEAD | tar -x -C "$SCRATCH/tree" || exit 2 |
| echo "tree: $(git -C "$ROOT" rev-parse --short HEAD)" |
| +echo "seeds: $SEEDS; timeout per run: ${TIMEOUT_MIN}m" |
| cp -a "$SCRATCH/tree/$P/." "$SCRATCH/orig/" || exit 2 |
| |
| # testOf CHECK: the test a mutant's run executes. |
| @@ -155,7 +188,7 @@ testOf() { |
| # run LOG TEST: one go test call of TEST against the scratch tree, verbose |
| # so that the test prints its --- PASS or --- FAIL line. |
| run() { |
| - (cd "$SCRATCH/tree" && ROLLOUT_SIM_SEEDS="$SEEDS" go test -v -p 1 -count=1 -timeout 20m \ |
| + (cd "$SCRATCH/tree" && ROLLOUT_SIM_SEEDS="$SEEDS" go test -v -p 1 -count=1 -timeout "${TIMEOUT_MIN}m" \ |
| -run "^$2\$" ./$P/) >"$1" 2>&1 |
| } |
| |
amadan