| 1 | """Re-run the six regressions the GPT-6 module review found surviving. |
| 2 | |
| 3 | Each mutation is applied to a copy of the tree, the suite is run, and the |
| 4 | mutation must FAIL the suite. A survivor means the tests still cannot tell. |
| 5 | """ |
| 6 | import io, os, shutil, subprocess, sys, tempfile |
| 7 | |
| 8 | ROOT = os.path.dirname(os.path.abspath(__file__)) |
| 9 | CACHE = os.path.join(ROOT, ".gocache") |
| 10 | |
| 11 | MUTATIONS = [ |
| 12 | ("11: non-2xx returns nothing instead of all-UNKNOWN", "s2.go", |
| 13 | "\tif status < 200 || status > 299 {\n\t\treturn allUnknown(len(asked))\n\t}", |
| 14 | "\tif status < 200 || status > 299 {\n\t\treturn nil\n\t}"), |
| 15 | |
| 16 | ("12a: the revision is ignored when matching answers to questions", "s2.go", |
| 17 | "\t\tpair := ItemRevision{ItemID: *r.ItemID, Revision: *r.Revision}", |
| 18 | "\t\tpair := ItemRevision{ItemID: *r.ItemID}"), |
| 19 | |
| 20 | ("12b: STALE is dropped from the closed vocabulary", "s2.go", |
| 21 | "\tcase VISIBLE, NOT_VISIBLE, STALE, UNKNOWN:", |
| 22 | "\tcase VISIBLE, NOT_VISIBLE, UNKNOWN:"), |
| 23 | |
| 24 | ("13: the batch cap is doubled", "subject.go", |
| 25 | "const MaxS2Pairs = 256", |
| 26 | "const MaxS2Pairs = 512"), |
| 27 | |
| 28 | ("14: a 403 loses its ErrContractBreach sentinel", "s3.go", |
| 29 | '\t\treturn nil, fmt.Errorf("%w: S3 answered 403 where the contract requires 404", ErrContractBreach)', |
| 30 | '\t\treturn nil, fmt.Errorf("S3 answered 403")'), |
| 31 | |
| 32 | ("10: redirects are followed", "transport.go", |
| 33 | '\t\t\treturn fmt.Errorf("suite: refusing to follow a redirect to %q", req.URL.Host)', |
| 34 | "\t\t\treturn nil"), |
| 35 | |
| 36 | ("1: the Host header override is not checked", "transport.go", |
| 37 | "\tif req.Host != \"\" {", |
| 38 | "\tif false {"), |
| 39 | |
| 40 | ("2: duplicate JSON keys are not rejected in S2", "s2.go", |
| 41 | "\tif err := rejectAmbiguousJSON(body); err != nil {\n\t\treturn allUnknown(len(asked))\n\t}", |
| 42 | "\tif false {\n\t\treturn allUnknown(len(asked))\n\t}"), |
| 43 | |
| 44 | ("9: hostname labels are not validated", "transport.go", |
| 45 | "\t\tif err := validLabel(label); err != nil {", |
| 46 | "\t\tif false {\n\t\t\t_ = label\n\t\tif err := error(nil); err != nil {"), |
| 47 | ] |
| 48 | |
| 49 | survivors = [] |
| 50 | for name, path, before, after in MUTATIONS: |
| 51 | work = tempfile.mkdtemp(prefix="mut-") |
| 52 | tree = os.path.join(work, "m") |
| 53 | # Only the module's own files: the worktree root also holds unreadable |
| 54 | # dotfiles that a whole-tree copy trips over. |
| 55 | os.makedirs(tree) |
| 56 | for entry in os.listdir(ROOT): |
| 57 | if entry.endswith(".go") or entry == "go.mod": |
| 58 | shutil.copy2(os.path.join(ROOT, entry), os.path.join(tree, entry)) |
| 59 | shutil.copytree(os.path.join(ROOT, "fixtures"), os.path.join(tree, "fixtures")) |
| 60 | shutil.copytree(os.path.join(ROOT, "conformance"), os.path.join(tree, "conformance")) |
| 61 | |
| 62 | target = os.path.join(tree, path) |
| 63 | src = io.open(target, encoding="utf-8").read() |
| 64 | if before not in src: |
| 65 | print("SKIP %s — anchor not found in %s" % (name, path)) |
| 66 | shutil.rmtree(work) |
| 67 | continue |
| 68 | io.open(target, "w", encoding="utf-8").write(src.replace(before, after, 1)) |
| 69 | |
| 70 | env = dict(os.environ, GOCACHE=CACHE) |
| 71 | r = subprocess.run(["go", "test", "./..."], cwd=tree, capture_output=True, text=True, env=env) |
| 72 | shutil.rmtree(work) |
| 73 | |
| 74 | if r.returncode == 0: |
| 75 | print("SURVIVED %s" % name) |
| 76 | survivors.append(name) |
| 77 | else: |
| 78 | print("caught %s" % name) |
| 79 | |
| 80 | print() |
| 81 | if survivors: |
| 82 | print("%d mutation(s) survived:" % len(survivors)) |
| 83 | for s in survivors: |
| 84 | print(" -", s) |
| 85 | sys.exit(1) |
| 86 | print("all %d mutations caught" % len(MUTATIONS)) |
| 87 | |
amadan