rastrillo / fiesta Public

Clone
git clone https://amadan.net/rastrillo/fiesta

Plain git — no account needed to clone.

Download

Download this file

1package fiesta
2
3import (
4 "encoding/json"
5 "go/ast"
6 "go/parser"
7 "go/token"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "regexp"
12 "sort"
13 "strconv"
14 "strings"
15 "testing"
16 "unicode"
17 "unicode/utf8"
18)
19
20// This file is rastrillo's amadan_ci_test.go, aimed at this repo's own
21// tree. .amadan/ci and .amadan/ci.d/ are what a step-reporting runner
22// executes, and nothing else gates them: a lost exec bit, or a ci.d
23// step that runs its own commands instead of naming a Makefile target,
24// resolves amadan's runner to "skipped", which renders as a pass. That
25// defect shipped five times in rastrillo before these checks existed;
26// Fiesta carries them from its first commit.
27
28// amadanCIDStepFiles lists .amadan/ci.d's entries, sorted, so every
29// check below walks the same list.
30func amadanCIDStepFiles(t *testing.T) []string {
31 t.Helper()
32 entries, err := os.ReadDir(".amadan/ci.d")
33 if err != nil {
34 t.Fatalf("reading .amadan/ci.d: %v", err)
35 }
36 var names []string
37 for _, e := range entries {
38 if e.IsDir() {
39 continue
40 }
41 names = append(names, e.Name())
42 }
43 sort.Strings(names)
44 if len(names) == 0 {
45 t.Fatal(".amadan/ci.d has no step files")
46 }
47 return names
48}
49
50// TestAmadanFilesAreExecutableInGit checks the executable bit the way
51// amadan's runner sees it: through git's index, not the working tree.
52// A filesystem-only check (os.Stat) would pass on a machine where the
53// bit is set locally but was never committed — the exact gap that lets
54// a non-executable step ship and resolve "skipped" (a pass) on every
55// other checkout.
56func TestAmadanFilesAreExecutableInGit(t *testing.T) {
57 out, err := exec.Command("git", "ls-files", "-s", ".amadan/").Output()
58 if err != nil {
59 t.Fatalf("git ls-files -s .amadan/: %v", err)
60 }
61 lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n")
62 if len(lines) == 0 || lines[0] == "" {
63 t.Fatal("git ls-files -s .amadan/ reported nothing tracked")
64 }
65
66 seen := map[string]bool{}
67 for _, line := range lines {
68 fields := strings.Fields(line)
69 if len(fields) < 4 {
70 t.Fatalf("unexpected `git ls-files -s` line: %q", line)
71 }
72 mode, path := fields[0], fields[3]
73 seen[path] = true
74 if mode != "100755" {
75 t.Errorf("%s is committed with mode %s, not 100755 — a runner reading the index (not the working tree) would resolve this job \"skipped\", which renders as a pass", path, mode)
76 }
77 }
78
79 want := []string{".amadan/ci"}
80 for _, name := range amadanCIDStepFiles(t) {
81 want = append(want, filepath.Join(".amadan/ci.d", name))
82 }
83 for _, w := range want {
84 if !seen[w] {
85 t.Errorf("%s is not tracked in git's index at all", w)
86 }
87 }
88}
89
90// execMakePattern anchors on the exact shape a ci.d step is allowed to
91// take: nothing but a call into the Makefile target of the same name.
92var execMakePattern = regexp.MustCompile(`^exec make ([A-Za-z0-9_-]+)$`)
93
94// amadanCIDTargets reads every .amadan/ci.d step, asserts each is a
95// bare `exec make <target>` (the Makefile's ci comment: "ci.d/ ...
96// never keeps its own copy of a command"), and returns the
97// step-file-name -> target map.
98func amadanCIDTargets(t *testing.T) map[string]string {
99 t.Helper()
100 targets := map[string]string{}
101 for _, name := range amadanCIDStepFiles(t) {
102 path := filepath.Join(".amadan/ci.d", name)
103 b, err := os.ReadFile(path)
104 if err != nil {
105 t.Fatalf("reading %s: %v", path, err)
106 }
107 var nonBlank []string
108 for _, l := range strings.Split(string(b), "\n") {
109 if strings.TrimSpace(l) != "" {
110 nonBlank = append(nonBlank, l)
111 }
112 }
113 if len(nonBlank) != 2 || nonBlank[0] != "#!/bin/sh" {
114 t.Errorf("%s must be exactly a `#!/bin/sh` shebang followed by one `exec make <target>` line; got:\n%s", path, b)
115 continue
116 }
117 m := execMakePattern.FindStringSubmatch(nonBlank[1])
118 if m == nil {
119 t.Errorf("%s's step is %q, not a bare `exec make <target>` — a step that runs its own command instead of a Makefile target is exactly the copy the Makefile says never to keep, and it can drift from what `make ci` actually runs without anything noticing", path, nonBlank[1])
120 continue
121 }
122 targets[name] = m[1]
123 }
124 return targets
125}
126
127// TestAmadanStepsAreBareExecMake is that assertion on its own, so a
128// broken step file fails here with the specific file and line named,
129// rather than only showing up as a set mismatch in the test below.
130func TestAmadanStepsAreBareExecMake(t *testing.T) {
131 amadanCIDTargets(t)
132}
133
134// ciPrereqPattern pulls `ci:`'s prerequisite list out of the Makefile,
135// following GNU Make's backslash-newline continuation so the
136// multi-line target list is captured whole.
137var ciPrereqPattern = regexp.MustCompile(`(?m)^ci:[ \t]*((?:.*\\\n)*.*)$`)
138
139// TestAmadanCITargetsMatchStepFiles enforces the promise the Makefile
140// makes and nothing else checks: add to `ci:` and to ci.d/ together,
141// or a step-reporting runner silently skips what you added. A target
142// added to `ci:` with no matching ci.d step never runs under amadan's
143// runner even though `make ci` covers it; a ci.d step naming a target
144// dropped from (or never added to) `ci:` reports a job that `make ci`
145// — the one gate — no longer executes. Either drift is silent without
146// this.
147func TestAmadanCITargetsMatchStepFiles(t *testing.T) {
148 mk, err := os.ReadFile("Makefile")
149 if err != nil {
150 t.Fatalf("reading Makefile: %v", err)
151 }
152 m := ciPrereqPattern.FindStringSubmatch(string(mk))
153 if m == nil {
154 t.Fatal("Makefile has no `ci:` target; `make ci` is the one gate every step file execs into")
155 }
156 joined := strings.ReplaceAll(m[1], "\\\n", " ")
157 ciTargets := map[string]bool{}
158 for _, f := range strings.Fields(joined) {
159 ciTargets[f] = true
160 }
161 if len(ciTargets) == 0 {
162 t.Fatal("Makefile's `ci:` target has no prerequisites")
163 }
164
165 stepTargets := map[string]bool{}
166 for step, target := range amadanCIDTargets(t) {
167 stepTargets[target] = true
168 if !ciTargets[target] {
169 t.Errorf(".amadan/ci.d/%s execs `make %s`, but %q is not one of ci:'s prerequisites in the Makefile — this step now reports a job `make ci` does not run", step, target, target)
170 }
171 }
172 for target := range ciTargets {
173 if !stepTargets[target] {
174 t.Errorf("Makefile's `ci:` target depends on %q, but no .amadan/ci.d/* step execs `make %s` — amadan's runner silently skips this half of `make ci`", target, target)
175 }
176 }
177}
178
179// TestAmadanSuiteShardsMatchRoles pins the suite step's @N to the
180// Makefile's SUITE. amadan starts N shards and each runs the role at
181// its index, so the two numbers are one fact written twice: a role
182// added without raising N runs nowhere while every shard reports ok.
183// hack/suite.sh refuses the mismatch at run time; this says so before
184// a runner is involved. It also refuses a role that is a ci
185// prerequisite as well, which would run it twice.
186func TestAmadanSuiteShardsMatchRoles(t *testing.T) {
187 if _, err := exec.LookPath("make"); err != nil {
188 t.Skip("make is not installed; the gate itself runs under make")
189 }
190 var step, shards string
191 for name, target := range amadanCIDTargets(t) {
192 if target != "suite" {
193 continue
194 }
195 if step != "" {
196 t.Fatalf("two .amadan/ci.d steps exec `make suite` (%s, %s); each would run every role", step, name)
197 }
198 step = name
199 if i := strings.LastIndex(name, "@"); i >= 0 {
200 shards = name[i+1:]
201 }
202 }
203 if step == "" {
204 t.Fatal("no .amadan/ci.d step execs `make suite`, so the suite's roles run nowhere under amadan")
205 }
206
207 out, err := exec.Command("make", "-s", "--no-print-directory", "suite-roles").Output()
208 if err != nil {
209 t.Fatalf("make suite-roles: %v", err)
210 }
211 roles := strings.Fields(string(out))
212 if len(roles) == 0 {
213 t.Fatal("make suite-roles printed no roles")
214 }
215 if want := strconv.Itoa(len(roles)); shards != want {
216 t.Errorf(".amadan/ci.d/%s runs as %q shards, but the Makefile's SUITE has %d roles; rename it to end in @%s", step, shards, len(roles), want)
217 }
218
219 mk, err := os.ReadFile("Makefile")
220 if err != nil {
221 t.Fatalf("reading Makefile: %v", err)
222 }
223 m := ciPrereqPattern.FindStringSubmatch(string(mk))
224 if m == nil {
225 t.Fatal("Makefile has no `ci:` target")
226 }
227 ci := map[string]bool{}
228 for _, f := range strings.Fields(strings.ReplaceAll(m[1], "\\\n", " ")) {
229 ci[f] = true
230 }
231 seen := map[string]bool{}
232 for _, r := range roles {
233 if seen[r] {
234 t.Errorf("SUITE lists %q twice; two shards would run it", r)
235 }
236 seen[r] = true
237 if ci[r] {
238 t.Errorf("%q is both a suite role and a prerequisite of ci, so make ci runs it twice", r)
239 }
240 }
241}
242
243// sliceRolePattern reads a sliced role: <mode>.<pkg>~<k>of<n>, mode
244// browser or test.
245var sliceRolePattern = regexp.MustCompile(`^(browser|test)\.(.+)~([0-9]+)of([0-9]+)$`)
246
247// TestSlicesPartitionTheirPackages is what lets a package run under
248// -run at all: a filter that matches nothing exits 0, and a slice is a
249// -run too, so this proves the slices cannot leave a test out. Every
250// slice 1..n of a package is a suite role, and hack/gotest.sh's deal
251// puts each name in exactly one slice and leaves no slice empty. The
252// names are every weighted one plus synthetic ones, so this compiles
253// nothing; the real list is dealt by the same function at run time.
254// It skips while no package is sliced.
255func TestSlicesPartitionTheirPackages(t *testing.T) {
256 if _, err := exec.LookPath("make"); err != nil {
257 t.Skip("make is not installed; the gate itself runs under make")
258 }
259 out, err := exec.Command("make", "-s", "--no-print-directory", "suite-roles").Output()
260 if err != nil {
261 t.Fatalf("make suite-roles: %v", err)
262 }
263 type sliced struct{ mode, pkg string }
264 slices := map[sliced]map[int]bool{}
265 counts := map[sliced]int{}
266 for _, r := range strings.Fields(string(out)) {
267 m := sliceRolePattern.FindStringSubmatch(r)
268 if m == nil {
269 continue
270 }
271 k, _ := strconv.Atoi(m[3])
272 n, _ := strconv.Atoi(m[4])
273 key := sliced{m[1], strings.ReplaceAll(m[2], ".", "/")}
274 if prev, ok := counts[key]; ok && prev != n {
275 t.Fatalf("%s %s is sliced both %d and %d ways", key.mode, key.pkg, prev, n)
276 }
277 counts[key] = n
278 if slices[key] == nil {
279 slices[key] = map[int]bool{}
280 }
281 slices[key][k] = true
282 }
283 if len(counts) == 0 {
284 t.Skip("no package is sliced")
285 }
286
287 weights, err := os.ReadFile("hack/test-weights.txt")
288 if err != nil {
289 t.Fatal(err)
290 }
291 for key, n := range counts {
292 name := key.mode + " " + key.pkg
293 for k := 1; k <= n; k++ {
294 if !slices[key][k] {
295 t.Errorf("%s is sliced %d ways but slice %d is no suite role, so its tests run nowhere", name, n, k)
296 }
297 }
298
299 var names []string
300 for _, l := range strings.Split(string(weights), "\n") {
301 if f := strings.Fields(l); len(f) == 4 && f[0] == key.mode && f[1] == key.pkg {
302 names = append(names, f[2])
303 }
304 }
305 for i := 0; i < 5*n; i++ {
306 names = append(names, "TestUnweighted"+strconv.Itoa(i))
307 }
308 deal := exec.Command("sh", "hack/gotest.sh", "--deal", key.mode, key.pkg, strconv.Itoa(n))
309 deal.Stdin = strings.NewReader(strings.Join(names, "\n") + "\n")
310 got, err := deal.Output()
311 if err != nil {
312 t.Fatalf("dealing %s: %v", name, err)
313 }
314 dealt := map[string]int{}
315 used := map[int]bool{}
316 for _, l := range strings.Split(strings.TrimSpace(string(got)), "\n") {
317 f := strings.Fields(l)
318 if len(f) != 2 {
319 t.Fatalf("deal line %q is not `<k> <name>`", l)
320 }
321 k, err := strconv.Atoi(f[0])
322 if err != nil || k < 1 || k > n {
323 t.Errorf("%s: %s was dealt to slice %q, outside 1..%d", name, f[1], f[0], n)
324 }
325 used[k] = true
326 dealt[f[1]]++
327 }
328 for _, nm := range names {
329 if dealt[nm] != 1 {
330 t.Errorf("%s: %s was dealt %d times, not once", name, nm, dealt[nm])
331 }
332 }
333 if len(dealt) != len(names) {
334 t.Errorf("%s: dealt %d names from %d", name, len(dealt), len(names))
335 }
336 for k := 1; k <= n; k++ {
337 if !used[k] {
338 t.Errorf("%s: slice %d of %d was dealt nothing from %d names", name, k, n, len(names))
339 }
340 }
341 }
342}
343
344// testNames reads the tests, examples and fuzz targets go test would
345// find in each package of ./..., built with tags (none, or browser):
346// import path to the set of names. It parses the files go list says
347// the build includes rather than compiling anything, so it costs a
348// second, not a build of every test binary.
349func testNames(t *testing.T, tags string) map[string]map[string]bool {
350 t.Helper()
351 args := []string{"list", "-e", "-json"}
352 if tags != "" {
353 args = append(args, "-tags", tags)
354 }
355 out, err := exec.Command("go", append(args, "./...")...).Output()
356 if err != nil {
357 t.Fatalf("go %s ./...: %v", strings.Join(args, " "), err)
358 }
359 names := map[string]map[string]bool{}
360 dec := json.NewDecoder(strings.NewReader(string(out)))
361 for dec.More() {
362 var pkg struct {
363 ImportPath, Dir string
364 TestGoFiles, XTestGoFiles []string
365 Error *struct{ Err string }
366 }
367 if err := dec.Decode(&pkg); err != nil {
368 t.Fatal(err)
369 }
370 // A package whose files the tags exclude entirely is not a
371 // package of this build; any other error is a broken tree.
372 if pkg.Error != nil {
373 if strings.Contains(pkg.Error.Err, "build constraints exclude all Go files") {
374 continue
375 }
376 t.Fatalf("go list %s: %s", pkg.ImportPath, pkg.Error.Err)
377 }
378 set := map[string]bool{}
379 for _, f := range append(pkg.TestGoFiles, pkg.XTestGoFiles...) {
380 file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(pkg.Dir, f), nil, parser.SkipObjectResolution)
381 if err != nil {
382 t.Fatal(err)
383 }
384 for _, d := range file.Decls {
385 fn, ok := d.(*ast.FuncDecl)
386 if !ok || fn.Recv != nil {
387 continue
388 }
389 for _, prefix := range []string{"Test", "Example", "Fuzz"} {
390 rest, ok := strings.CutPrefix(fn.Name.Name, prefix)
391 if r, _ := utf8.DecodeRuneInString(rest); ok && (rest == "" || !unicode.IsLower(r)) && fn.Name.Name != "TestMain" {
392 set[fn.Name.Name] = true
393 }
394 }
395 }
396 }
397 names[pkg.ImportPath] = set
398 }
399 return names
400}
401
402// TestEveryTestRunsInExactlyOneRole holds the suite's roles to running
403// each test once. A package's plain tests (those the untagged build
404// has) run in root, or in its test.<pkg> roles where ROOT_SPLIT names
405// it; its browser tests (those only the browser build has) run in its
406// browser roles, which is hack/gotest.sh's rule for a browser role. So
407// every package with a browser test must be a browser package, every
408// browser package must have one, every split package must exist, and
409// root must be a role. That a sliced package's slices partition it is
410// TestSlicesPartitionTheirPackages.
411func TestEveryTestRunsInExactlyOneRole(t *testing.T) {
412 if _, err := exec.LookPath("make"); err != nil {
413 t.Skip("make is not installed; the gate itself runs under make")
414 }
415 out, err := exec.Command("make", "-s", "--no-print-directory", "suite-roles").Output()
416 if err != nil {
417 t.Fatalf("make suite-roles: %v", err)
418 }
419 modPath := "amadan.net/rastrillo/fiesta"
420 pathOf := func(role, mode string) string {
421 rel := strings.TrimPrefix(role, mode+".")
422 rel, _, _ = strings.Cut(rel, "~")
423 return modPath + "/" + strings.ReplaceAll(rel, ".", "/")
424 }
425 root := false
426 browserRoles, testRoles := map[string]bool{}, map[string]bool{}
427 for _, r := range strings.Fields(string(out)) {
428 switch {
429 case r == "root":
430 root = true
431 case strings.HasPrefix(r, "browser."):
432 browserRoles[pathOf(r, "browser")] = true
433 case strings.HasPrefix(r, "test."):
434 testRoles[pathOf(r, "test")] = true
435 }
436 }
437 if !root {
438 t.Fatal("the suite has no root role, so the plain tests of every unsplit package run nowhere")
439 }
440
441 plain, tagged := testNames(t, ""), testNames(t, "browser")
442 for pkg := range testRoles {
443 if _, ok := plain[pkg]; !ok {
444 t.Errorf("%s has test roles but the untagged build has no such package", pkg)
445 }
446 }
447 pkgs := map[string]bool{}
448 for pkg := range plain {
449 pkgs[pkg] = true
450 }
451 for pkg := range tagged {
452 pkgs[pkg] = true
453 }
454 ran := 0
455 for pkg := range pkgs {
456 browserOnly := 0
457 names := map[string]bool{}
458 for n := range plain[pkg] {
459 names[n] = true
460 }
461 for n := range tagged[pkg] {
462 names[n] = true
463 }
464 for n := range names {
465 roles := 0
466 if plain[pkg][n] {
467 roles++ // root, or the package's test roles
468 }
469 if tagged[pkg][n] && !plain[pkg][n] {
470 browserOnly++
471 if browserRoles[pkg] {
472 roles++
473 }
474 }
475 if roles != 1 {
476 t.Errorf("%s %s runs in %d roles, not one", pkg, n, roles)
477 }
478 ran += roles
479 }
480 if browserRoles[pkg] && browserOnly == 0 {
481 t.Errorf("%s has browser roles but no test only the browser build has; its roles would run nothing", pkg)
482 }
483 }
484 for pkg := range browserRoles {
485 if !pkgs[pkg] {
486 t.Errorf("%s has browser roles but no such package", pkg)
487 }
488 }
489 if ran < 500 {
490 t.Errorf("only %d tests found across the suite; the reading has stopped seeing them", ran)
491 }
492 t.Logf("%d tests, each in one role", ran)
493}
494