amadan
e2e: one top-level test per boundary so the gate's shard deal can spread the recovery matrix; the e2e default is four shards
TestEveryBoundaryRecovers ran the whole boundary × fault × second-crash matrix as one test, 60% of the e2e suite's time (1492 s under load on 2026-10-23), and the gate deals tests across shards by name, so that shard alone could outlast CI's 30-minute job whatever E2E_SHARDS was. Each boundary now has its own TestBoundaryRecovers<Name>; TestEveryBoundaryRecovers keeps the plan's name, checks every boundary has its test (and no test names a boundary that is gone), and keeps the ordinary-progression check. The gate's E2E_SHARDS default rises from 2 to 4: the witnesses mostly wait on ticks.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2 files changed,
+93
−19
.amadan/test-gate.sh+6 −3internal/e2e/selfserve/job_boundary_test.go+87 −16
CI failed — run details
CI log
=== gate === .amadan/test-gate.sh amadan: step gate failed: signal: terminated amadan: job failed at step gate — its output is above, not at the tail of this log
diff --git a/.amadan/test-gate.sh b/.amadan/test-gate.sh| index 9e408df39..328a2ee84 100755 |
| --- a/.amadan/test-gate.sh |
| +++ b/.amadan/test-gate.sh |
| @@ -235,7 +235,10 @@ wait |
| # each witness runs two boxes and a console in one process, and when |
| # they ran beside the console shards the runner killed the gate (five |
| # runs on 2026-10-23: three bare 30-minute timeouts, two terminations). |
| -# Alone they take under ten minutes across two shards. Discovery first, |
| +# They are dealt across four shards (E2E_SHARDS): the witnesses mostly wait |
| +# on ticks, so the shards cost little CPU, and no single test may dominate a |
| +# shard (the boundary matrix is one top-level test per boundary for that |
| +# reason). Discovery first, |
| # as for the split packages: a witness package that lists no test fails |
| # the gate rather than passing with nothing run. |
| if ! discover go test -count=1 -tags e2e -list '.*' ./internal/e2e/selfserve >"$out/list-e2e.raw" 2>&1; then |
| @@ -252,10 +255,10 @@ else |
| # whose pattern matches nothing fails the gate. |
| grep -E '^Test' "$out/list-e2e.raw" | sort -u >"$out/list-e2e" || true |
| listed=$(wc -l <"$out/list-e2e") |
| - awk -v n="${E2E_SHARDS:-2}" '{ print (NR - 1) % n "\t" $0 }' "$out/list-e2e" >"$out/deal-e2e" |
| + awk -v n="${E2E_SHARDS:-4}" '{ print (NR - 1) % n "\t" $0 }' "$out/list-e2e" >"$out/deal-e2e" |
| i=0 |
| launched=0 |
| - while [ "$i" -lt "${E2E_SHARDS:-2}" ]; do |
| + while [ "$i" -lt "${E2E_SHARDS:-4}" ]; do |
| pattern=$(awk -v i="$i" -F '\t' '$1 == i { printf "%s%s", sep, $2; sep = "|" }' "$out/deal-e2e") |
| if [ -n "$pattern" ]; then |
| start "e2e@$i" go test -count=1 -timeout 30m -tags e2e -run "^($pattern)\$" ./internal/e2e/selfserve |
diff --git a/internal/e2e/selfserve/job_boundary_test.go b/internal/e2e/selfserve/job_boundary_test.go| index 5e328fe43..0b80114af 100644 |
| --- a/internal/e2e/selfserve/job_boundary_test.go |
| +++ b/internal/e2e/selfserve/job_boundary_test.go |
| @@ -117,32 +117,103 @@ var jobFaults = []jobFault{ |
| {name: "commit after the context was cancelled", point: "written", put: putHeld}, |
| } |
| -// TestEveryBoundaryRecovers is Review Focus 1 for the job's writes before |
| -// the barrier (queue, staging, restage, precheck, estimate, the maintenance |
| -// round, the queued abort, the abort, its finish, the cut's restage): at |
| -// every boundary a crash before the request, with it persisted, after the |
| -// write, before the confirmation, before the bump and before notified_seq, |
| -// a lost reply, and a store that commits the write after the client's |
| -// context was cancelled; each crash recovered by a console that reopens the |
| -// same SQLite file, with and without a second crash during the recovery. |
| -// The row reconciles forward and never regresses, the request in flight |
| -// keeps its persisted identity (no second request of its kind is issued |
| -// while it is unconfirmed), the late commit is confirmed without a second |
| -// write, and the serial is bumped for the landed write. |
| -func TestEveryBoundaryRecovers(t *testing.T) { |
| +// The witnesses below are Review Focus 1 for the job's writes before the |
| +// barrier (queue, staging, restage, precheck, estimate, the maintenance |
| +// round, the queued abort, the abort, its finish, the cut's restage, and the |
| +// cut's own writes): at every boundary a crash before the request, with it |
| +// persisted, after the write, before the confirmation, before the bump and |
| +// before notified_seq, a lost reply, and a store that commits the write |
| +// after the client's context was cancelled; each crash recovered by a |
| +// console that reopens the same SQLite file, with and without a second |
| +// crash during the recovery. The row reconciles forward and never |
| +// regresses, the request in flight keeps its persisted identity (no second |
| +// request of its kind is issued while it is unconfirmed), the late commit |
| +// is confirmed without a second write, and the serial is bumped for the |
| +// landed write. |
| +// |
| +// One top-level test per boundary, not one test with a boundary loop: the |
| +// gate deals the package's tests across shards by name, and the whole |
| +// matrix in one test was 60% of the suite's time in one shard (2026-10-23, |
| +// 1492 s under load), past CI's 30-minute job. TestEveryBoundaryRecovers |
| +// keeps the plan's name and pins the matrix complete: every boundary has |
| +// its test, and the ordinary progression bumps the serial once per landed |
| +// write. |
| +func boundaryRecovers(t *testing.T, name string) { |
| for _, b := range jobBoundaries { |
| + if b.name != name { |
| + continue |
| + } |
| for _, f := range jobFaults { |
| for _, second := range []bool{false, true} { |
| if second && f.point == "" { |
| continue |
| } |
| - name := b.name + "/" + f.name |
| + n := f.name |
| if second { |
| - name += "/second crash" |
| + n += "/second crash" |
| } |
| - t.Run(name, func(t *testing.T) { everyBoundary(t, b, f, second) }) |
| + t.Run(n, func(t *testing.T) { everyBoundary(t, b, f, second) }) |
| } |
| } |
| + return |
| + } |
| + t.Fatalf("no boundary named %q in jobBoundaries", name) |
| +} |
| + |
| +func TestBoundaryRecoversQueue(t *testing.T) { boundaryRecovers(t, "queue") } |
| +func TestBoundaryRecoversStaging(t *testing.T) { boundaryRecovers(t, "staging") } |
| +func TestBoundaryRecoversRestage(t *testing.T) { boundaryRecovers(t, "restage") } |
| +func TestBoundaryRecoversPrecheck(t *testing.T) { boundaryRecovers(t, "precheck") } |
| +func TestBoundaryRecoversEstimate(t *testing.T) { boundaryRecovers(t, "estimate") } |
| +func TestBoundaryRecoversMaintain(t *testing.T) { boundaryRecovers(t, "maintain") } |
| +func TestBoundaryRecoversAbortQueued(t *testing.T) { boundaryRecovers(t, "abort-queued") } |
| +func TestBoundaryRecoversAbort(t *testing.T) { boundaryRecovers(t, "abort") } |
| +func TestBoundaryRecoversFinish(t *testing.T) { boundaryRecovers(t, "finish") } |
| +func TestBoundaryRecoversCutRestage(t *testing.T) { boundaryRecovers(t, "cut-restage") } |
| +func TestBoundaryRecoversStart(t *testing.T) { boundaryRecovers(t, "start") } |
| +func TestBoundaryRecoversSeal(t *testing.T) { boundaryRecovers(t, "seal") } |
| +func TestBoundaryRecoversFlip(t *testing.T) { boundaryRecovers(t, "flip") } |
| +func TestBoundaryRecoversOpen(t *testing.T) { boundaryRecovers(t, "open") } |
| +func TestBoundaryRecoversFinishGate1(t *testing.T) { boundaryRecovers(t, "finish-gate1") } |
| +func TestBoundaryRecoversFinishDone(t *testing.T) { boundaryRecovers(t, "finish-done") } |
| +func TestBoundaryRecoversAbortAfterStart(t *testing.T) { boundaryRecovers(t, "abort-after-start") } |
| + |
| +// boundaryTests names the top-level test of every boundary; a boundary |
| +// added to jobBoundaries without one fails TestEveryBoundaryRecovers. |
| +var boundaryTests = map[string]func(*testing.T){ |
| + "queue": TestBoundaryRecoversQueue, |
| + "staging": TestBoundaryRecoversStaging, |
| + "restage": TestBoundaryRecoversRestage, |
| + "precheck": TestBoundaryRecoversPrecheck, |
| + "estimate": TestBoundaryRecoversEstimate, |
| + "maintain": TestBoundaryRecoversMaintain, |
| + "abort-queued": TestBoundaryRecoversAbortQueued, |
| + "abort": TestBoundaryRecoversAbort, |
| + "finish": TestBoundaryRecoversFinish, |
| + "cut-restage": TestBoundaryRecoversCutRestage, |
| + "start": TestBoundaryRecoversStart, |
| + "seal": TestBoundaryRecoversSeal, |
| + "flip": TestBoundaryRecoversFlip, |
| + "open": TestBoundaryRecoversOpen, |
| + "finish-gate1": TestBoundaryRecoversFinishGate1, |
| + "finish-done": TestBoundaryRecoversFinishDone, |
| + "abort-after-start": TestBoundaryRecoversAbortAfterStart, |
| +} |
| + |
| +func TestEveryBoundaryRecovers(t *testing.T) { |
| + for _, b := range jobBoundaries { |
| + if boundaryTests[b.name] == nil { |
| + t.Errorf("boundary %q has no top-level TestBoundaryRecovers test", b.name) |
| + } |
| + } |
| + for name := range boundaryTests { |
| + found := false |
| + for _, b := range jobBoundaries { |
| + found = found || b.name == name |
| + } |
| + if !found { |
| + t.Errorf("boundaryTests names %q, which is not in jobBoundaries", name) |
| + } |
| } |
| t.Run("ordinary progression bumps once per landed write", func(t *testing.T) { |
| w := newJobWorld(t) |