amadan

carlos / platform Public

console-invariant-parallel

Squashed from console-invariant-parallel into main.
claude 521c6ee912f045967d913694cfe95eb5bf76affb parent 4a1e4b2
10 files changed, +186 −36
  • .amadan/test-weights.tsv +2 −2
  • internal/console/claim_invariant_fixtures_test.go +1 −0
  • internal/console/claim_invariant_probes_nonapp_test.go +4 −1
  • internal/console/claim_invariant_probes_read_test.go +9 −3
  • internal/console/claim_invariant_probes_test.go +33 −5
  • internal/console/claim_invariant_probes_worker_test.go +5 −1
  • internal/console/claim_invariant_regionha_test.go +1 −1
  • internal/console/claim_invariant_test.go +113 −16
  • internal/console/claim_invariant_world_test.go +13 −6
  • internal/console/regioncandidates_test.go +5 −1

CI passed — run details

CI log 
=== gate ===
.amadan/test-gate.sh
=== launchctx (0 s)
ok  	amadan.net/carlos/platform.git/launchctx	0.013s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
=== vet (3 s)
=== gofmt (0 s)
gofmt -l .: clean
=== rest (127 s)
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	4.395s
ok  	github.com/carlosframework/platform/cmd/moveprobe	11.654s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.711s
ok  	github.com/carlosframework/platform/internal/activator	61.127s
ok  	github.com/carlosframework/platform/internal/agentsock	0.017s
ok  	github.com/carlosframework/platform/internal/alerting	0.005s
ok  	github.com/carlosframework/platform/internal/analytics	0.021s
ok  	github.com/carlosframework/platform/internal/appconfig	2.540s
ok  	github.com/carlosframework/platform/internal/apperr	0.075s
ok  	github.com/carlosframework/platform/internal/applog	0.141s
ok  	github.com/carlosframework/platform/internal/awsinv	0.003s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.537s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.011s
ok  	github.com/carlosframework/platform/internal/caps	0.013s
ok  	github.com/carlosframework/platform/internal/certwarm	0.006s
ok  	github.com/carlosframework/platform/internal/cgroup	0.702s
ok  	github.com/carlosframework/platform/internal/changelog	0.351s
ok  	github.com/carlosframework/platform/internal/claimactivation	0.791s
ok  	github.com/carlosframework/platform/internal/clamd	0.054s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.443s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.010s
ok  	github.com/carlosframework/platform/internal/dbverify	0.004s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.003s
ok  	github.com/carlosframework/platform/internal/docsite	0.009s
ok  	github.com/carlosframework/platform/internal/domains	2.523s
ok  	github.com/carlosframework/platform/internal/doorkey	0.008s
ok  	github.com/carlosframework/platform/internal/economics	0.004s
ok  	github.com/carlosframework/platform/internal/edge	45.193s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.136s
ok  	github.com/carlosframework/platform/internal/gate	1.656s
ok  	github.com/carlosframework/platform/internal/genselect	0.015s
ok  	github.com/carlosframework/platform/internal/geo	0.016s
ok  	github.com/carlosframework/platform/internal/geoip	0.029s
ok  	github.com/carlosframework/platform/internal/historyreadback	0.153s
ok  	github.com/carlosframework/platform/internal/host	121.739s
ok  	github.com/carlosframework/platform/internal/hostactors	5.626s
ok  	github.com/carlosframework/platform/internal/hostactors/awsmembership	0.008s
ok  	github.com/carlosframework/platform/internal/hostbox	2.859s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.599s
ok  	github.com/carlosframework/platform/internal/hostenv	0.007s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.006s
ok  	github.com/carlosframework/platform/internal/hostexec	5.300s
ok  	github.com/carlosframework/platform/internal/hostfence	0.036s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.527s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.021s
ok  	github.com/carlosframework/platform/internal/hostguard	1.009s
ok  	github.com/carlosframework/platform/internal/hostops	1.757s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.003s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.003s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.475s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.144s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.002s
ok  	github.com/carlosframework/platform/internal/infraview	0.002s
ok  	github.com/carlosframework/platform/internal/instancectl	5.263s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.810s
ok  	github.com/carlosframework/platform/internal/instancefs	0.020s
ok  	github.com/carlosframework/platform/internal/instancekey	0.018s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.011s
ok  	github.com/carlosframework/platform/internal/layout	1.614s
ok  	github.com/carlosframework/platform/internal/lbhealth	0.002s
ok  	github.com/carlosframework/platform/internal/lease	10.132s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.400s
ok  	github.com/carlosframework/platform/internal/ledger	0.029s
ok  	github.com/carlosframework/platform/internal/mail	0.197s
ok  	github.com/carlosframework/platform/internal/maintenance	0.083s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.545s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.082s
ok  	github.com/carlosframework/platform/internal/metrics	0.005s
ok  	github.com/carlosframework/platform/internal/moves	21.234s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.827s
ok  	github.com/carlosframework/platform/internal/objstore	0.224s
ok  	github.com/carlosframework/platform/internal/ops	0.180s
ok  	github.com/carlosframework/platform/internal/park	0.231s
ok  	github.com/carlosframework/platform/internal/plans	0.006s
ok  	github.com/carlosframework/platform/internal/privatefile	0.003s
ok  	github.com/carlosframework/platform/internal/provision	13.364s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.010s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	5.566s
ok  	github.com/carlosframework/platform/internal/relay	7.247s
ok  	github.com/carlosframework/platform/internal/release	24.454s
ok  	github.com/carlosframework/platform/internal/rollout	3.819s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.004s
ok  	github.com/carlosframework/platform/internal/rootaws	0.007s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.058s
ok  	github.com/carlosframework/platform/internal/rootstate	0.026s
ok  	github.com/carlosframework/platform/internal/runs	0.604s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.004s
ok  	github.com/carlosframework/platform/internal/schedule	2.676s
ok  	github.com/carlosframework/platform/internal/seeds	0.011s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	10.861s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.537s
ok  	github.com/carlosframework/platform/internal/status	0.059s
ok  	github.com/carlosframework/platform/internal/steering	0.006s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.218s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.004s
ok  	github.com/carlosframework/platform/internal/termfence	0.006s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.007s
ok  	github.com/carlosframework/platform/internal/unixdial	0.002s
ok  	github.com/carlosframework/platform/internal/updatewords	0.006s
ok  	github.com/carlosframework/platform/internal/usage	0.005s
ok  	github.com/carlosframework/platform/internal/version	0.004s
ok  	github.com/carlosframework/platform/internal/vet	1.553s
=== internal_console@0 (79 s)
ok  	github.com/carlosframework/platform/internal/console	77.559s
=== internal_console@1 (36 s)
ok  	github.com/carlosframework/platform/internal/console	34.808s
=== internal_console@2 (41 s)
ok  	github.com/carlosframework/platform/internal/console	39.553s
=== internal_console@3 (36 s)
ok  	github.com/carlosframework/platform/internal/console	35.206s
=== internal_console@4 (36 s)
ok  	github.com/carlosframework/platform/internal/console	35.174s
=== internal_console@5 (34 s)
ok  	github.com/carlosframework/platform/internal/console	32.904s
=== internal_console@6 (35 s)
ok  	github.com/carlosframework/platform/internal/console	33.735s
=== internal_console@7 (42 s)
ok  	github.com/carlosframework/platform/internal/console	41.219s
=== root@0 (31 s)
ok  	github.com/carlosframework/platform	30.403s
=== root@1 (43 s)
ok  	github.com/carlosframework/platform	42.108s
=== root@2 (43 s)
ok  	github.com/carlosframework/platform	42.213s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
linux/arm64 carlos binary: 49873058 bytes (ceiling 52000000)
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.024s
=== RUN   TestT4StorageProbeMinio
2026/10/21 18:17:25 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18E09E3F1B2F16FE, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.023s
minio contract: PASS
diff --git a/.amadan/test-weights.tsv b/.amadan/test-weights.tsv
index d994e6ff6..eb56aab72 100644
--- a/.amadan/test-weights.tsv
+++ b/.amadan/test-weights.tsv
@@ -1,8 +1,8 @@
# package test seconds (.amadan/test-weights.sh, 2026-10-21)
-github.com/carlosframework/platform/internal/console TestClaimConsoleMutationInvariant 199.7
+github.com/carlosframework/platform/internal/console TestClaimConsoleMutationInvariant 60.1
+github.com/carlosframework/platform/internal/console TestClaimConsoleMutationInvariantWitnesses 12.3
github.com/carlosframework/platform/internal/console TestRetiringAnInstancePurgesItsMoveLogRows 9.3
github.com/carlosframework/platform/internal/console TestDeclaredLostIsLoggedByTheConsole 7.2
-github.com/carlosframework/platform/internal/console TestClaimConsoleMutationInvariantWitnesses 6.6
github.com/carlosframework/platform/internal/console TestClaimMailPushCountsOutboundEffects 6.4
github.com/carlosframework/platform/internal/console TestEconomicsPassWritesCensusOnce 5.0
github.com/carlosframework/platform/internal/console TestCallbackPostsToSelectedHost 3.8
diff --git a/internal/console/claim_invariant_fixtures_test.go b/internal/console/claim_invariant_fixtures_test.go
index d09d1d291..8919640e9 100644
--- a/internal/console/claim_invariant_fixtures_test.go
+++ b/internal/console/claim_invariant_fixtures_test.go
@@ -1876,6 +1876,7 @@ var claimFixtures = map[string]claimFixture{
"worker StartMailEventPush": {door: &claimDoor{
apps: claimRefs("bab/titogo"),
setup: func(t *testing.T, cx *claimCase) {
+ claimMustRunAlone(t, "worker StartMailEventPush")
old := mailPushClient.Transport
mailPushClient.Transport = claimOKTransport{rec: cx.rec}
t.Cleanup(func() { mailPushClient.Transport = old })
diff --git a/internal/console/claim_invariant_probes_nonapp_test.go b/internal/console/claim_invariant_probes_nonapp_test.go
index 9e7c1697e..8e16fe554 100644
--- a/internal/console/claim_invariant_probes_nonapp_test.go
+++ b/internal/console/claim_invariant_probes_nonapp_test.go
@@ -814,7 +814,8 @@ var claimNonAppProbes = map[string]claimProbe{
},
},
"route PUT /api/cli/system/geoip": {
- setup: func(t *testing.T, cx *claimCase) { withFixtureProbes(t) },
+ serial: "withFixtureProbes swaps the package variable geoipProbeAddrs",
+ setup: func(t *testing.T, cx *claimCase) { claimMustRunAlone(t, "geoip probe"); withFixtureProbes(t) },
do: func(t *testing.T, cx *claimCase) claimReply {
return cx.apiReply(t, http.MethodPut, "/api/cli/system/geoip", string(fixture(t, "GeoIP2-City-Test.mmdb")))
},
@@ -834,7 +835,9 @@ var claimNonAppProbes = map[string]claimProbe{
},
},
"route DELETE /api/cli/system/geoip": {
+ serial: "withFixtureProbes swaps the package variable geoipProbeAddrs",
setup: func(t *testing.T, cx *claimCase) {
+ claimMustRunAlone(t, "geoip probe")
withFixtureProbes(t)
if _, err := cx.srv.geoip.Publish(context.Background(), strings.NewReader(string(fixture(t, "GeoIP2-City-Test.mmdb"))), geoipPublishOpts{Origin: "manual"}); err != nil {
t.Fatalf("stage a published geoip database: %v", err)
diff --git a/internal/console/claim_invariant_probes_read_test.go b/internal/console/claim_invariant_probes_read_test.go
index 307543163..5c814388d 100644
--- a/internal/console/claim_invariant_probes_read_test.go
+++ b/internal/console/claim_invariant_probes_read_test.go
@@ -84,6 +84,7 @@ func claimWorldApply(t *testing.T, cx *claimCase) {
func claimWorldGeoip(t *testing.T, cx *claimCase) {
t.Helper()
+ claimMustRunAlone(t, "claimWorldGeoip")
withFixtureProbes(t)
if _, err := cx.srv.geoip.Publish(context.Background(), bytes.NewReader(fixture(t, "GeoIP2-City-Test.mmdb")), geoipPublishOpts{Origin: "manual"}); err != nil {
t.Fatalf("stage a published geoip database: %v", err)
@@ -200,8 +201,9 @@ var claimReadProbes = map[string]claimProbe{
"/api/cli/apps/bab/titogo/pending", "<pending>"),
"route GET /api/cli/apps/{account}/{app}/pipeline": claimAPIRead(nil, "/api/cli/apps/bab/titogo/pipeline", `"pipeline"`),
"route GET /api/cli/apps/{account}/{app}/regions": claimAPIRead(nil, "/api/cli/apps/bab/titogo/regions", `"regions"`),
- "route GET /api/cli/apps/{account}/{app}/relay": claimAPIRead(func(t *testing.T, cx *claimCase) { cx.relayEnabled(t) },
- "/api/cli/apps/bab/titogo/relay", `"app":"titogo"`, `"enabled":true`),
+ "route GET /api/cli/apps/{account}/{app}/relay": claimSerialProbe("relayEnabled stages the region's relay master through t.Setenv: process environment",
+ claimAPIRead(func(t *testing.T, cx *claimCase) { cx.relayEnabled(t) },
+ "/api/cli/apps/bab/titogo/relay", `"app":"titogo"`, `"enabled":true`)),
"route GET /api/cli/apps/{account}/{app}/releases": claimAPIRead(nil, "/api/cli/apps/bab/kass/releases", `"version":"v1"`),
"route GET /api/cli/apps/{account}/{app}/releases/{version}/authorize": claimAPIRead(nil,
"/api/cli/apps/bab/kass/releases/v1/authorize", `"version":"v1"`, `"manifest_digest":"`),
@@ -232,7 +234,11 @@ var claimReadProbes = map[string]claimProbe{
"route GET /api/cli/accounts/{account}/plan": claimAPIRead(func(t *testing.T, cx *claimCase) { cx.srv.cfg.PlanCatalog = examplePlanCatalog(t) },
"/api/cli/accounts/bab/plan", `"plan":"free"`),
"route GET /api/cli/system/antivirus": claimAPIRead(nil, "/api/cli/system/antivirus", `"wired":true`),
- "route GET /api/cli/system/geoip": claimAPIRead(claimWorldGeoip, "/api/cli/system/geoip", `"pointer"`, "GeoIP2-City"),
+ "route GET /api/cli/system/geoip": func() claimProbe {
+ p := claimAPIRead(claimWorldGeoip, "/api/cli/system/geoip", `"pointer"`, "GeoIP2-City")
+ p.serial = "claimWorldGeoip swaps the package variable geoipProbeAddrs (withFixtureProbes)"
+ return p
+ }(),
"route GET /api/cli/system/jobs/{id}": {
setup: func(t *testing.T, cx *claimCase) { cx.vars["job"] = cx.systemRelease(t, "canary", "66587fa") },
do: claimGetAPI("/api/cli/system/jobs/<job>"),
diff --git a/internal/console/claim_invariant_probes_test.go b/internal/console/claim_invariant_probes_test.go
index 3e1ed8121..77914218c 100644
--- a/internal/console/claim_invariant_probes_test.go
+++ b/internal/console/claim_invariant_probes_test.go
@@ -80,6 +80,16 @@ type claimProbe struct {
// effect proves from the world that the handler did its work, and returns
// "" when it did. It is required for every nonApp entry.
effect func(t *testing.T, cx *claimCase, r claimReply) string
+ // serial, when set, says why this probe touches process-wide state (the
+ // environment, the log output, a package variable). Such a probe runs
+ // alone, before any parallel probe starts.
+ serial string
+}
+
+// claimSerialProbe marks p serial, with the reason.
+func claimSerialProbe(reason string, p claimProbe) claimProbe {
+ p.serial = reason
+ return p
}
// claimWatched are the apps whose SQL and bucket state every probe must leave
@@ -417,18 +427,36 @@ func claimProbeKeys(fixtures map[string]claimFixture) []string {
// runClaimProbes runs every read and nonApp probe on its own console, so no
// probe's world (a closed session, a deleted account) disturbs another's.
-func runClaimProbes(t *testing.T, fixtures map[string]claimFixture, probes map[string]claimProbe, opts ...func(*Config)) {
+// With parallel above zero, at most that many run at once; a serial probe
+// still runs alone, before the parallel ones start (they wait until this
+// function returns). The caller's t.Run returns when all of them are done.
+func runClaimProbes(t *testing.T, fixtures map[string]claimFixture, probes map[string]claimProbe, parallel int, opts ...func(*Config)) {
t.Helper()
+ run := func(t *testing.T, key string, p claimProbe) {
+ cx := newClaimCaseWith(t, opts...)
+ for _, problem := range runClaimProbe(t, cx, key, p) {
+ t.Error(problem)
+ }
+ }
+ var deferred []string
for _, key := range claimProbeKeys(fixtures) {
p, ok := probes[key]
if !ok {
continue // the census names it
}
+ if parallel > 0 && strings.TrimSpace(p.serial) == "" {
+ deferred = append(deferred, key)
+ continue
+ }
+ t.Run(key, func(t *testing.T) { run(t, key, p) })
+ }
+ slots := make(chan struct{}, max(parallel, 1))
+ for _, key := range deferred {
+ p := probes[key]
t.Run(key, func(t *testing.T) {
- cx := newClaimCaseWith(t, opts...)
- for _, problem := range runClaimProbe(t, cx, key, p) {
- t.Error(problem)
- }
+ t.Parallel()
+ claimParallelSlot(t, slots)
+ run(t, key, p)
})
}
}
diff --git a/internal/console/claim_invariant_probes_worker_test.go b/internal/console/claim_invariant_probes_worker_test.go
index c75eb79e2..765f8db9a 100644
--- a/internal/console/claim_invariant_probes_worker_test.go
+++ b/internal/console/claim_invariant_probes_worker_test.go
@@ -32,8 +32,10 @@ import (
// it must have said about a real finding.
func claimLogWorkerProbe(name string, setup func(*testing.T, *claimCase), want ...string) claimProbe {
return claimProbe{
- setup: setup,
+ serial: "captures the process-wide log output (captureLogTo): another case's log lines would land in its answer",
+ setup: setup,
do: func(t *testing.T, cx *claimCase) claimReply {
+ claimMustRunAlone(t, "log worker probe "+name)
var logs strings.Builder
restore := captureLogTo(&logs)
reply := claimWorkerReply(name)(t, cx)
@@ -155,7 +157,9 @@ var claimWorkerProbes = map[string]claimProbe{
},
},
"worker StartHostReports": {
+ serial: "swaps the package variable hostRunningImage",
setup: func(t *testing.T, cx *claimCase) {
+ claimMustRunAlone(t, "worker StartHostReports")
// The box's guard files are not here; the seam says what this
// process is running, as the guard would.
prev := hostRunningImage
diff --git a/internal/console/claim_invariant_regionha_test.go b/internal/console/claim_invariant_regionha_test.go
index a8e5dcfc9..f1c797977 100644
--- a/internal/console/claim_invariant_regionha_test.go
+++ b/internal/console/claim_invariant_regionha_test.go
@@ -158,5 +158,5 @@ func TestClaimRegionHAConsoleRouteCensus(t *testing.T) {
// TestClaimRegionHAEntriesAreProbed drives every region-HA-only entry's
// probe on a console built with region HA on.
func TestClaimRegionHAEntriesAreProbed(t *testing.T) {
- runClaimProbes(t, claimRegionHAFixtures, claimRegionHAProbes, claimRegionHAOn)
+ runClaimProbes(t, claimRegionHAFixtures, claimRegionHAProbes, 0, claimRegionHAOn)
}
diff --git a/internal/console/claim_invariant_test.go b/internal/console/claim_invariant_test.go
index 6f674b7eb..5e438dd64 100644
--- a/internal/console/claim_invariant_test.go
+++ b/internal/console/claim_invariant_test.go
@@ -53,6 +53,8 @@ import (
"slices"
"sort"
"strings"
+ "sync"
+ "sync/atomic"
"testing"
"time"
@@ -178,6 +180,74 @@ var claimCoordinators = map[string]string{
"and the function's own writes reach Put, Delete or DeleteIfMatch)",
}
+// claimCaseParallelism bounds how many door, refusal and probe subtests run
+// at once. Each one builds its own console and measures what a request left
+// behind by waiting for quiet windows (quiesce). Too many at once starve the
+// CPU, and a handler goroutine that misses its window is measured with the
+// wrong request; the bound keeps that contention down.
+const claimCaseParallelism = 8
+
+// claimParallelCases counts the claim cases running under t.Parallel. A
+// fixture that swaps process-wide state (a package variable, the log
+// output, a shared client's transport) calls claimMustRunAlone, which
+// fails the case if any parallel case is running, itself included. A swap
+// added to a fixture that is not listed serial therefore fails loudly
+// instead of leaking into a neighbour's measurement.
+var claimParallelCases atomic.Int32
+
+func claimMustRunAlone(t *testing.T, what string) {
+ t.Helper()
+ if n := claimParallelCases.Load(); n > 0 {
+ t.Fatalf("%s swaps process-wide state but runs beside %d parallel claim case(s): list it serial (claimSerialDoors, or the probe's serial field)", what, n)
+ }
+}
+
+// claimParallelSlot takes a slot for a parallel claim case and counts it
+// as running until its own cleanup is done.
+func claimParallelSlot(t *testing.T, slots chan struct{}) {
+ slots <- struct{}{}
+ // Registered first, so it runs last: the slot is held until the case's
+ // own cleanup (servers, store) is done.
+ t.Cleanup(func() { <-slots })
+ claimParallelCases.Add(1)
+ t.Cleanup(func() { claimParallelCases.Add(-1) })
+}
+
+// claimSerialDoors are the door and refusal fixtures that must not run in
+// parallel, each with the reason. They run first, one at a time, before any
+// parallel door starts. A fixture whose staging touches process-wide state
+// (the environment, the log output, a package variable) belongs here.
+var claimSerialDoors = map[string]string{
+ "route POST /api/cli/relay/rotate": "relayEnabled and relayStageMaster stage the region's relay masters " +
+ "through t.Setenv: process environment, which t.Parallel forbids",
+ "route POST /api/cli/apps/{account}/{app}/relay": "relayArmed stages the region's relay master through t.Setenv: " +
+ "process environment, which t.Parallel forbids",
+ "route POST /api/cli/apps/{account}/{app}/relay/adopted": "relayEnabled stages the region's relay master through t.Setenv: " +
+ "process environment, which t.Parallel forbids",
+ "route POST /api/cli/apps/{account}/{app}/relay/disable": "relayEnabled stages the region's relay master through t.Setenv: " +
+ "process environment, which t.Parallel forbids",
+ "worker StartRelay": "relayEnabled stages the region's relay master through t.Setenv: " +
+ "process environment, which t.Parallel forbids",
+ "worker StartMailEventPush": "swaps mailPushClient.Transport, a package variable, for its recorder",
+}
+
+// claimSerialDoorCensus: every serial entry names a door or refusal fixture
+// and carries its reason.
+func claimSerialDoorCensus(fixtures map[string]claimFixture, serial map[string]string) []string {
+ var problems []string
+ for key, reason := range serial {
+ f, ok := fixtures[key]
+ switch {
+ case !ok || (f.door == nil && f.refused == nil):
+ problems = append(problems, fmt.Sprintf("claimSerialDoors names %s, which is not a door or refusal fixture", key))
+ case strings.TrimSpace(reason) == "":
+ problems = append(problems, fmt.Sprintf("claimSerialDoors names %s with no reason", key))
+ }
+ }
+ sort.Strings(problems)
+ return problems
+}
+
func TestClaimConsoleMutationInvariant(t *testing.T) {
census := newClaimCase(t)
if !census.srv.cfg.HostClaim {
@@ -209,30 +279,56 @@ func TestClaimConsoleMutationInvariant(t *testing.T) {
}
sort.Strings(keys)
- reached := map[string]bool{}
- for _, key := range keys {
- f := claimFixtures[key]
- if f.refused != nil {
- t.Run(key, func(t *testing.T) {
+ if problems := claimSerialDoorCensus(claimFixtures, claimSerialDoors); len(problems) != 0 {
+ for _, p := range problems {
+ t.Error(p)
+ }
+ t.FailNow()
+ }
+
+ // Doors and refusals, each on its own console. The group returns only
+ // when every one of them has finished, parallel ones included, so the
+ // writer check below sees every door's open run.
+ var (
+ reachedMu sync.Mutex
+ reached = map[string]bool{}
+ )
+ t.Run("doors", func(t *testing.T) {
+ slots := make(chan struct{}, claimCaseParallelism)
+ run := func(t *testing.T, f claimFixture) {
+ if f.refused != nil {
for _, p := range runClaimRefusal(t, f.refused) {
t.Error(p)
}
- })
- continue
- }
- if f.door == nil {
- continue
- }
- t.Run(key, func(t *testing.T) {
+ return
+ }
problems, writers := runClaimDoor(t, f.door)
for _, p := range problems {
t.Error(p)
}
+ reachedMu.Lock()
for _, w := range writers {
reached[w] = true
}
- })
- }
+ reachedMu.Unlock()
+ }
+ // The serial ones first, one at a time, while nothing else runs:
+ // a parallel subtest only starts once this function has returned.
+ for _, key := range keys {
+ if f := claimFixtures[key]; (f.door != nil || f.refused != nil) && claimSerialDoors[key] != "" {
+ t.Run(key, func(t *testing.T) { run(t, f) })
+ }
+ }
+ for _, key := range keys {
+ if f := claimFixtures[key]; (f.door != nil || f.refused != nil) && claimSerialDoors[key] == "" {
+ t.Run(key, func(t *testing.T) {
+ t.Parallel()
+ claimParallelSlot(t, slots)
+ run(t, f)
+ })
+ }
+ }
+ })
// Lifecycle variants: further actions on an entry above whose only
// effect is a host claim written under their own registration grant
@@ -242,9 +338,10 @@ func TestClaimConsoleMutationInvariant(t *testing.T) {
// Reads and nonApp entries are probed with their own valid request on
// their own console (claim_invariant_probes_test.go): a request the
// handler turned away is not coverage, and a mutator proves its effect
- // from the world.
+ // from the world. They run in parallel too, after every door, except the
+ // probes marked serial (claimProbe.serial), which run alone first.
t.Run("probes", func(t *testing.T) {
- runClaimProbes(t, claimFixtures, claimProbes)
+ runClaimProbes(t, claimFixtures, claimProbes, claimCaseParallelism)
})
var unreached []string
diff --git a/internal/console/claim_invariant_world_test.go b/internal/console/claim_invariant_world_test.go
index 980a96b04..6b56214f3 100644
--- a/internal/console/claim_invariant_world_test.go
+++ b/internal/console/claim_invariant_world_test.go
@@ -718,18 +718,25 @@ func (cx *claimCase) foreign(t *testing.T, ref claimAppRef) func() {
}
}
-// quiesce waits until no raw effect has arrived for one 100ms window (at
-// most 5s), so a goroutine a handler started is measured with its request.
+// quiesce waits until no raw effect has arrived for two consecutive 100ms
+// windows (at most 5s), so a goroutine a handler started is measured with its
+// request. Two windows, not one: the invariant runs its doors in parallel
+// (claimCaseParallelism), and a goroutine starved of CPU for one window must
+// still land in its own request's measurement, not the next one's.
func (cx *claimCase) quiesce() {
+ const window = 100 * time.Millisecond
deadline := time.Now().Add(5 * time.Second)
- n := cx.rec.mark()
+ n, quiet := cx.rec.mark(), 0
for time.Now().Before(deadline) {
- time.Sleep(100 * time.Millisecond)
+ time.Sleep(window)
m := cx.rec.mark()
- if m == n {
+ if m != n {
+ n, quiet = m, 0
+ continue
+ }
+ if quiet++; quiet == 2 {
return
}
- n = m
}
}
diff --git a/internal/console/regioncandidates_test.go b/internal/console/regioncandidates_test.go
index 6c598227a..c33f31773 100644
--- a/internal/console/regioncandidates_test.go
+++ b/internal/console/regioncandidates_test.go
@@ -433,7 +433,11 @@ func TestLiveEdgesSnapshotLoadFailureIsNotFreshAndLoggedOnce(t *testing.T) {
t.Fatalf("round %d: snapshots unloadable, uk = %+v, want ineligible", i, c)
}
}
- if n := strings.Count(buf.String(), "box snapshots"); n > 1 {
+ // Count the load-failure line itself. "box snapshots" alone also
+ // matches the request-level incomplete-read line, which has its own
+ // once-per-process latch: the test passed only when an earlier test in
+ // the same process had already tripped it.
+ if n := strings.Count(buf.String(), "could not load box snapshots"); n > 1 {
t.Fatalf("load failure logged %d times, want at most once: %s", n, buf.String())
}
}