carlos / platform Public

ops tests: the under-the-lock count test fails if the count moves before the lock (#284)

Summary

Astra's fifth pass: MERGE-READY with one P3, that the regression test
would still pass with the count taken before the lock. The second run
now records when it counts and the config changes while the first run
holds the lock; checked against a mutant that counts first (fails on
both assertions).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 1 week ago fa63c1eadca20dc9cf15b098bf4d68349792438e parent 2ec2e90

1 file changed, +23 −14

internal/ops/ops_test.go +23 −14

CI passed — run details

CI log

=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	348.489s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.313s
ok  	github.com/carlosframework/platform/cmd/moveprobe	93.222s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.386s
ok  	github.com/carlosframework/platform/internal/activator	226.678s
ok  	github.com/carlosframework/platform/internal/agentsock	0.015s
ok  	github.com/carlosframework/platform/internal/alerting	0.003s
ok  	github.com/carlosframework/platform/internal/analytics	0.022s
ok  	github.com/carlosframework/platform/internal/appconfig	2.356s
ok  	github.com/carlosframework/platform/internal/apperr	0.071s
ok  	github.com/carlosframework/platform/internal/applog	0.123s
ok  	github.com/carlosframework/platform/internal/awsinv	0.003s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.761s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.008s
ok  	github.com/carlosframework/platform/internal/caps	0.011s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.697s
ok  	github.com/carlosframework/platform/internal/changelog	0.295s
ok  	github.com/carlosframework/platform/internal/clamd	0.054s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	365.386s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	5.743s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.010s
ok  	github.com/carlosframework/platform/internal/dbverify	0.004s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.003s
ok  	github.com/carlosframework/platform/internal/docsite	0.004s
ok  	github.com/carlosframework/platform/internal/domains	2.123s
ok  	github.com/carlosframework/platform/internal/doorkey	0.006s
ok  	github.com/carlosframework/platform/internal/economics	0.007s
ok  	github.com/carlosframework/platform/internal/edge	26.130s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.124s
ok  	github.com/carlosframework/platform/internal/gate	1.655s
ok  	github.com/carlosframework/platform/internal/genselect	0.015s
ok  	github.com/carlosframework/platform/internal/geo	0.019s
ok  	github.com/carlosframework/platform/internal/geoip	0.031s
ok  	github.com/carlosframework/platform/internal/host	128.427s
ok  	github.com/carlosframework/platform/internal/hostactors	0.649s
ok  	github.com/carlosframework/platform/internal/hostbox	2.228s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.511s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.015s
ok  	github.com/carlosframework/platform/internal/hostexec	5.073s
ok  	github.com/carlosframework/platform/internal/hostfence	0.033s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.598s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.015s
ok  	github.com/carlosframework/platform/internal/hostguard	0.970s
ok  	github.com/carlosframework/platform/internal/hostops	1.482s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.015s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.016s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.460s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.148s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.015s
ok  	github.com/carlosframework/platform/internal/infraview	0.015s
ok  	github.com/carlosframework/platform/internal/instancectl	1.814s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.717s
ok  	github.com/carlosframework/platform/internal/instancefs	0.020s
ok  	github.com/carlosframework/platform/internal/instancekey	0.018s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.022s
ok  	github.com/carlosframework/platform/internal/layout	1.469s
ok  	github.com/carlosframework/platform/internal/lease	9.468s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.022s
ok  	github.com/carlosframework/platform/internal/ledger	0.011s
ok  	github.com/carlosframework/platform/internal/mail	0.166s
ok  	github.com/carlosframework/platform/internal/maintenance	0.082s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.537s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.066s
ok  	github.com/carlosframework/platform/internal/metrics	0.006s
ok  	github.com/carlosframework/platform/internal/moves	21.086s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.780s
ok  	github.com/carlosframework/platform/internal/objstore	0.171s
ok  	github.com/carlosframework/platform/internal/ops	0.774s
ok  	github.com/carlosframework/platform/internal/park	0.200s
ok  	github.com/carlosframework/platform/internal/privatefile	0.003s
ok  	github.com/carlosframework/platform/internal/provision	11.721s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.010s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	4.137s
ok  	github.com/carlosframework/platform/internal/relay	7.223s
ok  	github.com/carlosframework/platform/internal/release	19.604s
ok  	github.com/carlosframework/platform/internal/rollout	3.669s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.003s
ok  	github.com/carlosframework/platform/internal/rootaws	0.006s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.008s
ok  	github.com/carlosframework/platform/internal/rootstate	0.025s
ok  	github.com/carlosframework/platform/internal/s3compat	0.007s
ok  	github.com/carlosframework/platform/internal/safefile	0.004s
ok  	github.com/carlosframework/platform/internal/schedule	2.304s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	9.671s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.287s
ok  	github.com/carlosframework/platform/internal/status	0.039s
ok  	github.com/carlosframework/platform/internal/steering	0.007s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.247s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.004s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.005s
ok  	github.com/carlosframework/platform/internal/updatewords	0.004s
ok  	github.com/carlosframework/platform/internal/usage	0.004s
ok  	github.com/carlosframework/platform/internal/version	0.002s
ok  	github.com/carlosframework/platform/internal/vet	1.540s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.022s
=== RUN   TestT4StorageProbeMinio
2026/10/14 00:33:42 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DE3E234767350D, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.020s
minio contract: PASS

diff --git a/internal/ops/ops_test.go b/internal/ops/ops_test.go

index 4353dbd12..4f9b05d01 100644

--- a/internal/ops/ops_test.go

+++ b/internal/ops/ops_test.go

@@ -8,6 +8,7 @@ import (

"slices"

"strconv"

"strings"

+ "sync/atomic"

"testing"

"time"

@@ -1043,9 +1044,9 @@ func TestWriteMemoryDropInRunsOneAtATime(t *testing.T) {

func dbs(n int) func() (int, error) { return func() (int, error) { return n, nil } }

// The cap is sized from the config that is on disk when the run holds the

-// lock, not from what the run counted before it queued. Run A counted 32

-// databases, then waited behind run B, which wrote a config of 200 and its

-// cap. A must leave the cap for 200, not put back the one for 32.

+// lock, not from what the run counted before it queued. Run A arrives while

+// run B holds the lock; the config changes before B lets go. A must not

+// count until then, and must size the cap for what the config says then.

func TestWriteMemoryDropInSizesFromTheConfigUnderTheLock(t *testing.T) {

dir := t.TempDir()

path := filepath.Join(dir, "d", "50-memory.conf")

@@ -1059,37 +1060,45 @@ func TestWriteMemoryDropInSizesFromTheConfigUnderTheLock(t *testing.T) {

t.Fatal(err)

}

- count := func() (int, error) { return configDBCount(cfg) }

- write(32) // A's config, written before it queues for the lock

+ var aCounted atomic.Bool

+ aCount := func() (int, error) {

+ aCounted.Store(true)

+ return configDBCount(cfg)

+ }

aDone := make(chan int, 1)

bReload := func() error {

- // B holds the lock. A arrives, having counted 32, and waits.

+ // B holds the lock. A arrives and has to wait.

go func() {

- n, _, err := writeMemoryDropIn(path, count, func() error { return nil })

+ n, _, err := writeMemoryDropIn(path, aCount, func() error { return nil })

if err != nil {

t.Error(err)

}

aDone <- n

}()

- time.Sleep(200 * time.Millisecond)

+ time.Sleep(300 * time.Millisecond)

+ if aCounted.Load() {

+ t.Error("A counted the config while B still held the lock")

+ }

+ // The config moves on while A is still queued.

+ write(300)

return nil

}

- write(200) // B's config lands, then B takes the lock and caps it

- if n, changed, err := writeMemoryDropIn(path, count, bReload); n != 200 || !changed || err != nil {

+ write(200)

+ if n, changed, err := writeMemoryDropIn(path, func() (int, error) { return configDBCount(cfg) }, bReload); n != 200 || !changed || err != nil {

t.Fatalf("B: n=%d changed=%v err=%v, want 200, true, nil", n, changed, err)

}

select {

case n := <-aDone:

- if n != 200 {

- t.Fatalf("A sized the cap for %d databases, want the 200 the config names", n)

+ if n != 300 {

+ t.Fatalf("A sized the cap for %d databases, want the 300 the config named once A held the lock", n)

}

case <-time.After(10 * time.Second):

t.Fatal("A never ran")

}

body, _ := os.ReadFile(path)

- if want := "MemoryMax=" + strconv.FormatInt(LitestreamMemoryMax(200)>>20, 10) + "M\n"; !strings.Contains(string(body), want) {

- t.Fatalf("the cap left on disk is not the one for 200 databases:\n%s", body)

+ if want := "MemoryMax=" + strconv.FormatInt(LitestreamMemoryMax(300)>>20, 10) + "M\n"; !strings.Contains(string(body), want) {

+ t.Fatalf("the cap left on disk is not the one for 300 databases:\n%s", body)

}

if _, _, err := writeMemoryDropIn(path, func() (int, error) { return 0, errors.New("config unreadable") }, func() error { return nil }); err == nil {