carlos / platform Public

Claim activation tools, run 8: invariant coverage complete (not for merge)

Absent-journal reboots and explicit active-timer completion controls; 105 reboots and 32 pending-update cases on production units.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Paul Campbell pushed by claude 1 week ago 918d0e1a53b717e109d5103d0a48bb1d7c26a80f parent d474c7c

1 file changed, +50 −9

internal/host/activation_matrix_rootrunner_linux_test.go +50 −9

CI passed — run details

CI log

=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	297.461s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.003s
ok  	github.com/carlosframework/platform/cmd/moveprobe	93.186s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.699s
ok  	github.com/carlosframework/platform/internal/activator	249.812s
ok  	github.com/carlosframework/platform/internal/agentsock	0.018s
ok  	github.com/carlosframework/platform/internal/alerting	0.004s
ok  	github.com/carlosframework/platform/internal/analytics	0.018s
ok  	github.com/carlosframework/platform/internal/appconfig	2.468s
ok  	github.com/carlosframework/platform/internal/apperr	0.071s
ok  	github.com/carlosframework/platform/internal/applog	0.125s
ok  	github.com/carlosframework/platform/internal/awsinv	0.003s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.519s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.009s
ok  	github.com/carlosframework/platform/internal/caps	0.012s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.696s
ok  	github.com/carlosframework/platform/internal/changelog	0.304s
ok  	github.com/carlosframework/platform/internal/claimactivation	0.024s
ok  	github.com/carlosframework/platform/internal/clamd	0.054s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	389.752s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	5.655s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.009s
ok  	github.com/carlosframework/platform/internal/dbverify	0.005s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.005s
ok  	github.com/carlosframework/platform/internal/docsite	0.004s
ok  	github.com/carlosframework/platform/internal/domains	2.394s
ok  	github.com/carlosframework/platform/internal/doorkey	0.007s
ok  	github.com/carlosframework/platform/internal/economics	0.004s
ok  	github.com/carlosframework/platform/internal/edge	26.615s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.132s
ok  	github.com/carlosframework/platform/internal/gate	1.656s
ok  	github.com/carlosframework/platform/internal/genselect	0.014s
ok  	github.com/carlosframework/platform/internal/geo	0.015s
ok  	github.com/carlosframework/platform/internal/geoip	0.034s
ok  	github.com/carlosframework/platform/internal/historyreadback	0.154s
ok  	github.com/carlosframework/platform/internal/host	176.061s
ok  	github.com/carlosframework/platform/internal/hostactors	0.328s
ok  	github.com/carlosframework/platform/internal/hostbox	2.681s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.535s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.006s
ok  	github.com/carlosframework/platform/internal/hostexec	4.744s
ok  	github.com/carlosframework/platform/internal/hostfence	0.042s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.399s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.014s
ok  	github.com/carlosframework/platform/internal/hostguard	1.034s
ok  	github.com/carlosframework/platform/internal/hostops	1.625s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.002s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.002s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.452s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.182s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.002s
ok  	github.com/carlosframework/platform/internal/infraview	0.002s
ok  	github.com/carlosframework/platform/internal/instancectl	5.214s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.722s
ok  	github.com/carlosframework/platform/internal/instancefs	0.025s
ok  	github.com/carlosframework/platform/internal/instancekey	0.018s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.008s
ok  	github.com/carlosframework/platform/internal/layout	1.536s
ok  	github.com/carlosframework/platform/internal/lease	9.794s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.010s
ok  	github.com/carlosframework/platform/internal/ledger	0.011s
ok  	github.com/carlosframework/platform/internal/mail	0.157s
ok  	github.com/carlosframework/platform/internal/maintenance	0.085s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.464s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.093s
ok  	github.com/carlosframework/platform/internal/metrics	0.008s
ok  	github.com/carlosframework/platform/internal/moves	20.960s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.816s
ok  	github.com/carlosframework/platform/internal/objstore	0.178s
ok  	github.com/carlosframework/platform/internal/ops	0.172s
ok  	github.com/carlosframework/platform/internal/park	0.210s
ok  	github.com/carlosframework/platform/internal/privatefile	0.003s
ok  	github.com/carlosframework/platform/internal/provision	17.498s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.007s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	5.380s
ok  	github.com/carlosframework/platform/internal/relay	7.227s
ok  	github.com/carlosframework/platform/internal/release	21.373s
ok  	github.com/carlosframework/platform/internal/rollout	3.644s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.003s
ok  	github.com/carlosframework/platform/internal/rootaws	0.006s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.057s
ok  	github.com/carlosframework/platform/internal/rootstate	0.026s
ok  	github.com/carlosframework/platform/internal/runs	0.585s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.004s
ok  	github.com/carlosframework/platform/internal/schedule	2.490s
ok  	github.com/carlosframework/platform/internal/seeds	0.010s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	8.576s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.270s
ok  	github.com/carlosframework/platform/internal/status	0.037s
ok  	github.com/carlosframework/platform/internal/steering	0.005s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.186s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.007s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.002s
ok  	github.com/carlosframework/platform/internal/updatewords	0.004s
ok  	github.com/carlosframework/platform/internal/usage	0.004s
ok  	github.com/carlosframework/platform/internal/version	0.001s
ok  	github.com/carlosframework/platform/internal/vet	1.564s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
linux/arm64 carlos binary: 48693410 bytes (ceiling 52000000)
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.022s
=== RUN   TestT4StorageProbeMinio
2026/10/16 23:24:54 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DF261FCFC636D3, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.021s
minio contract: PASS

diff --git a/internal/host/activation_matrix_rootrunner_linux_test.go b/internal/host/activation_matrix_rootrunner_linux_test.go

index 99b3ce629..4889241bb 100644

--- a/internal/host/activation_matrix_rootrunner_linux_test.go

+++ b/internal/host/activation_matrix_rootrunner_linux_test.go

@@ -186,7 +186,15 @@ func activationMatrixResume(t *testing.T, tx activationOffline, point string) {

if before.Phase != "done" {

activationMatrixPending(t, tx, tx.restore)

}

- tx.checkpoint = func(name string) error { activationMatrixCheck(t, tx, name); return nil }

+ tx.checkpoint = func(name string) error {

+ for unit := range foreign {

+ if name == "stage-"+unit || name == "replace-"+unit {

+ t.Fatal("closed continuation replaced released unit", unit, name)

+ }

+ activationMatrixCheck(t, tx, name)

+ return nil

+ }

deadline := time.Now().Add(20 * time.Second)

for {

saved, err := os.ReadFile(filepath.Join(tx.state, "transaction.json"))

@@ -212,9 +220,13 @@ func activationMatrixResume(t *testing.T, tx activationOffline, point string) {

t.Fatal("unfinished continuation", r.Phase, r.EdgePending, r.TriggersPending)

}

for name, snapshot := range foreign {

- got, err := activationUnitSnapshot(filepath.Join(tx.units, name), name)

- if err != nil || !reflect.DeepEqual(got, snapshot) {

- t.Fatal("closed continuation overwrote recovered unit", name, err)

+ path := filepath.Join(tx.units, name)

+ got, err := activationUnitSnapshot(path, name)

+ if err != nil || sameActivationFile(path, filepath.Join(tx.state, name+".original")) || sameActivationFile(path, filepath.Join(tx.state, name+".mask")) {

+ t.Fatal("closed continuation reclaimed released unit", name, err)

+ }

+ if !reflect.DeepEqual(got, snapshot) {

+ t.Log("ordinary writer changed released unit; continuation made no replacement", name)

}

// Once done is durable, timer-driven adoption is ordinary operation. The

@@ -275,6 +287,11 @@ func TestRootActivationProductionInvariantBootStep(t *testing.T) {

if c.Boot == string(boot) {

t.Fatal("no actual reboot")

}

+ if !c.Pending {

+ if _, err := os.Lstat(filepath.Join(tx.updateDir(), "journal.json")); !os.IsNotExist(err) {

+ t.Fatal("absent-journal reboot resumed with a journal", err)

+ }

activationMatrixCheck(t, tx, "boot-resume")

if c.Pending && offlineRecord(t, tx).Phase == "done" {

activationMatrixRecovered(t, tx)

@@ -296,6 +313,7 @@ func TestRootActivationProductionInvariantBootStep(t *testing.T) {

point, interrupted := activationMatrixInterrupt(t, tx, c.Cut)

if !interrupted {

activationMatrixResume(t, tx, point)

+ t.Logf("matrix completion-control timer=%t boot_pending=%t cut=%d interrupted=false", c.Timer, os.Getenv("ACTIVATION_MATRIX_PENDING") != "absent", c.Cut)

if c.Timer {

cleanupActivationProduction(t, tx)

if err := os.Remove("/etc/tmpfiles.d/carlos.conf"); err != nil && !os.IsNotExist(err) {

@@ -309,6 +327,11 @@ func TestRootActivationProductionInvariantBootStep(t *testing.T) {

continue

}

c.Pending = os.Getenv("ACTIVATION_MATRIX_PENDING") != "absent"

+ if !c.Pending {

+ if _, err := os.Lstat(filepath.Join(tx.updateDir(), "journal.json")); !os.IsNotExist(err) {

+ t.Fatal("absent-journal reboot prepared with a journal", err)

+ }

if c.Pending && offlineRecord(t, tx).Phase == "done" {

activationMatrixClosedJournal(t, tx)

} else if c.Pending {

@@ -486,14 +509,20 @@ func TestRootActivationProductionDonePendingMatrix(t *testing.T) {

t.Fatal(err)

}

tx.checkpoint = nil

- cuts = append(cuts, count+1)

- for _, cut := range cuts {

+ if len(cuts) == 0 {

+ t.Fatal("no closed continuation checkpoints")

+ }

+ complete := false

+ for cut := cuts[0]; cut <= 120; cut++ {

activationMatrixBaseline(t, tx, timer)

- point, _ := activationMatrixInterrupt(t, tx, cut)

+ point, interrupted := activationMatrixInterrupt(t, tx, cut)

r := offlineRecord(t, tx)

if r.Phase != "done" {

t.Fatal("closed cut changed", cut, point, r.Phase)

}

+ if !interrupted && (r.EdgePending || r.TriggersPending) {

+ t.Fatal("uninterrupted control left continuation pending")

+ }

activationMatrixClosedJournal(t, tx)

if order == "pending-first" {

if r.EdgePending || r.TriggersPending {

@@ -503,7 +532,12 @@ func TestRootActivationProductionDonePendingMatrix(t *testing.T) {

if err != nil {

t.Fatal(err)

}

- if err := tx.restore(t.Context(), false); err != nil {

+ noop := tx

+ noop.systemctl = func(context.Context, ...string) ([]byte, error) {

+ t.Fatal("finished transaction called systemctl")

+ return nil, nil

+ }

+ if err := noop.restore(t.Context(), false); err != nil {

t.Fatal("finished transaction resumed work", err)

}

after, err := os.ReadFile(filepath.Join(tx.state, "transaction.json"))

@@ -514,7 +548,14 @@ func TestRootActivationProductionDonePendingMatrix(t *testing.T) {

}

activationMatrixRecovered(t, tx)

activationMatrixResume(t, tx, point)

- t.Logf("post-done reboot=false timer=%t pending=recovered order=%s cut=%d point=%s", timer, order, cut, point)

+ t.Logf("post-done reboot=false timer=%t pending=recovered order=%s cut=%d point=%s interrupted=%t", timer, order, cut, point, interrupted)

+ if !interrupted {

+ complete = true

+ break

+ }

+ if !complete {

+ t.Fatal("post-done matrix did not reach uninterrupted completion")

}

})