amadan

carlos / platform Public

claims-on-gate-recover

Squashed from claims-on-gate-recover into main.
claude abcdd57789575d9cdeb6f9cdecab54f90e3c08b6 parent 521c6ee
19 files changed, +3080 −107
  • docs/superpowers/notes/census-proposal-claims-on-gate-recover.md +263 −0
  • internal/host/guardtransaction.go +130 −9
  • internal/host/plan14_generation_kernel_linux_test.go +3 −3
  • internal/host/recover_settlement_test.go +797 −0
  • internal/host/rootgeneration_census_linux.go +111 −7
  • internal/host/rootgeneration_census_recover_linux_test.go +792 −0
  • internal/host/rootgeneration_kernel_linux.go +40 −5
  • internal/host/rootgeneration_refusal.go +42 −0
  • internal/host/rootgeneration_retired_linux.go +2 −1
  • internal/host/rootgeneration_services_linux.go +314 −49
  • internal/host/rootmovesfloor_refresh.go +49 −4
  • internal/host/systemapply.go +116 −24
  • internal/host/systemapply_claimsgate_test.go +240 −0
  • internal/host/systemparticipation.go +23 −5
  • internal/provision/plan14_root_reads_test.go +21 −0
  • internal/provision/testdata/plan14-root-reads-claims-on-gate-recover-merge-main.tsv +2 −0
  • internal/provision/testdata/plan14-root-reads-claims-on-gate-recover.tsv +33 −0
  • internal/rootparticipation/holdupdate_test.go +68 −0
  • internal/rootparticipation/ledger.go +34 −0

CI passed — run details

CI log 
=== gate ===
.amadan/test-gate.sh
=== launchctx (1 s)
ok  	amadan.net/carlos/platform.git/launchctx	0.012s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
=== vet (19 s)
=== gofmt (0 s)
gofmt -l .: clean
=== rest (157 s)
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.591s
ok  	github.com/carlosframework/platform/cmd/moveprobe	11.781s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.746s
ok  	github.com/carlosframework/platform/internal/activator	63.332s
ok  	github.com/carlosframework/platform/internal/agentsock	0.029s
ok  	github.com/carlosframework/platform/internal/alerting	0.008s
ok  	github.com/carlosframework/platform/internal/analytics	0.036s
ok  	github.com/carlosframework/platform/internal/appconfig	2.963s
ok  	github.com/carlosframework/platform/internal/apperr	0.124s
ok  	github.com/carlosframework/platform/internal/applog	0.215s
ok  	github.com/carlosframework/platform/internal/awsinv	0.014s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.013s
ok  	github.com/carlosframework/platform/internal/boxid	0.610s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.013s
ok  	github.com/carlosframework/platform/internal/caps	0.024s
ok  	github.com/carlosframework/platform/internal/certwarm	0.007s
ok  	github.com/carlosframework/platform/internal/cgroup	0.769s
ok  	github.com/carlosframework/platform/internal/changelog	0.427s
ok  	github.com/carlosframework/platform/internal/claimactivation	1.021s
ok  	github.com/carlosframework/platform/internal/clamd	0.065s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.157s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.011s
ok  	github.com/carlosframework/platform/internal/dbverify	0.010s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.007s
ok  	github.com/carlosframework/platform/internal/docsite	0.013s
ok  	github.com/carlosframework/platform/internal/domains	2.745s
ok  	github.com/carlosframework/platform/internal/doorkey	0.022s
ok  	github.com/carlosframework/platform/internal/economics	0.025s
ok  	github.com/carlosframework/platform/internal/edge	45.757s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.009s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.174s
ok  	github.com/carlosframework/platform/internal/gate	1.667s
ok  	github.com/carlosframework/platform/internal/genselect	0.030s
ok  	github.com/carlosframework/platform/internal/geo	0.038s
ok  	github.com/carlosframework/platform/internal/geoip	0.037s
ok  	github.com/carlosframework/platform/internal/historyreadback	0.229s
ok  	github.com/carlosframework/platform/internal/host	127.408s
ok  	github.com/carlosframework/platform/internal/hostactors	5.683s
ok  	github.com/carlosframework/platform/internal/hostactors/awsmembership	0.010s
ok  	github.com/carlosframework/platform/internal/hostbox	2.959s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.891s
ok  	github.com/carlosframework/platform/internal/hostenv	0.007s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.006s
ok  	github.com/carlosframework/platform/internal/hostexec	5.355s
ok  	github.com/carlosframework/platform/internal/hostfence	0.060s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.280s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.020s
ok  	github.com/carlosframework/platform/internal/hostguard	1.070s
ok  	github.com/carlosframework/platform/internal/hostops	1.952s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.002s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.004s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.687s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.242s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.006s
ok  	github.com/carlosframework/platform/internal/infraview	0.008s
ok  	github.com/carlosframework/platform/internal/instancectl	5.231s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.798s
ok  	github.com/carlosframework/platform/internal/instancefs	0.011s
ok  	github.com/carlosframework/platform/internal/instancekey	0.005s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.013s
ok  	github.com/carlosframework/platform/internal/layout	1.811s
ok  	github.com/carlosframework/platform/internal/lbhealth	0.009s
ok  	github.com/carlosframework/platform/internal/lease	10.187s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.595s
ok  	github.com/carlosframework/platform/internal/ledger	0.022s
ok  	github.com/carlosframework/platform/internal/mail	0.261s
ok  	github.com/carlosframework/platform/internal/maintenance	0.097s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.620s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.106s
ok  	github.com/carlosframework/platform/internal/metrics	0.010s
ok  	github.com/carlosframework/platform/internal/moves	22.384s
ok  	github.com/carlosframework/platform/internal/moves/archive	1.074s
ok  	github.com/carlosframework/platform/internal/objstore	0.380s
ok  	github.com/carlosframework/platform/internal/ops	0.227s
ok  	github.com/carlosframework/platform/internal/park	0.317s
ok  	github.com/carlosframework/platform/internal/plans	0.006s
ok  	github.com/carlosframework/platform/internal/privatefile	0.005s
ok  	github.com/carlosframework/platform/internal/provision	23.577s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.009s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	5.649s
ok  	github.com/carlosframework/platform/internal/relay	7.238s
ok  	github.com/carlosframework/platform/internal/release	26.704s
ok  	github.com/carlosframework/platform/internal/rollout	3.819s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.003s
ok  	github.com/carlosframework/platform/internal/rootaws	0.009s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.059s
ok  	github.com/carlosframework/platform/internal/rootstate	0.032s
ok  	github.com/carlosframework/platform/internal/runs	0.568s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.005s
ok  	github.com/carlosframework/platform/internal/schedule	2.757s
ok  	github.com/carlosframework/platform/internal/seeds	0.021s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	11.795s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.409s
ok  	github.com/carlosframework/platform/internal/status	0.107s
ok  	github.com/carlosframework/platform/internal/steering	0.008s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.321s
ok  	github.com/carlosframework/platform/internal/svcrole	0.003s
ok  	github.com/carlosframework/platform/internal/telemetry	0.006s
ok  	github.com/carlosframework/platform/internal/termfence	0.005s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.006s
ok  	github.com/carlosframework/platform/internal/unixdial	0.006s
ok  	github.com/carlosframework/platform/internal/updatewords	0.005s
ok  	github.com/carlosframework/platform/internal/usage	0.006s
ok  	github.com/carlosframework/platform/internal/version	0.006s
ok  	github.com/carlosframework/platform/internal/vet	1.740s
=== internal_console@0 (77 s)
ok  	github.com/carlosframework/platform/internal/console	76.067s
=== internal_console@1 (37 s)
ok  	github.com/carlosframework/platform/internal/console	35.612s
=== internal_console@2 (43 s)
ok  	github.com/carlosframework/platform/internal/console	41.804s
=== internal_console@3 (38 s)
ok  	github.com/carlosframework/platform/internal/console	36.362s
=== internal_console@4 (37 s)
ok  	github.com/carlosframework/platform/internal/console	36.125s
=== internal_console@5 (35 s)
ok  	github.com/carlosframework/platform/internal/console	33.971s
=== internal_console@6 (38 s)
ok  	github.com/carlosframework/platform/internal/console	36.332s
=== internal_console@7 (44 s)
ok  	github.com/carlosframework/platform/internal/console	42.451s
=== root@0 (34 s)
ok  	github.com/carlosframework/platform	32.758s
=== root@1 (45 s)
ok  	github.com/carlosframework/platform	43.392s
=== root@2 (46 s)
ok  	github.com/carlosframework/platform	45.154s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
linux/arm64 carlos binary: 49938594 bytes (ceiling 52000000)
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.025s
=== RUN   TestT4StorageProbeMinio
2026/10/21 19:22:57 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18E0A1D2ADD36E35, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.024s
minio contract: PASS
diff --git a/docs/superpowers/notes/census-proposal-claims-on-gate-recover.md b/docs/superpowers/notes/census-proposal-claims-on-gate-recover.md
new file mode 100644
index 000000000..5f73ca302
--- /dev/null
+++ b/docs/superpowers/notes/census-proposal-claims-on-gate-recover.md
@@ -0,0 +1,263 @@
+# Census: the claims-on update gate and recover's generation census
+
+Status: PROPOSED. Every row carries `rev3=PROPOSED-claims-on-gate-recover`.
+The ruling is asked of the census owner on #280.
+
+The branch is `claims-on-gate-recover`, from main 3d444a1b. It fixes two
+defects a deployment harness found on a box running with CARLOS_HOST_CLAIM
+on: (a) the update gate rolled every update back, and its rollback too,
+because a claims-on edge takes about 22 s to listen and the gate waited 20 s;
+(b) carlos-system-recover's root generation census refused racily and, since
+the edge's guard drop-in `Requires=` the recover unit, kept the edge down.
+
+It adds one census overlay,
+`internal/provision/testdata/plan14-root-reads-claims-on-gate-recover.tsv`,
+loaded last in `TestPlan14RootReadCensus`, after scan-demand-revival. Every
+row is authority. There is no legacy row and no frozen declaration, so the
+seal stands at `a21dba12…f0cc262`. The `internal/host` envelope is re-pinned.
+The record-write census (`TestPlan9B8RecordWriteCensus`) does not change: no
+record write is added or moved.
+
+## Ruling 6
+
+No-census boxes see no change, and claims-off boxes see none in the gate. A
+claims-off box that retains a census (claims were on and were turned off)
+still uses root participation, so it receives the recover settlement
+changes below exactly as a claims-on box does; that is the existing
+retained-census rule, not a new one.
+
+- The gate's new window and fail-fast are keyed on CARLOS_HOST_CLAIM, read
+ with `hostwrite.Enabled` from the update and recover units' environment
+ (both load host.env). Claims off, the gate is exactly the 20 s one and
+ reads no `systemctl show` inside its wait
+ (`TestClaimsOffGateIsUnchangedByAnEdgeThatListensAt22s`,
+ `TestClaimsOffGateWaitsItsFullWindowThroughACrash`).
+- The recover deferral applies only where root participation is required
+ (a census). A no-census box's refused settlement fails recover as on main
+ (`TestRuling6NoCensusRecoverIsUnchanged`).
+- The generation census itself still runs only where it ran before: it is
+ reached only through the ledger's launch verifier, which a no-census box
+ never consults.
+
+## The units
+
+### (a) The update gate
+
+| unit | change | executing identity | disposition |
+|---|---|---|---|
+| `internal/host/systemapply.go` `RunSystemApply` | re-pinned, no read of its own: picks the gate window through `systemApplyGate` (CARLOS_HOST_CLAIM by `hostwrite.Enabled`) | root apply and recovery, uid 0 | authority, D,F,J4, unchanged |
+
+The gate (`systemApplier.gate`, `gateCheck`, `gateFailFast`, `systemGateWaitFor`)
+reads only `systemctl is-active` and `systemctl show` answers it already read,
+and has no row.
+
+### (b) Recover's generation census
+
+| unit | change | executing identity | disposition |
+|---|---|---|---|
+| `internal/host/guardtransaction.go` `systemApplier.checkGuardRecovery` | re-pinned: its reads are unchanged; the two no-journal settlements call `settleWithoutJournal`, which on a census box wraps a refused `controllersRecovered` in `errSettlementDeferred` | root system update and recovery, uid 0 | authority, D, unchanged |
+| `internal/host/systemapply.go` `systemApplier.Run` | re-pinned: its lock-file read is unchanged; under `--recover` a deferred settlement is logged and not fatal, and is not tried again at the end of the run | root apply and recovery, uid 0 | authority, D,F,J4, unchanged |
+| `internal/host/systemparticipation.go` `systemApplier.checkUnarmedParticipationRecovery` | re-pinned, no read of its own: with no journal open its settlement goes through `settleWithoutJournal` | root system update and recovery, uid 0 | authority, G,F, unchanged |
+| `internal/host/rootgeneration_kernel_linux.go` `generationKernel.absent` | re-pinned: same reads; a task refusal is typed (`generationTaskRefusal`, same text) so the retry can tell task-level refusals from structural ones | root census, uid 0 | authority, E2, unchanged |
+| `internal/host/rootgeneration_kernel_linux.go` `generationKernel.task` | re-pinned: same reads; a start time that moves mid-inspection refuses as `errGenerationTaskReplaced` (still `os.ErrPermission`); every reference check still runs on every accepted service task | root census, uid 0 | authority, E2, unchanged |
+| `internal/host/rootgeneration_census_linux.go` `generationCensusRetry` | re-pinned, no read of its own: whole passes; every pass and its blocking reads run under one 20 s deadline, and a pass that ends past it refuses; task-level refusals a later pass can clear retry with backoff from 250 ms inside it; reference, namespace, mount and structural refusals refuse at once | root census, uid 0 | authority, E2, unchanged |
+| `internal/host/rootgeneration_services_linux.go` `generationServiceReader` | re-pinned, now a binding with no read: the production `generationServiceEnv` | root census, uid 0 | authority, E2, unchanged |
+| `internal/host/rootgeneration_services_linux.go` `generationServiceEnv.reader` | new: `generationServiceReader`'s body. The exe `Readlink` moves first, for diagnostics only. Every refusal names check, unit, cgroup and exe. A non-root task, or a root task of a unit with a selected image (the guarded edge), whose exe reads `<path> (deleted)` goes to `deletedProgram` | root census, uid 0 | authority, E2 (its caller's) |
+| `internal/host/rootgeneration_services_linux.go` `generationServiceShow` | new: the `systemctl show` moved out unchanged | root census, uid 0 | authority, E2 (its caller's) |
+| `internal/host/rootgeneration_services_linux.go` `generationServiceTrust` | new: the unit-file `EvalSymlinks` and `OpenTrustedFile` moved out unchanged | root census, uid 0 | authority, E2 (its caller's) |
+| `internal/host/rootgeneration_services_linux.go` `generationServiceEnv.deletedProgram` | new root read: opens a service task's image through `/proc/<pid>/exe`, `Fstat`s it and hashes it | root census, uid 0, in carlos-system-recover, carlos-system-update, instance-control's start and the guard-exec prelude | authority, F,J4 |
+
+### Why the deleted-exe read is F,J4
+
+`deletedProgram` makes the proof the guard makes before it execs a binary,
+on the same kind of input. The guard's reads are F,J4 root-input:
+`hostfloor` `OpenProtected`, `Disk.VerifyFile` and `ExecutingHash` (which
+opens `/proc/self/exe` and hashes it), and in this package
+`readRootFloorBinary`. `deletedProgram` opens the running image of another
+task the same way, requires a regular file owned by root, with no links left
+and no group or other write, hashes at most 512 MiB of it, and accepts only a
+digest equal to the signed router build of `<opt>/carlos` or
+`<opt>/carlos.prev`, which `readRootFloorBinary` proves from the identity
+cache the box holds. Classifying it at E2 with its caller would understate
+what it decides: a match is what lets an unselected inode count as the
+unit's program.
+
+It is reached for a non-root task, and for a root task of a unit with a
+selected image (the guarded edge, whose selected program is
+`<opt>/carlos`), when the unit selects `<opt>/carlos` and the exe link reads
+`<opt>/carlos (deleted)`. A root task of a unit with no selected image keeps
+main's path walk, which a deleted exe fails, so the strict root scan
+applies to it (`TestRecoverCensusDeletedExeOfARootTask`). Anything else
+refuses as before. Acceptance only lets the census go on to the
+reference checks: exe, cwd, root, descriptors, maps and mounts are still
+compared against the retired generations
+(`TestRecoverCensusStillChecksReferencesOfAnAcceptedDeletedExe`).
+
+## Fix round 1 (Astra's review of 8eed5c19)
+
+| unit | change | disposition |
+|---|---|---|
+| `systemApplier.checkGuardRecovery` | every no-journal settlement (controllers census, completion or withdrawal, the census again inside `EndParticipation`) in the marker, no-marker and already-completed branches goes through `settleWithoutJournal`, which now defers only a result that is a census refusal and nothing else (`onlyCensusRefusal`); an unfinished transaction's marker is removed inside the settlement, after its census. Round 0's deferral of any controller failure is withdrawn | authority, D, re-pinned |
+| `systemApplier.checkUnarmedParticipationRecovery` | the same census-only rule | authority, G,F, re-pinned |
+| `productionGenerationKernelCensus` | every refusal typed `generationCensusRefusal`; each pass's lookup is built on the census deadline context and wires its recheck | authority, E2, re-pinned at main's class |
+| `productionGenerationAbsence`, `generationRetiredController`, `type generationKernel` | wire or carry the lookup's recheck | authority, E2, re-pinned at main's class |
+| `generationKernel.task` | holds a trusted task to its proven image: recheck before each use of the trust and at the end; no recheck wired refuses every trusted task | authority, E2, re-pinned |
+| `generationCensusRetry` | one 20 s deadline shared by every pass and its blocking reads; a pass ending past it refuses | authority, E2, re-pinned |
+| `generationServiceReader`, `generationServiceEnv.reader`, `generationServiceEnv.deletedProgram` | the lookup returns classify and recheck; the hashed descriptor is kept to bind the task; the hash stops at the deadline | re-pinned at their classes |
+| `generationServiceEnv.recheck` (new root read) | re-opens `/proc/<pid>/exe` and requires the hashed inode (device, inode, mount ID, file handle), still root-owned, unlinked, not group or other writable | authority, F,J4, as `deletedProgram` |
+| `generationExeIdentity` (new) | `Fstat`, `Statx` and `name_to_handle_at` of an open image descriptor | authority, F,J4, as its callers |
+| `generationContextReader.Read` (new) | the hash's reader over the already open image; stops at the deadline | authority, F,J4, as its caller |
+
+No execution exclusion exists for a service task's own `exec`: root's
+launch exclusion (`participationRelease`, `Ledger.ExcludeLaunches`) covers
+claim launches only. The rechecks are sampled identity checks at observation
+points (before each use of the trust and at the end of the inspection), not
+execution exclusion: a replacement visible at a recheck refuses, but a task
+that execs an unsigned helper after the final `/proc/<pid>/exe` open keeps
+its classification until the next pass. Main's live-exe path (the
+selected-program walk and plain non-root services) has the same race after
+its one observation; this branch extends that exposure to the deleted,
+root-owned, protected inodes whose bytes match the signed current or
+previous router build, which main refused outright. It does not admit an
+unsigned helper on its own. The binding applies only to deleted-image
+acceptance; live-image rechecks return at once. Main's path is unchanged
+here and needs a ruling. Still no legacy row, so the
+seal stands; `internal/host`'s envelope is re-pinned again.
+
+## Fix round 2 (Astra's round 2 on ec057b9b)
+
+- **Post-journal settlement (finding 1a).** `closeGuardJournal` durably
+ removes the journal, then settles (`endGuard`, `clearUpdateBegun`). That
+ settlement ran unwrapped, so a census refusal there failed recover and the
+ edge's dependency job with the files whole and the journal gone: the Run
+ 11b/11d shape. It now goes through `settleWithoutJournal`, forward and
+ backward, guarded and unarmed, and `Run` under `--recover` consumes the
+ deferral from `recoverIfNeeded` without keeping the fatal error or
+ retrying settlement. Non-census failures stay fatal.
+- **Acquisition barrier (finding 1b).** With no update record (a prepared
+ transaction never begun), `EndParticipation`'s `BeginUpdate` withdrew the
+ record it wrote when its census refused, so a deferral left `Acquire`
+ open. `settleWithoutJournal` now calls `Ledger.HoldUpdate` first, under the
+ transaction lock: root's update record kept, or written under the
+ transaction's id, with no census of its own. `Acquire` reads the ledger
+ census and `update.json`, so it stays refused while recover defers. A
+ barrier that cannot be written leaves the refusal fatal. An
+ already-completed transaction with no record left replays at once and
+ runs no census. `Ledger.HoldUpdate` is a new unit at its siblings' class
+ (D).
+- **The signed-image proof under the deadline (finding 3).**
+ `readRootFloorBinary` becomes a wrapper (no read) over
+ `readRootFloorBinaryContext`, whose hash stops between reads and whose
+ identity scan stops before each identity once the census context ends.
+ `generationContextReader` moves beside it. The two helpers are witnessed
+ directly; the lookup is witnessed to hand the deadline to the proof; the
+ plumbing from `readRootFloorBinaryContext` into the helpers is not
+ witnessed by a non-root test (the proof needs root-owned files), so a
+ mutant that drops the context inside it survives.
+- **Prose (finding 4).** The rows and this note now describe the rechecks
+ as sampled identity checks, name the final-open race as shared with main's
+ live-exe path, state the deadline's coverage and its gaps, and no longer
+ say the three restored tests expect exit 0.
+
+## Fix round 3 (Astra's round 3 on d16cb5f0)
+
+- **The legacy marker's settlement (finding 1b, unarmed).** With root
+ participation, no armed floor, a legacy `update-begun.json` and neither an
+ update record nor a journal, `checkUnarmedParticipationRecovery` returns
+ without settling and the run's end settled the marker through
+ `clearUpdateBegun` directly, logging any error: its `BeginUpdate` was
+ refused by the census, which withdrew the record it wrote, so recover
+ exited 0 with acquisitions open. `settleUpdateBegun` now settles through
+ `settleWithoutJournal` with the `recover-legacy` barrier, and `settleMarker`
+ decides what the outcome means for the run: a deferral under `--recover`
+ is logged and left to a later run; a barrier that could not be written
+ (`errAcquisitionBarrier`) joins the run's error in every mode; every
+ other marker failure is logged as on main.
+ `TestRecoverHoldsAcquisitionsForALegacyMarker`,
+ `TestRecoverFailsALegacyMarkerWhoseBarrierCannotBeWritten`.
+- **Prose (findings 2 and 3 of round 3).** The deadline is described as
+ cooperative cancellation at checkpoints with the uncancelled work named,
+ and the boot-delay figures as estimates; every row that said a trusted
+ task is "held to" its image now says it is rechecked at observation points
+ and names what a recheck cannot see.
+
+## Fix round 4 (Astra's round 4 on 2a823aef)
+
+- **A barrier prerequisite failure is a barrier failure.** In
+ `settleWithoutJournal`, when `ParticipationRequired` itself failed (the
+ store did not open, a retained census did not read) the original census
+ refusal was returned bare, so the legacy marker's settlement logged it and
+ recover exited 0 without a barrier. It now returns `errAcquisitionBarrier`
+ with both errors kept, which `settleMarker` makes fatal in every mode; a
+ box where participation is not required still falls back as before.
+ `TestRecoverFailsALegacyMarkerWhoseBarrierPrerequisiteFails` injects the
+ failure after `BeginUpdate` has withdrawn its record.
+- **Ruling 6 wording** qualified: a claims-off box with a retained census
+ still uses participation and receives the settlement changes.
+
+## The census window and the worst-case boot delay
+
+Each call of root's kernel census runs under one deadline, 20 s from its
+start (`generationCensusRetryFor`). Every pass gets a context that ends at
+the deadline. The deadline is cooperative cancellation at the implemented
+checkpoints, plus the rejection of late results: `systemctl show` runs
+through `exec.CommandContext`; the deleted-image hash checks the context
+between reads; (fix round 2) the signed-image proof,
+`readRootFloorBinaryContext`, checks it between reads of `<opt>/carlos` and
+`<opt>/carlos.prev` and before each cached identity. A pass that ends past
+the deadline refuses even if it succeeded, and no pass starts after it.
+Between checkpoints the work is not cancelled: the opens and metadata
+operations, the identity cache's directory enumeration (`Names`, a
+`ReadDir`), a whole identity record's read (`io.ReadAll`), a `systemctl
+show` already started, one `Read` of an image, and the versions-tree walk and
+mount checks before the first pass (bounded at 8192 entries). The figures
+below are therefore estimates, not bounds.
+
+On the recover path, with the census refusing every time, the edge's start
+waits at most for, in order:
+
+1. the guard-exec recover prelude's census (`rootFloorDowngradeIdle`), when
+ a moves floor is armed: 20 s;
+2. recover's first controllers census: 20 s;
+3. root's pending-controller recovery (`recoverRootExecution`), which runs
+ under its own 30 s context, plus two cleanups of up to 30 s each;
+4. recover's second controllers census: 20 s.
+
+Then recover defers and exits 0. That is about 90 s, and about 150 s when
+both cleanups run to their bounds, as estimates: each 20 s window can
+overrun by the uncancelled work named above. The recover unit's TimeoutStartSec is
+10 min. instance-control's start runs its own census under the same 20 s
+window.
+
+## Tests named in the rows
+
+- (a): `TestSystemGateWaitIsKeyedOnClaimsMode`,
+ `TestSystemApplyReadsClaimsModeFromTheUnitEnvironment`,
+ `TestClaimsOnGatePassesAnEdgeThatListensAt22s`,
+ `TestClaimsOnGateWaitsUpTo120s`, `TestClaimsOnGateFailsFastOnACrashRestart`,
+ `TestClaimsOnGateFailsFastOnAFailedOrInactiveEdge`,
+ `TestClaimsOnRollbackGateUsesTheSameRule`,
+ `TestClaimsOnRollbackGateFailsFastToo`, and the two claims-off tests above.
+- (b) fix 1: `TestRecoverDefersACensusRefusalWithNoJournalOpen`,
+ `TestRecoverFailsACensusRefusalWithAJournalOpen`,
+ `TestRecoverDefersACensusRefusalWithoutAnArmedFloor`,
+ `TestApplyStillFailsACensusRefusalWithNoJournalOpen`,
+ `TestRuling6NoCensusRecoverIsUnchanged`; `TestGuardRecoveryRefusesControllersAfterBegin`,
+ `TestPlan9M8GuardRecoveryRecoversPendingWithoutJournal` and
+ `TestSystemGuardRegionHAKeepsFailedCompletionEvidence` are unchanged from
+ main: deferral is census-only (fix round 1), so a live or unreadable
+ controller and a failed completion still fail as they did.
+- (b) fix 2: `TestRecoverCensusAcceptsADeletedExeOfTheCurrentOrPreviousBuild`,
+ `TestRecoverCensusAcceptsTheEdgesDeletedExeInSupervise`,
+ `TestRecoverCensusRefusesOtherDeletedExes`,
+ `TestRecoverCensusRefusesADeletedExeNotOwnedByRoot`,
+ `TestRecoverCensusStillChecksReferencesOfAnAcceptedDeletedExe`.
+- (b) fix 3: `TestRecoverCensusRetriesATransientRefusalWithinTheWindow`,
+ `TestRecoverCensusRetriesOnlyTaskLevelTransients`,
+ `TestPlan14GenerationCensusRetriesWholeObservation` (unchanged).
+- (b) fix 4: `TestRecoverCensusRefusalsNameCheckUnitCgroupAndExe`.
+
+The census tests drive the production service lookup over a fake `/proc`,
+replacing only `systemctl`, the unit files' ownership check, the
+signed-binary proof and the owner uid, which a non-root test cannot provide.
+No root-runner test was run for this branch.
diff --git a/internal/host/guardtransaction.go b/internal/host/guardtransaction.go
index 3a2dacc3b..5d3c91eb9 100644
--- a/internal/host/guardtransaction.go
+++ b/internal/host/guardtransaction.go
@@ -4,10 +4,12 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"github.com/carlosframework/platform/internal/hostactors"
execlease "github.com/carlosframework/platform/internal/hostexec"
"github.com/carlosframework/platform/internal/hostfloor"
"github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootparticipation"
"os"
"path/filepath"
"slices"
@@ -137,10 +139,14 @@ func (a *systemApplier) settleGuard(ctx context.Context, tx guardTransaction, be
return err
}
if !finished {
- if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")); err != nil {
- return err
- }
- return a.floor.EndParticipation(ctx, tx.ID, func() error { return hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json")) })
+ // The marker goes inside the settlement, after root's census, so a
+ // census that refuses leaves it with the rest of the transaction.
+ return a.floor.EndParticipation(ctx, tx.ID, func() error {
+ if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")); err != nil {
+ return err
+ }
+ return hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json"))
+ })
}
if beforeCompletion != nil {
if err := beforeCompletion(); err != nil {
@@ -248,12 +254,19 @@ func (a *systemApplier) checkGuardRecovery(ctx context.Context) error {
}
completed, readErr := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json"))
if readErr == nil && string(completed) == tx.ID {
- return a.floor.EndParticipation(ctx, tx.ID, nil)
+ // Completion recorded and root's marker gone. With root's update
+ // record gone too the transaction is settled: a replay has
+ // nothing left to close, and runs no census.
+ open, err := a.updateRecordOpen(ctx)
+ if err != nil || !open {
+ return err
+ }
+ return a.settleWithoutJournal(ctx, tx.ID, a.floor.EndParticipation(ctx, tx.ID, nil))
}
if readErr != nil && !errors.Is(readErr, os.ErrNotExist) {
return readErr
}
- return a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) })
+ return a.settleWithoutJournal(ctx, tx.ID, a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) }))
}
if err != nil {
return err
@@ -269,7 +282,7 @@ func (a *systemApplier) checkGuardRecovery(ctx context.Context) error {
return hostfloor.ErrFloor
}
if j == nil {
- return a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) })
+ return a.settleWithoutJournal(ctx, tx.ID, a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) }))
}
if err := a.controllersRecovered(ctx); err != nil {
return err
@@ -279,6 +292,104 @@ func (a *systemApplier) checkGuardRecovery(ctx context.Context) error {
}
return a.guardImage(systemPrevPath(a.optDir))
}
+
+// errSettlementDeferred marks a settlement that root's generation census
+// refused with no journal open. No apply is half done and the files are
+// whole, so there is nothing for recovery to restore and no reason to keep
+// the edge down (its drop-in Requires= the recover unit). The transaction
+// evidence stays as it was: the reservation, root's marker and update
+// record, so every new claim acquisition stays refused, and the next apply,
+// or recover on the next edge start, settles it. Run under --recover logs
+// it and exits 0; the apply returns it as before. beginExecutableChange
+// treats a refused census as a deferral too (systemparticipation.go).
+var errSettlementDeferred = errors.New("participation settlement deferred")
+
+// errAcquisitionBarrier marks a deferral that could not hold claim
+// acquisitions refused (Ledger.HoldUpdate failed). It is never swallowed:
+// recover must not exit 0 with acquisitions open (Astra rounds 2 and 3).
+var errAcquisitionBarrier = errors.New("acquisition barrier")
+
+// settleWithoutJournal is the result of one settlement made with no
+// journal open: the pending-controller recovery, the completion or
+// withdrawal, and the end of root's update record, whose census runs again
+// inside EndParticipation; and, after an apply's journal is durably
+// removed, the settlement that follows it (closeGuardJournal). A result
+// that is only a census refusal, on a box with root participation, is
+// wrapped in errSettlementDeferred, once acquisitions are held refused:
+// root's update record is kept, or, where the refused census was the one
+// that would have begun it, written under barrier's id (Ledger.HoldUpdate).
+// Anything else is returned unchanged: an image that does not verify,
+// malformed state, a live or unreadable controller, a durable write that
+// failed, a barrier that could not be written, and every refusal on a box
+// without a census, which sees exactly main's recovery (ruling 6).
+func (a *systemApplier) settleWithoutJournal(ctx context.Context, barrier string, err error) error {
+ if err == nil || !onlyCensusRefusal(err) || ctx.Err() != nil {
+ return err
+ }
+ required, checkErr := a.participationDisk().ParticipationRequired(ctx)
+ if checkErr != nil {
+ // Whether a barrier is needed could not be established (the store
+ // did not open, a retained census did not read): that is a barrier
+ // failure too, never a plain refusal a caller may log (Astra
+ // round 4). Both errors are kept.
+ return errors.Join(err, fmt.Errorf("%w: participation required: %w", errAcquisitionBarrier, checkErr))
+ }
+ if !required {
+ return err
+ }
+ if holdErr := a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error {
+ return l.HoldUpdate(ctx, barrier)
+ }, func() error { return rootparticipation.ErrUnavailable }); holdErr != nil {
+ return errors.Join(err, fmt.Errorf("%w: %w", errAcquisitionBarrier, holdErr))
+ }
+ return fmt.Errorf("%w: no journal is open, so the files are whole; root's update record holds claim acquisitions refused until the settlement passes: %w", errSettlementDeferred, err)
+}
+
+// updateRecordOpen reports whether root's update record is there, on a box
+// with root participation. Without one it is false: main's legacy path.
+func (a *systemApplier) updateRecordOpen(ctx context.Context) (bool, error) {
+ open := false
+ err := a.participationDisk().WithParticipation(ctx, func(_ context.Context, l rootparticipation.Ledger) error {
+ _, err := l.UpdateID()
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ open = err == nil
+ return err
+ }, func() error { open = true; return nil })
+ return open, err
+}
+
+// deferSettlement reports whether Run, under --recover, leaves err's
+// settlement to a later run instead of failing: only an errSettlementDeferred.
+// It logs why.
+func (a *systemApplier) deferSettlement(recover bool, err error) bool {
+ if !recover || !errors.Is(err, errSettlementDeferred) {
+ return false
+ }
+ a.ops.logf("system apply: recover: %v; the edge may start, and the next apply or recover settles it", err)
+ return true
+}
+
+// settleMarker settles the update-begun marker at the end of a run
+// (settleUpdateBegun) and decides what its outcome means for the run: a
+// deferral under --recover is logged and left to a later run; a barrier
+// that could not be written joins the run's error, whatever the mode,
+// because acquisitions would otherwise be open with the settlement
+// unmade; a deferral under the apply is logged, as every marker failure
+// was on main, with acquisitions held refused by the barrier.
+func (a *systemApplier) settleMarker(ctx context.Context, recover bool, retErr *error) {
+ err := a.settleUpdateBegun(ctx)
+ if err == nil || a.deferSettlement(recover, err) {
+ return
+ }
+ if errors.Is(err, errAcquisitionBarrier) {
+ *retErr = errors.Join(*retErr, err)
+ return
+ }
+ a.ops.logf("system apply: %v", err)
+}
+
func (a *systemApplier) endGuard(ctx context.Context) error {
if !a.guarded() {
return nil
@@ -352,10 +463,20 @@ func (a *systemApplier) closeGuardJournal(ctx context.Context) error {
if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil {
return err
}
+ // The journal is gone and the files are whole: from here a settlement
+ // the census refuses is deferred as with no journal open.
if !active {
- return a.clearUpdateBegun(ctx)
+ return a.settleWithoutJournal(ctx, "recover-legacy", a.clearUpdateBegun(ctx))
+ }
+ err = a.endGuard(ctx)
+ if err == nil || !onlyCensusRefusal(err) {
+ return err
+ }
+ tx, txErr := a.guardTransaction()
+ if txErr != nil {
+ return errors.Join(err, txErr)
}
- return a.endGuard(ctx)
+ return a.settleWithoutJournal(ctx, tx.ID, err)
}
func (a *systemApplier) guardRun(ctx context.Context, wait time.Duration, name string, args ...string) (string, error) {
if id, required, err := a.rootMoves.read(); err != nil {
diff --git a/internal/host/plan14_generation_kernel_linux_test.go b/internal/host/plan14_generation_kernel_linux_test.go
index 687efe2d0..b4f0e192b 100644
--- a/internal/host/plan14_generation_kernel_linux_test.go
+++ b/internal/host/plan14_generation_kernel_linux_test.go
@@ -279,7 +279,7 @@ func TestPlan14GenerationServiceReferences(t *testing.T) {
case "target mount":
generationWrite(t, f.task+"/mountinfo", fmt.Sprintf("1 0 %d:%d / / rw - ext4 disk rw\n2 1 %d:%d %s /bound ro - ext4 disk ro\n", generationMajor(b.Executable.Device), generationMinor(b.Executable.Device), generationMajor(b.Executable.Device), generationMinor(b.Executable.Device), b.Tree))
}
- k := generationKernel{proc: f.proc, service: func(string, []byte) (bool, error) { return mode != "unknown", nil }}
+ k := generationKernel{proc: f.proc, service: func(string, []byte) (bool, error) { return mode != "unknown", nil }, recheck: func(string, bool) error { return nil }}
_, err := k.absent(t.Context(), f.reservation)
if (err == nil) != (mode == "service") {
t.Fatalf("service reference boundary: %v", err)
@@ -291,7 +291,7 @@ func TestPlan14GenerationServiceReferences(t *testing.T) {
func TestPlan14GenerationCensusRetriesWholeObservation(t *testing.T) {
for _, refusal := range []error{os.ErrNotExist, os.ErrPermission} {
calls := 0
- boot, digest, err := generationCensusRetry(t.Context(), func() (string, string, error) {
+ boot, digest, err := generationCensusRetry(t.Context(), func(context.Context) (string, string, error) {
calls++
if calls == 1 {
return "", "", fmt.Errorf("task exited: %w", refusal)
@@ -307,7 +307,7 @@ func TestPlan14GenerationCensusRetriesWholeObservation(t *testing.T) {
}
}
calls := 0
- _, _, err := generationCensusRetry(t.Context(), func() (string, string, error) { calls++; return "", "", os.ErrNotExist })
+ _, _, err := generationCensusRetry(t.Context(), func(context.Context) (string, string, error) { calls++; return "", "", os.ErrNotExist })
if err == nil || calls != 3 {
t.Fatal("unbounded or incomplete scan accepted", calls, err)
}
diff --git a/internal/host/recover_settlement_test.go b/internal/host/recover_settlement_test.go
new file mode 100644
index 000000000..632be5423
--- /dev/null
+++ b/internal/host/recover_settlement_test.go
@@ -0,0 +1,797 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "github.com/carlosframework/platform/internal/rootstate"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootparticipation"
+)
+
+// carlos-system-recover runs the root generation census to settle the
+// participation transaction an unfinished apply left open. The edge's guard
+// drop-in Requires= the recover unit, so a recover that fails keeps the
+// edge down. With no journal open the files are whole, so a census refusal
+// defers the settlement instead: recover exits 0 with the transaction still
+// open. With a journal open it fails as before.
+
+// harnessRefusal is the kernel census's refusal on the harness box, as
+// productionGenerationKernelCensus returns it and the launch verifier
+// passes it up unchanged.
+func harnessRefusal() error {
+ return &generationCensusRefusal{err: fmt.Errorf("/proc/1535/task/1535: service: %w", os.ErrPermission)}
+}
+
+// refuseCensus makes root's launch census refuse from its pass-th call on
+// (1 is every call), and stubs the pending controller recovery so the
+// refusal is the census's alone.
+func refuseCensusFrom(t *testing.T, f *applyFixture, pass int) *int {
+ t.Helper()
+ calls := 0
+ f.a.participation.VerifyLaunches = func(context.Context, uint64) error {
+ calls++
+ if calls < pass {
+ return nil
+ }
+ return harnessRefusal()
+ }
+ old := recoverPendingControllers
+ t.Cleanup(func() { recoverPendingControllers = old })
+ recoverPendingControllers = func(context.Context, string, string) error { return nil }
+ return &calls
+}
+
+func refuseCensus(t *testing.T, f *applyFixture) *int { return refuseCensusFrom(t, f, 1) }
+
+func logLines(f *applyFixture) *[]string {
+ var lines []string
+ logf := f.a.ops.logf
+ f.a.ops.logf = func(format string, args ...any) {
+ lines = append(lines, fmt.Sprintf(format, args...))
+ logf(format, args...)
+ }
+ return &lines
+}
+
+func TestRecoverDefersACensusRefusalWithNoJournalOpen(t *testing.T) {
+ for _, marker := range []bool{true, false} {
+ t.Run(map[bool]string{true: "marker", false: "no-marker"}[marker], func(t *testing.T) {
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ begun := filepath.Join(a.floor.Dir, "update-begun.json")
+ if !marker {
+ if err := os.Remove(begun); err != nil {
+ t.Fatal(err)
+ }
+ }
+ calls := refuseCensus(t, f)
+ lines := logLines(f)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover with no journal open = %v, want a deferral that exits 0", err)
+ }
+ // Once before root's pending-controller recovery and once after,
+ // and not again at the end of the run.
+ if *calls != 2 {
+ t.Fatalf("recover ran the census %d times, want 2", *calls)
+ }
+ // The transaction is still open: no completion, the reservation
+ // kept, root's update record kept, so acquisitions stay refused.
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) {
+ t.Fatalf("a refused census was settled: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); err != nil {
+ t.Fatalf("reservation discarded: %v", err)
+ }
+ if _, err := os.Stat(begun); marker && err != nil {
+ t.Fatalf("marker discarded: %v", err)
+ }
+ if got, err := a.participation.UpdateID(); err != nil || got != id {
+ t.Fatalf("update record = %q, %v; want %s still open", got, err, id)
+ }
+ if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil {
+ t.Fatal("a claim acquisition started while the transaction was open")
+ }
+ if len(f.calls) != 0 {
+ t.Fatal("recovery executed a binary or touched a unit", f.calls)
+ }
+ if !strings.Contains(strings.Join(*lines, "\n"), "service: permission denied") {
+ t.Fatalf("the deferral did not log the census's refusal: %q", *lines)
+ }
+ })
+ }
+}
+
+func TestRecoverFailsACensusRefusalWithAJournalOpen(t *testing.T) {
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ if err := a.journal(systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil {
+ t.Fatal(err)
+ }
+ refuseCensus(t, f)
+ err := a.Run(t.Context(), true)
+ if err == nil || errors.Is(err, errSettlementDeferred) || !strings.Contains(err.Error(), "service: permission denied") {
+ t.Fatalf("recover with a journal open = %v, want the census's refusal", err)
+ }
+ if _, err := os.Stat(a.file("journal.json")); err != nil {
+ t.Fatalf("journal discarded: %v", err)
+ }
+}
+
+func TestRecoverDefersACensusRefusalWithoutAnArmedFloor(t *testing.T) {
+ // checkUnarmedParticipationRecovery: an apply began root's update
+ // record and stopped before writing a journal.
+ f := newApplyFixture(t, "stable", "old")
+ if f.a.guarded() {
+ t.Fatal("fixture has an armed floor")
+ }
+ if err := f.a.beginExecutableChange(t.Context(), "new", f.now); err != nil {
+ t.Fatal(err)
+ }
+ calls := refuseCensus(t, f)
+ if err := f.a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover with no journal open = %v, want a deferral that exits 0", err)
+ }
+ if *calls != 2 {
+ t.Fatalf("recover ran the census %d times, want 2 (not again to settle the marker)", *calls)
+ }
+ if _, err := f.a.participation.UpdateID(); err != nil {
+ t.Fatalf("recover settled a refused census: %v", err)
+ }
+ // With a journal open it still fails.
+ if err := f.a.journal(systemJournal{Phase: systemPhaseInstalling, Target: "new"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.a.Run(t.Context(), true); err == nil {
+ t.Fatal("recover with a journal open passed a refused census")
+ }
+}
+
+func TestApplyStillFailsACensusRefusalWithNoJournalOpen(t *testing.T) {
+ // Only --recover defers: the apply's own start-up check returns the
+ // refusal as on main.
+ f, id := guardRecoveryFixture(t)
+ if err := f.a.floor.Begin(t.Context(), id, f.a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ refuseCensus(t, f)
+ if err := f.a.Run(t.Context(), false); err == nil || !strings.Contains(err.Error(), "service: permission denied") {
+ t.Fatalf("apply = %v, want the census's refusal", err)
+ }
+}
+
+func TestRuling6NoCensusRecoverIsUnchanged(t *testing.T) {
+ // A box without root participation (no census.json) never runs the
+ // generation census, and a refused settlement fails recover exactly as
+ // on main: the deferral is for census boxes only.
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ if err := a.participation.State.Remove("census.json"); err != nil {
+ t.Fatal(err)
+ }
+ if required, err := a.participationDisk().ParticipationRequired(t.Context()); err != nil || required {
+ t.Fatalf("fixture still requires participation: %t %v", required, err)
+ }
+ a.participation.VerifyLaunches = func(context.Context, uint64) error {
+ t.Fatal("a no-census box ran the generation census")
+ return nil
+ }
+ if err := os.WriteFile(filepath.Join(a.dataDir, hostexecDir, "host.json"), []byte(`{"state":"pending"}`), 0600); err != nil {
+ t.Fatal(err)
+ }
+ err := a.Run(t.Context(), true)
+ if err == nil || errors.Is(err, errSettlementDeferred) {
+ t.Fatalf("no-census recover = %v, want main's refusal", err)
+ }
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) {
+ t.Fatalf("completion written: %v", err)
+ }
+ // No census refusal can reach a no-census box's settlement, and one
+ // that did would not defer either.
+ if err := a.settleWithoutJournal(t.Context(), "recover-system", harnessRefusal()); errors.Is(err, errSettlementDeferred) {
+ t.Fatalf("a no-census box deferred: %v", err)
+ }
+}
+
+// openTransaction checks the guarded transaction is still open after a
+// deferral: the reservation and root's update record kept, so claim
+// acquisitions stay refused. marker says whether root's marker must be
+// there too.
+func openTransaction(t *testing.T, f *applyFixture, id string, marker bool) {
+ t.Helper()
+ a := f.a
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); err != nil {
+ t.Fatalf("reservation discarded: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, "update-begun.json")); marker && err != nil {
+ t.Fatalf("marker discarded: %v", err)
+ }
+ if got, err := a.participation.UpdateID(); err != nil || got != id {
+ t.Fatalf("update record = %q, %v; want %s still open", got, err, id)
+ }
+ if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil {
+ t.Fatal("a claim acquisition started while the transaction was open")
+ }
+}
+
+func TestRecoverDefersACensusRefusalOnlyAtTheEndOfParticipation(t *testing.T) {
+ // The pending-controller census passes; the one inside EndParticipation
+ // (EndUpdateWith -> idle) refuses. A finished transaction.
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ calls := refuseCensusFrom(t, f, 2)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v, want a deferral that exits 0", err)
+ }
+ if *calls != 2 {
+ t.Fatalf("census ran %d times, want 2 (controllers, then the end of participation)", *calls)
+ }
+ openTransaction(t, f, id, true)
+ // The next recover, with the census passing, settles it.
+ a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatal(err)
+ }
+ if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id {
+ t.Fatalf("completion = %q, %v", raw, err)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("update record after settlement: %v", err)
+ }
+}
+
+// unfinishedGuard makes the fixture's transaction one that never changed
+// the box: the signed reservation names another image as its target.
+func unfinishedGuard(t *testing.T, f *applyFixture, id string) {
+ t.Helper()
+ path := filepath.Join(f.a.floor.Dir, "activation.json")
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var tx guardTransaction
+ if err := json.Unmarshal(raw, &tx); err != nil {
+ t.Fatal(err)
+ }
+ var roster hostactors.Roster
+ if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil {
+ t.Fatal(err)
+ }
+ h := roster.Activations[id]
+ h.Hashes = map[string][]string{tx.Actor: {h.Hashes[tx.Actor][0], strings.Repeat("d", 64), h.Hashes[tx.Actor][2]}}
+ roster.Activations[id] = h
+ if err := roster.Validate("bucket"); err != nil {
+ t.Fatal(err)
+ }
+ if tx.Roster, err = release.SignRootEnvelope(release.RootSigner(f.priv), roster); err != nil {
+ t.Fatal(err)
+ }
+ raw, _ = json.Marshal(tx)
+ if err := hostfloor.Durable(path, raw, 0644); err != nil {
+ t.Fatal(err)
+ }
+ if finished, err := f.a.guardFinished(tx); err != nil || finished {
+ t.Fatalf("fixture transaction finished=%t err=%v, want unfinished", finished, err)
+ }
+}
+
+func TestRecoverDefersACensusRefusalOfAnUnfinishedTransaction(t *testing.T) {
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ unfinishedGuard(t, f, id)
+ refuseCensus(t, f)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v, want a deferral that exits 0", err)
+ }
+ // The withdrawal removes nothing until its census passes.
+ openTransaction(t, f, id, true)
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) {
+ t.Fatalf("an unfinished transaction recorded a completion: %v", err)
+ }
+ a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatal(err)
+ }
+ for _, name := range []string{"update-begun.json", "activation.json", "completed.json"} {
+ if _, err := os.Stat(filepath.Join(a.floor.Dir, name)); !os.IsNotExist(err) {
+ t.Fatalf("%s after the withdrawal: %v", name, err)
+ }
+ }
+}
+
+func TestRecoverDefersACensusRefusalOfAnAlreadyCompletedTransaction(t *testing.T) {
+ // Completion recorded and the marker gone, root's update record still
+ // open: only EndParticipation is left, and its census refuses.
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "completed.json"), []byte(id), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")); err != nil {
+ t.Fatal(err)
+ }
+ calls := refuseCensus(t, f)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v, want a deferral that exits 0", err)
+ }
+ if *calls != 1 {
+ t.Fatalf("census ran %d times, want 1", *calls)
+ }
+ openTransaction(t, f, id, false)
+ if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id {
+ t.Fatalf("completion = %q, %v", raw, err)
+ }
+}
+
+func TestRecoverStillFailsSettlementErrorsThatAreNotCensusRefusals(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ setup func(*testing.T, *applyFixture, string)
+ }{
+ {"malformed-ledger-state", func(t *testing.T, f *applyFixture, _ string) {
+ // The verifier refuses the ledger itself, not through the census.
+ f.a.participation.VerifyLaunches = func(context.Context, uint64) error { return rootparticipation.ErrUnavailable }
+ old := recoverPendingControllers
+ t.Cleanup(func() { recoverPendingControllers = old })
+ recoverPendingControllers = func(context.Context, string, string) error { return nil }
+ }},
+ {"live-controller", func(t *testing.T, f *applyFixture, _ string) {
+ if err := os.WriteFile(filepath.Join(f.a.dataDir, hostexecDir, "host.json"), []byte(`{"state":"pending"}`), 0600); err != nil {
+ t.Fatal(err)
+ }
+ old := recoverPendingControllers
+ t.Cleanup(func() { recoverPendingControllers = old })
+ recoverPendingControllers = func(context.Context, string, string) error { return nil }
+ }},
+ {"image-verification", func(t *testing.T, f *applyFixture, _ string) {
+ if err := os.WriteFile(systemBinaryPath(f.a.optDir), []byte("an unsigned image"), 0755); err != nil {
+ t.Fatal(err)
+ }
+ }},
+ {"marker-for-another-transaction", func(t *testing.T, f *applyFixture, _ string) {
+ if err := hostfloor.Durable(filepath.Join(f.a.floor.Dir, "update-begun.json"), []byte(`{"id":"`+strings.Repeat("e", 32)+`"}`), 0644); err != nil {
+ t.Fatal(err)
+ }
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f, id := guardRecoveryFixture(t)
+ if err := f.a.floor.Begin(t.Context(), id, f.a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ tc.setup(t, f, id)
+ err := f.a.Run(t.Context(), true)
+ if err == nil || errors.Is(err, errSettlementDeferred) {
+ t.Fatalf("recover = %v, want the settlement's own failure", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(f.a.floor.Dir, "activation.json")); statErr != nil {
+ t.Fatalf("reservation discarded: %v", statErr)
+ }
+ })
+ }
+}
+
+func TestOnlyCensusRefusal(t *testing.T) {
+ refusal := harnessRefusal()
+ for _, tc := range []struct {
+ err error
+ want bool
+ }{
+ {refusal, true},
+ {fmt.Errorf("end participation: %w", refusal), true},
+ {errors.Join(refusal, nil), true},
+ {errors.Join(refusal, harnessRefusal()), true},
+ {errors.Join(refusal, errors.New("remove update.json: read-only file system")), false},
+ {fmt.Errorf("x: %w", errors.Join(refusal, os.ErrPermission)), false},
+ {os.ErrPermission, false},
+ {rootparticipation.ErrUnavailable, false},
+ {nil, false},
+ } {
+ if got := onlyCensusRefusal(tc.err); got != tc.want {
+ t.Fatalf("onlyCensusRefusal(%v) = %t, want %t", tc.err, got, tc.want)
+ }
+ }
+}
+
+func TestRecoverWithoutAnArmedFloorKeepsALateRefusalNonFatal(t *testing.T) {
+ // The controllers' census passes; settling the update record at the end
+ // of the run (settleUpdateBegun -> EndUpdateWith) refuses. As on main
+ // that is logged, not fatal, and the record stays open.
+ f := newApplyFixture(t, "stable", "old")
+ if err := f.a.beginExecutableChange(t.Context(), "new", f.now); err != nil {
+ t.Fatal(err)
+ }
+ calls := refuseCensusFrom(t, f, 2)
+ if err := f.a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v", err)
+ }
+ if *calls < 2 {
+ t.Fatalf("census ran %d times; the late settlement never ran", *calls)
+ }
+ if _, err := f.a.participation.UpdateID(); err != nil {
+ t.Fatalf("a refused settlement closed the update record: %v", err)
+ }
+}
+
+func TestRecoverHoldsAcquisitionsWhenNoUpdateRecordWasBegun(t *testing.T) {
+ // A prepared transaction: the reservation persisted, root's marker and
+ // update record never written. The census refuses the settlement's
+ // BeginUpdate, which withdraws its own record; recover must still
+ // leave acquisitions refused.
+ for _, finished := range []bool{true, false} {
+ t.Run(map[bool]string{true: "finished", false: "unfinished"}[finished], func(t *testing.T) {
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if !finished {
+ unfinishedGuard(t, f, id)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("fixture has an update record: %v", err)
+ }
+ refuseCensus(t, f)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v, want a deferral that exits 0", err)
+ }
+ openTransaction(t, f, id, false)
+ a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("barrier left after settlement: %v", err)
+ }
+ if err := a.participation.Acquire(t.Context(), func() error { return nil }); err != nil {
+ t.Fatalf("acquisition after settlement = %v", err)
+ }
+ })
+ }
+}
+
+func TestRecoverReplaysASettledTransactionWithoutACensus(t *testing.T) {
+ // Completion recorded, marker and update record gone: settled. A
+ // replay closes nothing, runs no census and blocks nothing.
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatal(err)
+ }
+ if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id {
+ t.Fatalf("fixture not settled: %q %v", raw, err)
+ }
+ calls := refuseCensus(t, f)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("replay = %v", err)
+ }
+ if *calls != 0 {
+ t.Fatalf("a settled replay ran the census %d times", *calls)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("a settled replay left an update record: %v", err)
+ }
+}
+
+// journalOpen writes an apply's journal for a recovery to finish.
+func journalOpen(t *testing.T, f *applyFixture, j systemJournal) {
+ t.Helper()
+ j.StartedAt = f.now
+ if err := f.a.journal(j); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestRecoverDefersARefusalAfterTheJournalIsRemoved(t *testing.T) {
+ // Recovery's own census passes; the settlement after the journal is
+ // durably removed refuses. Unarmed forward (from gated) and backward
+ // (from installed); guarded through the journal close a guarded
+ // fixture can run, the legacy backing-up journal (the guarded forward
+ // and backward restores run a guarded image, which it cannot; the
+ // guarded backward close is TestGuardedRollbackCloseDefers...).
+ for _, tc := range []struct {
+ name string
+ guarded bool
+ forward bool
+ }{{"guarded-backing-up", true, true}, {"unarmed-forward", false, true}, {"unarmed-backward", false, false}} {
+ t.Run(tc.name, func(t *testing.T) {
+ var f *applyFixture
+ id := ""
+ if tc.guarded {
+ f, id = guardRecoveryFixture(t)
+ if err := f.a.floor.Begin(t.Context(), id, f.a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ f = newApplyFixture(t, "stable", "old-build")
+ if err := f.a.beginExecutableChange(t.Context(), "new-build", f.now); err != nil {
+ t.Fatal(err)
+ }
+ id, _ = f.a.participation.UpdateID()
+ }
+ a := f.a
+ // carlos.prev is the same signed build: the guard verifies it
+ // before a recovery with a journal open.
+ body, _ := os.ReadFile(systemBinaryPath(a.optDir))
+ if err := os.WriteFile(systemPrevPath(a.optDir), body, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ prev := systemPrevPath(a.optDir)
+ f.answer["version:"+prev] = "carlos old-build\n"
+ f.answer["plan:"+prev] = f.answer["plan:"+systemBinaryPath(a.optDir)]
+ f.answer["units:"+prev] = f.answer["units:"+systemBinaryPath(a.optDir)]
+ if tc.guarded {
+ journalOpen(t, f, systemJournal{Phase: systemPhaseBackingUp, Target: "new-build", PrevBuild: "old-build"})
+ } else if tc.forward {
+ journalOpen(t, f, systemJournal{Phase: systemPhaseGated, Target: "old-build", PrevBuild: "old-build"})
+ } else {
+ journalOpen(t, f, systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build"})
+ }
+ old := recoverPendingControllers
+ t.Cleanup(func() { recoverPendingControllers = old })
+ recoverPendingControllers = func(context.Context, string, string) error { return nil }
+ removed := false
+ a.participation.VerifyLaunches = func(context.Context, uint64) error {
+ if j, _ := readSystemJournal(a.optDir); j == nil {
+ removed = true
+ return harnessRefusal()
+ }
+ return nil
+ }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v, want a deferral that exits 0", err)
+ }
+ if !removed {
+ t.Fatal("the census never ran after the journal was removed")
+ }
+ if j, _ := readSystemJournal(a.optDir); j != nil {
+ t.Fatal("journal kept")
+ }
+ want := systemOutcomeRolledBack
+ if tc.forward {
+ want = systemOutcomeApplied
+ }
+ if r := f.result(); !tc.guarded && (r == nil || r.Outcome != want) {
+ t.Fatalf("result = %+v, want %s", r, want)
+ }
+ if got, err := a.participation.UpdateID(); err != nil || got != id {
+ t.Fatalf("update record = %q, %v; want %s held", got, err, id)
+ }
+ if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil {
+ t.Fatal("a claim acquisition started while the settlement was deferred")
+ }
+ // The next recover, with the census passing, settles it.
+ a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("update record after settlement: %v", err)
+ }
+ })
+ }
+}
+
+func TestRecoverStillFailsANonCensusErrorAfterTheJournalIsRemoved(t *testing.T) {
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ journalOpen(t, f, systemJournal{Phase: systemPhaseGated, Target: "old-build", PrevBuild: "old-build"})
+ a.participation.VerifyLaunches = func(context.Context, uint64) error {
+ if j, _ := readSystemJournal(a.optDir); j == nil {
+ return rootparticipation.ErrUnavailable
+ }
+ return nil
+ }
+ err := a.Run(t.Context(), true)
+ if err == nil || errors.Is(err, errSettlementDeferred) {
+ t.Fatalf("recover = %v, want the settlement's own failure", err)
+ }
+}
+
+func TestGuardedRollbackCloseDefersARefusalAfterTheJournalIsRemoved(t *testing.T) {
+ // The backward close of a guarded transaction (closeGuardRollback, as
+ // recovery's rollback and the apply's rollback call it): the rollback is
+ // recorded, the journal removed, and the settlement's census refuses.
+ // Restoring carlos.prev runs a guarded image, which this fixture
+ // cannot, so the close is driven directly; Run's consumption of the
+ // deferral is the forward case's.
+ f, id := guardRecoveryFixture(t)
+ a := f.a
+ if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
+ t.Fatal(err)
+ }
+ journalOpen(t, f, systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build"})
+ refuseCensus(t, f)
+ err := a.closeGuardRollback(t.Context())
+ if !errors.Is(err, errSettlementDeferred) || !a.deferSettlement(true, err) {
+ t.Fatalf("rollback close = %v, want a deferral", err)
+ }
+ if a.deferSettlement(false, err) {
+ t.Fatal("the apply consumed a deferral")
+ }
+ if j, _ := readSystemJournal(a.optDir); j != nil {
+ t.Fatal("journal kept")
+ }
+ if begun, err := hostfloor.ReadBegun(a.floor.Dir); err != nil || !begun.RolledBack {
+ t.Fatalf("rollback not recorded: %+v %v", begun, err)
+ }
+ openTransaction(t, f, id, true)
+}
+
+func TestRecoverDoesNotDeferWithoutABarrier(t *testing.T) {
+ // A barrier that cannot be written (here an id the ledger refuses)
+ // leaves the census refusal fatal: recover never exits 0 with
+ // acquisitions open.
+ f, _ := guardRecoveryFixture(t)
+ err := f.a.settleWithoutJournal(t.Context(), "bad id", harnessRefusal())
+ if err == nil || errors.Is(err, errSettlementDeferred) || !strings.Contains(err.Error(), "acquisition barrier") {
+ t.Fatalf("settlement without a barrier = %v", err)
+ }
+ if err := f.a.participation.Acquire(t.Context(), func() error { return nil }); err != nil {
+ t.Fatalf("fixture: %v", err)
+ }
+}
+
+// failingBarrierStore is the participation store with one write refused:
+// the barrier's update.json, written after BeginUpdate's own write of it
+// was withdrawn by a census refusal.
+type failingBarrierStore struct {
+ rootstate.Store
+ writes int
+}
+
+func (f *failingBarrierStore) Write(name string, raw []byte) error {
+ if name == "update.json" {
+ f.writes++
+ if f.writes == 2 {
+ return errors.New("barrier write refused for the test")
+ }
+ }
+ return f.Store.Write(name, raw)
+}
+
+// legacyMarker writes the update-begun marker alone: root participation,
+// no armed floor, no update record, no journal (Astra round 3).
+func legacyMarker(t *testing.T, f *applyFixture) {
+ t.Helper()
+ a := f.a
+ if a.guarded() {
+ t.Fatal("fixture has an armed floor")
+ }
+ if err := os.MkdirAll(systemGuardDir(a.optDir), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := writeJSONDurable(updateBegunPath(a.optDir), updateBegun{Target: "new", At: f.now}, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("fixture has an update record: %v", err)
+ }
+ if j, err := readSystemJournal(a.optDir); err != nil || j != nil {
+ t.Fatalf("fixture has a journal: %v %v", j, err)
+ }
+}
+
+func TestRecoverHoldsAcquisitionsForALegacyMarker(t *testing.T) {
+ // checkUnarmedParticipationRecovery sees no update record and no
+ // journal and returns; the run's end settles the marker through
+ // clearUpdateBegun, whose BeginUpdate the census refuses (and which
+ // withdraws the record it wrote). Recover must exit 0 with the marker
+ // kept and acquisitions held refused, and settle both once the census
+ // passes.
+ f := newApplyFixture(t, "stable", "old")
+ a := f.a
+ legacyMarker(t, f)
+ refuseCensus(t, f)
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatalf("recover = %v, want a deferral that exits 0", err)
+ }
+ if _, err := os.Lstat(updateBegunPath(a.optDir)); err != nil {
+ t.Fatalf("the marker went without a settlement: %v", err)
+ }
+ if id, err := a.participation.UpdateID(); err != nil || id != "recover-legacy" {
+ t.Fatalf("barrier = %q, %v; want root's update record under recover-legacy", id, err)
+ }
+ if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil {
+ t.Fatal("an acquisition passed while recover had deferred the legacy marker's settlement")
+ }
+ a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil }
+ if err := a.Run(t.Context(), true); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Lstat(updateBegunPath(a.optDir)); !os.IsNotExist(err) {
+ t.Fatalf("marker left after settlement: %v", err)
+ }
+ if _, err := a.participation.UpdateID(); !os.IsNotExist(err) {
+ t.Fatalf("barrier left after settlement: %v", err)
+ }
+ if err := a.participation.Acquire(t.Context(), func() error { return nil }); err != nil {
+ t.Fatalf("acquisition after settlement = %v", err)
+ }
+}
+
+func TestRecoverFailsALegacyMarkerWhoseBarrierCannotBeWritten(t *testing.T) {
+ // The same start, with the barrier's write refused: recover must fail,
+ // not exit 0 with acquisitions open; the marker stays for the next run.
+ f := newApplyFixture(t, "stable", "old")
+ a := f.a
+ legacyMarker(t, f)
+ refuseCensus(t, f)
+ a.participation.State = &failingBarrierStore{Store: a.participation.State}
+ err := a.Run(t.Context(), true)
+ if err == nil || !errors.Is(err, errAcquisitionBarrier) {
+ t.Fatalf("recover with no barrier = %v, want it fatal", err)
+ }
+ if _, err := os.Lstat(updateBegunPath(a.optDir)); err != nil {
+ t.Fatalf("the marker went without a settlement: %v", err)
+ }
+}
+
+// prerequisiteFailingStore is the participation store whose census stops
+// reading once BeginUpdate has withdrawn the update record it wrote: the
+// barrier's prerequisite, ParticipationRequired, then fails.
+type prerequisiteFailingStore struct {
+ rootstate.Store
+ withdrawn bool
+}
+
+func (p *prerequisiteFailingStore) Remove(name string) error {
+ if name == "update.json" {
+ p.withdrawn = true
+ }
+ return p.Store.Remove(name)
+}
+
+func (p *prerequisiteFailingStore) Read(name string) ([]byte, error) {
+ if name == "census.json" && p.withdrawn {
+ return nil, errors.New("census read refused for the test")
+ }
+ return p.Store.Read(name)
+}
+
+func TestRecoverFailsALegacyMarkerWhoseBarrierPrerequisiteFails(t *testing.T) {
+ // Astra round 4: after the census refused BeginUpdate and the record it
+ // wrote was withdrawn, the check whether a barrier is needed fails. That
+ // is a barrier failure: recover must not log it and exit 0.
+ f := newApplyFixture(t, "stable", "old")
+ a := f.a
+ legacyMarker(t, f)
+ refuseCensus(t, f)
+ a.participation.State = &prerequisiteFailingStore{Store: a.participation.State}
+ err := a.Run(t.Context(), true)
+ if err == nil || !errors.Is(err, errAcquisitionBarrier) {
+ t.Fatalf("recover with the barrier prerequisite failing = %v, want it fatal", err)
+ }
+ if _, err := os.Lstat(updateBegunPath(a.optDir)); err != nil {
+ t.Fatalf("the marker went without a settlement: %v", err)
+ }
+}
diff --git a/internal/host/rootgeneration_census_linux.go b/internal/host/rootgeneration_census_linux.go
index 8b49b6f0f..200ae15bd 100644
--- a/internal/host/rootgeneration_census_linux.go
+++ b/internal/host/rootgeneration_census_linux.go
@@ -5,9 +5,11 @@ package host
import (
"context"
"errors"
+ "fmt"
"io/fs"
"os"
"path/filepath"
+ "time"
"github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/rootstate"
@@ -70,7 +72,16 @@ func checkedGenerationTree(ctx context.Context, tree string) error {
return nil
})
}
-func productionGenerationKernelCensus(ctx context.Context) (string, string, error) {
+
+// productionGenerationKernelCensus is root's kernel generation census. Its
+// every refusal is a generationCensusRefusal, so a no-journal recovery can
+// tell it from the other ways a settlement fails.
+func productionGenerationKernelCensus(ctx context.Context) (boot, digest string, err error) {
+ defer func() {
+ if err != nil {
+ boot, digest, err = "", "", &generationCensusRefusal{err: err}
+ }
+ }()
if os.Geteuid() != 0 {
return "", "", os.ErrPermission
}
@@ -86,18 +97,111 @@ func productionGenerationKernelCensus(ctx context.Context) (string, string, erro
return "", "", err
}
records, _ := ctx.Value(generationCensusReservationsKey{}).([]release.GenerationReservation)
- return generationCensusRetry(ctx, func() (string, string, error) {
- return (generationKernel{proc: "/proc", cgroups: "/sys/fs/cgroup", contained: records, service: generationServiceReader(ctx)}).census(ctx, tree)
+ return generationCensusRetry(ctx, func(ctx context.Context) (string, string, error) {
+ service, recheck := generationServiceReader(ctx)
+ return (generationKernel{proc: "/proc", cgroups: "/sys/fs/cgroup", contained: records, service: service, recheck: recheck}).census(ctx, tree)
})
}
-func generationCensusRetry(ctx context.Context, inspect func() (string, string, error)) (string, string, error) {
- for attempt := 0; ; attempt++ {
+// generationCensusWindow bounds a whole census, its passes and the waits
+// between them: a task that exited or was replaced mid-pass, or one the
+// service lookup refused (a unit still activating, the guard's prelude, a
+// login's or sshd's short-lived child), is waited out inside it. A task
+// still refused, or a pass still running, when it ends refuses the census.
+const generationCensusWindow = 20 * time.Second
+
+// errGenerationCensusLate is a census that did not finish inside its
+// window: the last pass ran past it, successful or not, or no pass could
+// start before it ended.
+var errGenerationCensusLate = fmt.Errorf("generation census ran past its window: %w", os.ErrPermission)
+
+// generationTaskRefusal is a census refusal of one task, named by its /proc
+// path. Its text is the path and the cause, as before it had a type.
+type generationTaskRefusal struct {
+ task string
+ err error
+}
+
+func (r *generationTaskRefusal) Error() string { return r.task + ": " + r.err.Error() }
+
+func (r *generationTaskRefusal) Unwrap() error { return r.err }
+
+// errGenerationTaskReplaced is a task whose start time moved during its
+// inspection: the PID now names another task. It is still a refusal
+// (os.ErrPermission), but one a fresh pass can clear.
+var errGenerationTaskReplaced = fmt.Errorf("task replaced during the census: %w", os.ErrPermission)
+
+// transient is true for the task-level refusals a later pass can clear:
+// the task exited or was replaced, or the service lookup refused it. A
+// task that references a retired generation, or a namespace or mount
+// refusal, is not: those refuse at once.
+func (r *generationTaskRefusal) transient() bool {
+ var service *generationServiceRefusal
+ return errors.Is(r.err, os.ErrNotExist) || errors.Is(r.err, unix.ESRCH) || errors.Is(r.err, errGenerationTaskReplaced) || errors.As(r.err, &service)
+}
+
+func generationCensusRetry(ctx context.Context, inspect func(context.Context) (string, string, error)) (string, string, error) {
+ return generationCensusRetryFor(ctx, inspect, generationCensusWindow, time.Now, generationSleep)
+}
+
+func generationSleep(ctx context.Context, d time.Duration) error {
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-t.C:
+ return nil
+ }
+}
+
+// generationCensusRetryFor runs whole census passes under one deadline,
+// window from its start. Each pass gets a context that ends at the
+// deadline, which its blocking reads (systemctl show, the image hash) obey,
+// and a pass that ends past the deadline refuses even if it succeeded, so
+// nothing it saw is accepted late. A transient task refusal is retried with
+// backoff (250 ms, doubling to 4 s) while the wait ends before the
+// deadline; then the last refusal is returned as it is. Any other
+// missing-file or vanished-process error is retried at once, up to three
+// passes, as before. Everything else returns at once. now is the clock the
+// deadline is read on.
+func generationCensusRetryFor(ctx context.Context, inspect func(context.Context) (string, string, error), window time.Duration, now func() time.Time, sleep func(context.Context, time.Duration) error) (string, string, error) {
+ deadline := now().Add(window)
+ bounded, cancel := context.WithTimeout(ctx, window)
+ defer cancel()
+ late := func(last error) error {
+ if last == nil {
+ return errGenerationCensusLate
+ }
+ return fmt.Errorf("%w (last pass: %v)", errGenerationCensusLate, last)
+ }
+ delay, quick := 250*time.Millisecond, 0
+ var last error
+ for {
if err := ctx.Err(); err != nil {
return "", "", err
}
- boot, digest, err := inspect()
- if attempt == 2 || (!errors.Is(err, os.ErrNotExist) && !errors.Is(err, unix.ESRCH)) {
+ if !now().Before(deadline) || bounded.Err() != nil {
+ return "", "", late(last)
+ }
+ boot, digest, err := inspect(bounded)
+ if ctx.Err() == nil && (!now().Before(deadline) || bounded.Err() != nil) {
+ return "", "", late(err)
+ }
+ if err == nil {
+ return boot, digest, nil
+ }
+ last = err
+ var task *generationTaskRefusal
+ switch {
+ case errors.As(err, &task) && task.transient():
+ if !now().Add(delay).Before(deadline) || sleep(bounded, delay) != nil {
+ return boot, digest, err
+ }
+ delay = min(2*delay, 4*time.Second)
+ case quick < 2 && (errors.Is(err, os.ErrNotExist) || errors.Is(err, unix.ESRCH)):
+ quick++
+ default:
return boot, digest, err
}
}
diff --git a/internal/host/rootgeneration_census_recover_linux_test.go b/internal/host/rootgeneration_census_recover_linux_test.go
new file mode 100644
index 000000000..8a5f0c2c3
--- /dev/null
+++ b/internal/host/rootgeneration_census_recover_linux_test.go
@@ -0,0 +1,792 @@
+//go:build linux
+
+package host
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+// The generation census that carlos-system-recover runs, on the two
+// refusals the harness box hit: a non-root service still running the
+// /opt/carlos/carlos an apply renamed over ("(deleted)"), and short-lived
+// tasks the service lookup refused with a bare "permission denied". These
+// drive the production service lookup (generationServiceEnv.reader) over a
+// fake /proc, replacing only systemctl, the unit files' ownership check,
+// the signed-binary proof and the owner uid a non-root test cannot have.
+
+type deletedExeFixture struct {
+ generationProcFixture
+ opt, image string
+ env generationServiceEnv
+ current string // digest the env proves for <opt>/carlos
+ prev string // and for <opt>/carlos.prev; "" is no carlos.prev
+ execStart string
+ mainPID string
+ unit string
+ tt *testing.T
+ // deletedName is the exe link's base name; "" is "carlos (deleted)".
+ deletedName string
+ // imageFile is the open descriptor of the task's current image.
+ imageFile *os.File
+}
+
+func (f *deletedExeFixture) t() *testing.T { return f.tt }
+
+const deletedExeBody = "carlos router build, now renamed over"
+
+func deletedExe(t *testing.T) *deletedExeFixture {
+ t.Helper()
+ g := generationProc(t)
+ f := &deletedExeFixture{generationProcFixture: g, opt: filepath.Join(t.TempDir(), "opt", "carlos"), mainPID: "41", unit: "carlos-relay.service"}
+ if err := os.MkdirAll(f.opt, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ f.execStart = "{ path=" + f.opt + "/carlos ; argv[]=" + f.opt + "/carlos relay ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=41 ; code=(null) ; status=0/0 }"
+ generationWrite(t, f.task+"/status", "Tgid:\t41\nUid:\t1001\t1001\t1001\t1001\nKthread:\t0\n")
+ group := "/system.slice/carlos-relay.service"
+ generationWrite(t, f.task+"/cgroup", "0::"+group+"\n")
+ generationWrite(t, f.proc+"/41/cgroup", "0::"+group+"\n")
+ // The image the task runs: open, then unlinked, as the apply's rename
+ // leaves the edge's. The exe link reads "<opt>/carlos (deleted)" and
+ // opens that image, as /proc/<pid>/exe does.
+ f.image = filepath.Join(t.TempDir(), "image")
+ f.writeImage(t, deletedExeBody, 0o755, true)
+ sum := sha256.Sum256([]byte(deletedExeBody))
+ f.current = "not-this-build"
+ f.prev = hex.EncodeToString(sum[:])
+ f.env = generationServiceEnv{proc: f.proc, opt: f.opt, owner: uint32(os.Getuid()),
+ show: func(_ context.Context, unit string) (map[string]string, error) {
+ if unit != f.unit {
+ return nil, fmt.Errorf("systemctl show %s: no such unit", unit)
+ }
+ return map[string]string{"ControlGroup": group, "MainPID": f.mainPID, "ExecStart": f.execStart, "FragmentPath": "/etc/systemd/system/" + unit}, nil
+ },
+ trust: func(string) error { return nil },
+ binary: func(_ context.Context, opt, path string) (rootFloorBinary, error) {
+ switch path {
+ case filepath.Join(opt, "carlos"):
+ return rootFloorBinary{Digest: f.current}, nil
+ case filepath.Join(opt, "carlos.prev"):
+ if f.prev == "" {
+ return rootFloorBinary{}, os.ErrNotExist
+ }
+ return rootFloorBinary{Digest: f.prev}, nil
+ }
+ return rootFloorBinary{}, os.ErrPermission
+ },
+ }
+ return f
+}
+
+// writeImage replaces the task's image with body at mode, unlinked or not.
+func (f *deletedExeFixture) writeImage(t *testing.T, body string, mode os.FileMode, unlink bool) {
+ t.Helper()
+ path := f.image + fmt.Sprint(time.Now().UnixNano())
+ if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(path, mode); err != nil {
+ t.Fatal(err)
+ }
+ img, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { img.Close() })
+ f.imageFile = img
+ if unlink {
+ if err := os.Remove(path); err != nil {
+ t.Fatal(err)
+ }
+ }
+ name := f.deletedName
+ if name == "" {
+ name = "carlos (deleted)"
+ }
+ deleted := filepath.Join(f.opt, name)
+ os.Remove(deleted)
+ generationLink(t, fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), img.Fd()), deleted)
+ os.Remove(f.task + "/exe")
+ generationLink(t, deleted, f.task+"/exe")
+}
+
+// kernelCensus is one census pass through the production lookup, as
+// productionGenerationKernelCensus wires it: classify and recheck.
+func (f *deletedExeFixture) kernelCensus(ctx context.Context) error {
+ service, recheck := f.env.reader(ctx)
+ _, err := (generationKernel{proc: f.proc, service: service, recheck: recheck}).absent(ctx, f.reservation)
+ return err
+}
+
+func (f *deletedExeFixture) census(t *testing.T) error {
+ t.Helper()
+ return f.kernelCensus(t.Context())
+}
+
+func TestRecoverCensusAcceptsADeletedExeOfTheCurrentOrPreviousBuild(t *testing.T) {
+ for _, which := range []string{"previous", "current", "current-no-prev"} {
+ t.Run(which, func(t *testing.T) {
+ f := deletedExe(t)
+ switch which {
+ case "current":
+ f.current, f.prev = f.prev, "other-build"
+ case "current-no-prev":
+ f.current, f.prev = f.prev, ""
+ }
+ if err := f.census(t); err != nil {
+ t.Fatalf("census refused a deleted exe of the %s signed build: %v", which, err)
+ }
+ })
+ }
+}
+
+func TestRecoverCensusAcceptsTheEdgesDeletedExeInSupervise(t *testing.T) {
+ // The harness's b2 task: carlos-edge.service's main process, in its
+ // supervise/ group, still running the pre-apply image, with and
+ // without the guard drop-in (whose ExecStart is the guard and whose
+ // selected program is <opt>/carlos).
+ for _, guarded := range []bool{false, true} {
+ t.Run(map[bool]string{false: "unguarded", true: "guarded"}[guarded], func(t *testing.T) {
+ f := deletedExe(t)
+ f.unit = "carlos-edge.service"
+ if guarded {
+ f.execStart = "{ path=/opt/carlos/system/guard/carlos ; argv[]=/opt/carlos/system/guard/carlos system guard-exec edge ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=41 ; code=(null) ; status=0/0 }"
+ binary := f.env.binary
+ f.env.binary = func(ctx context.Context, opt, path string) (rootFloorBinary, error) {
+ if path == "/opt/carlos/system/guard/carlos" {
+ return rootFloorBinary{Digest: "guard"}, nil
+ }
+ b, err := binary(ctx, opt, path)
+ b.File.Inode = 7 // the live <opt>/carlos, not the deleted image
+ return b, err
+ }
+ }
+ base := "/system.slice/carlos-edge.service"
+ generationWrite(t, f.task+"/cgroup", "0::"+base+"/supervise\n")
+ generationWrite(t, f.proc+"/41/cgroup", "0::"+base+"/supervise\n")
+ show := f.env.show
+ f.env.show = func(ctx context.Context, unit string) (map[string]string, error) {
+ props, err := show(ctx, unit)
+ if err == nil {
+ props["ControlGroup"] = base
+ }
+ return props, err
+ }
+ if err := f.census(t); err != nil {
+ t.Fatalf("census refused the edge's deleted signed image: %v", err)
+ }
+ // The same edge running some other deleted image still refuses.
+ f.writeImage(t, "an image no identity signs", 0o755, true)
+ if err := f.census(t); err == nil {
+ t.Fatal("census accepted an unsigned deleted image in the edge")
+ }
+ })
+ }
+}
+
+func TestRecoverCensusRefusesOtherDeletedExes(t *testing.T) {
+ for _, attack := range []string{"unsigned", "no-proof", "linked", "group-writable", "not-router-binary", "other-program", "deleted-helper", "other-path"} {
+ t.Run(attack, func(t *testing.T) {
+ f := deletedExe(t)
+ switch attack {
+ case "unsigned":
+ f.writeImage(t, "some other build", 0o755, true)
+ case "no-proof":
+ f.env.binary = func(context.Context, string, string) (rootFloorBinary, error) {
+ return rootFloorBinary{}, errRootFloorHistory
+ }
+ case "linked":
+ f.writeImage(t, deletedExeBody, 0o755, false)
+ case "group-writable":
+ f.writeImage(t, deletedExeBody, 0o775, true)
+ case "not-router-binary":
+ // The unit runs another program: a deleted <opt>/carlos is
+ // not what it selects.
+ f.execStart = strings.ReplaceAll(f.execStart, f.opt+"/carlos", f.opt+"/helper")
+ case "other-program":
+ f.execStart = strings.ReplaceAll(f.execStart, f.opt+"/carlos", "/usr/bin/carlos")
+ case "other-path":
+ // The unit selects <opt>/carlos, but the task's image was
+ // somewhere else when it was deleted.
+ f.deletedName = "carlos-copy (deleted)"
+ f.writeImage(t, deletedExeBody, 0o755, true)
+ case "deleted-helper":
+ // The unit's own program, deleted, with a signed router
+ // build's bytes: only <opt>/carlos is ever accepted.
+ f.execStart = strings.ReplaceAll(f.execStart, f.opt+"/carlos", f.opt+"/helper")
+ f.deletedName = "helper (deleted)"
+ f.writeImage(t, deletedExeBody, 0o755, true)
+ }
+ err := f.census(t)
+ if err == nil {
+ t.Fatalf("%s: census accepted the deleted exe", attack)
+ }
+ if !errors.Is(err, os.ErrPermission) && attack != "no-proof" {
+ t.Fatalf("%s: refusal %v is not a permission refusal", attack, err)
+ }
+ if !strings.Contains(err.Error(), "deleted exe check refused") {
+ t.Fatalf("%s: refusal %q does not name the deleted exe check", attack, err)
+ }
+ })
+ }
+}
+
+func TestRecoverCensusRefusesADeletedExeNotOwnedByRoot(t *testing.T) {
+ if os.Getuid() == 0 {
+ t.Skip("the fixture's image is root-owned when run as root")
+ }
+ f := deletedExe(t)
+ f.env.owner = 0 // production: the image must be root's
+ if err := f.census(t); err == nil || !strings.Contains(err.Error(), "not an unlinked root-owned file") {
+ t.Fatalf("census = %v, want a refusal of a non-root-owned image", err)
+ }
+}
+
+func TestRecoverCensusStillChecksReferencesOfAnAcceptedDeletedExe(t *testing.T) {
+ for _, ref := range []string{"fd", "mapped", "cwd"} {
+ t.Run(ref, func(t *testing.T) {
+ f := deletedExe(t)
+ b := f.reservation.Binding
+ switch ref {
+ case "fd":
+ generationLink(t, b.Path, f.task+"/fd/3")
+ case "mapped":
+ generationWrite(t, f.task+"/maps", fmt.Sprintf("1000-2000 r-xp 0000 %x:%x %d %s\n", generationMajor(b.Executable.Device), generationMinor(b.Executable.Device), b.Executable.Inode, b.Path))
+ case "cwd":
+ generationRelink(t, b.Tree, f.task+"/cwd")
+ }
+ if err := f.census(t); err == nil {
+ t.Fatalf("a deleted-exe task referencing a retired generation (%s) was accepted", ref)
+ }
+ })
+ }
+}
+
+func TestRecoverCensusRefusalsNameCheckUnitCgroupAndExe(t *testing.T) {
+ for _, tc := range []struct {
+ name, check, unit, group string
+ setup func(*deletedExeFixture)
+ }{
+ {"activating", "main pid", "carlos-relay.service", "/system.slice/carlos-relay.service", func(f *deletedExeFixture) { f.mainPID = "0" }},
+ {"session", "unit", "session-3.scope", "/user.slice/user-1000.slice/session-3.scope", func(f *deletedExeFixture) {
+ generationWrite(f.t(), f.task+"/cgroup", "0::/user.slice/user-1000.slice/session-3.scope\n")
+ }},
+ {"live-exe", "exe", "carlos-relay.service", "/system.slice/carlos-relay.service", func(f *deletedExeFixture) {
+ // A live (not deleted) exe the lookup cannot walk to as a
+ // root-owned file under the task's root.
+ generationRelink(f.t(), f.unrelated, f.task+"/exe")
+ generationRelink(f.t(), "/", f.task+"/root")
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := deletedExe(t)
+ f.tt = t
+ tc.setup(f)
+ exe, _ := os.Readlink(f.task + "/exe")
+ err := f.census(t)
+ if err == nil {
+ t.Fatal("census accepted")
+ }
+ for _, want := range []string{"/41/task/41: service: " + tc.check, "service: " + tc.check + " check refused", "unit " + tc.unit, "cgroup " + tc.group, "exe " + exe} {
+ if !strings.Contains(err.Error(), want) {
+ t.Fatalf("refusal %q does not name %q", err, want)
+ }
+ }
+ })
+ }
+}
+
+func TestRecoverCensusRetriesATransientRefusalWithinTheWindow(t *testing.T) {
+ clock := time.Unix(0, 0)
+ var slept []time.Duration
+ now := func() time.Time { return clock }
+ sleep := func(_ context.Context, d time.Duration) error {
+ if clock.Sub(time.Unix(0, 0)) > time.Minute {
+ return errors.New("unbounded retry") // ends a run with no window
+ }
+ slept = append(slept, d)
+ clock = clock.Add(d)
+ return nil
+ }
+
+ // A unit still activating, through the real census pass: it clears on
+ // the fourth pass.
+ f := deletedExe(t)
+ passes := 0
+ inspect := func(ctx context.Context) (string, string, error) {
+ passes++
+ f.mainPID = "0"
+ if passes == 4 {
+ f.mainPID = "41"
+ }
+ err := f.kernelCensus(ctx)
+ return "boot", "digest", err
+ }
+ if _, _, err := generationCensusRetryFor(t.Context(), inspect, 20*time.Second, now, sleep); err != nil {
+ t.Fatalf("a transient refusal did not clear within the window: %v", err)
+ }
+ if passes != 4 || fmt.Sprint(slept) != "[250ms 500ms 1s]" {
+ t.Fatalf("passes=%d slept=%v, want 4 passes after 250ms, 500ms, 1s", passes, slept)
+ }
+
+ // The same refusal, persistent: refused once the next wait would end
+ // past the window, with the last pass's refusal.
+ clock, slept, passes = time.Unix(0, 0), nil, 0
+ persistent := func(ctx context.Context) (string, string, error) {
+ passes++
+ f.mainPID = "0"
+ err := f.kernelCensus(ctx)
+ return "", "", err
+ }
+ _, _, err := generationCensusRetryFor(t.Context(), persistent, 20*time.Second, now, sleep)
+ if err == nil || !strings.Contains(err.Error(), "main pid check refused") {
+ t.Fatalf("persistent refusal = %v", err)
+ }
+ var total time.Duration
+ for _, d := range slept {
+ total += d
+ }
+ if total > 20*time.Second || total < 15*time.Second || passes != len(slept)+1 {
+ t.Fatalf("persistent refusal: %d passes, slept %v (%s), want about 20 s then refuse", passes, slept, total)
+ }
+}
+
+func TestRecoverCensusRetriesOnlyTaskLevelTransients(t *testing.T) {
+ sleep := func(context.Context, time.Duration) error { return nil }
+ for _, tc := range []struct {
+ name string
+ err error
+ retry bool
+ }{
+ {"service", &generationTaskRefusal{task: "/proc/9/task/9", err: fmt.Errorf("service: %w", &generationServiceRefusal{check: "unit", err: os.ErrPermission})}, true},
+ {"exited", &generationTaskRefusal{task: "/proc/9/task/9", err: os.ErrNotExist}, true},
+ {"replaced", &generationTaskRefusal{task: "/proc/9/task/9", err: errGenerationTaskReplaced}, true},
+ {"reference", &generationTaskRefusal{task: "/proc/9/task/9", err: fmt.Errorf("fd: %w", os.ErrPermission)}, false},
+ {"mounts", &generationTaskRefusal{task: "/proc/9/task/9", err: fmt.Errorf("mounts: %w", os.ErrPermission)}, false},
+ {"structural", os.ErrPermission, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ passes := 0
+ _, _, err := generationCensusRetryFor(t.Context(), func(context.Context) (string, string, error) {
+ passes++
+ if passes == 2 {
+ return "boot", "digest", nil
+ }
+ return "", "", tc.err
+ }, 20*time.Second, time.Now, sleep)
+ if tc.retry && (err != nil || passes != 2) {
+ t.Fatalf("transient %s: passes=%d err=%v", tc.name, passes, err)
+ }
+ if !tc.retry && (err == nil || passes != 1) {
+ t.Fatalf("%s retried: passes=%d err=%v", tc.name, passes, err)
+ }
+ })
+ }
+ // A cancelled wait refuses with the refusal it was waiting out.
+ ctx, cancel := context.WithCancel(t.Context())
+ refusal := &generationTaskRefusal{task: "/proc/9/task/9", err: os.ErrNotExist}
+ _, _, err := generationCensusRetryFor(ctx, func(context.Context) (string, string, error) { return "", "", refusal }, 20*time.Second, time.Now, func(context.Context, time.Duration) error { cancel(); return context.Canceled })
+ if !errors.Is(err, refusal) {
+ t.Fatalf("cancelled wait = %v", err)
+ }
+}
+
+func TestProductionGenerationCensusRefusalsAreTyped(t *testing.T) {
+ // Every refusal of the production census is a generationCensusRefusal,
+ // which is what lets a no-journal recovery defer it. As a non-root test
+ // the census refuses at its first check.
+ if os.Geteuid() == 0 {
+ t.Skip("the census does not refuse at once as root")
+ }
+ _, _, err := productionGenerationKernelCensus(t.Context())
+ var refusal *generationCensusRefusal
+ if !errors.As(err, &refusal) || !onlyCensusRefusal(err) || !errors.Is(err, os.ErrPermission) {
+ t.Fatalf("census refusal = %#v, want a typed census refusal", err)
+ }
+}
+
+// execRace runs one census pass through the production lookup with a hook
+// around each of its rechecks: before(final) runs before the recheck and
+// after(final) after it. The hooks stand in for the task exec'ing (the
+// same PID, the same start time) while the census inspects it.
+func (f *deletedExeFixture) execRace(t *testing.T, after func(final bool, call int), before func(final bool, call int)) error {
+ t.Helper()
+ classify, recheck := f.env.reader(t.Context())
+ calls := 0
+ hooked := func(path string, final bool) error {
+ calls++
+ if before != nil {
+ before(final, calls)
+ }
+ err := recheck(path, final)
+ if after != nil {
+ after(final, calls)
+ }
+ return err
+ }
+ _, err := (generationKernel{proc: f.proc, service: classify, recheck: hooked}).absent(t.Context(), f.reservation)
+ return err
+}
+
+func TestRecoverCensusRefusesATaskThatExecsAfterItsImageWasProven(t *testing.T) {
+ helper := func(f *deletedExeFixture) { generationRelink(t, f.unrelated, f.task+"/exe") }
+ for _, tc := range []struct {
+ name string
+ race func(f *deletedExeFixture) (after, before func(bool, int))
+ }{
+ // The task execs an unsigned helper between the hash and the first
+ // use of the trust.
+ {"after-classification", func(f *deletedExeFixture) (func(bool, int), func(bool, int)) {
+ return nil, func(_ bool, call int) {
+ if call == 1 {
+ helper(f)
+ }
+ }
+ }},
+ // It execs another image that reads the same, "<opt>/carlos
+ // (deleted)" with signed bytes, but is not the inode hashed.
+ {"same-name-other-inode", func(f *deletedExeFixture) (func(bool, int), func(bool, int)) {
+ return nil, func(_ bool, call int) {
+ if call == 1 {
+ f.writeImage(t, deletedExeBody, 0o755, true)
+ }
+ }
+ }},
+ // It execs the helper after the first check and is back on the
+ // proven image by the end: the check at the mounts use sees it.
+ {"between-uses", func(f *deletedExeFixture) (func(bool, int), func(bool, int)) {
+ target, err := os.Readlink(f.task + "/exe")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return func(final bool, call int) {
+ if !final && call == 1 {
+ helper(f)
+ }
+ }, func(final bool, _ int) {
+ if final {
+ generationRelink(t, target, f.task+"/exe")
+ }
+ }
+ }},
+ // The proven image itself is made group-writable mid-inspection.
+ {"image-made-writable", func(f *deletedExeFixture) (func(bool, int), func(bool, int)) {
+ return nil, func(_ bool, call int) {
+ if call == 1 {
+ if err := f.imageFile.Chmod(0o775); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ }},
+ // It execs the helper after the last use, before the end.
+ {"before-the-end", func(f *deletedExeFixture) (func(bool, int), func(bool, int)) {
+ return func(final bool, call int) {
+ if !final && call == 2 {
+ helper(f)
+ }
+ }, nil
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := deletedExe(t)
+ if err := f.execRace(t, nil, nil); err != nil {
+ t.Fatalf("census without a race refused: %v", err)
+ }
+ after, before := tc.race(f)
+ err := f.execRace(t, after, before)
+ if err == nil {
+ t.Fatal("census accepted a task that exec'd away from its proven image")
+ }
+ if !strings.Contains(err.Error(), "exe binding check refused") {
+ t.Fatalf("refusal %q does not name the exe binding check", err)
+ }
+ var task *generationTaskRefusal
+ if !errors.As(err, &task) || !task.transient() {
+ t.Fatalf("refusal %v is not a transient task refusal a later pass can clear", err)
+ }
+ })
+ }
+}
+
+func TestRecoverCensusNeedsABindingForATrustedTask(t *testing.T) {
+ // A kernel wired with a lookup but no recheck refuses every task the
+ // lookup trusts.
+ f := deletedExe(t)
+ classify, _ := f.env.reader(t.Context())
+ if _, err := (generationKernel{proc: f.proc, service: classify}).absent(t.Context(), f.reservation); err == nil || !strings.Contains(err.Error(), "no executable binding") {
+ t.Fatalf("census without a recheck = %v", err)
+ }
+}
+
+func TestRecoverCensusDeadlineCountsInspectionTime(t *testing.T) {
+ clock := time.Unix(0, 0)
+ now := func() time.Time { return clock }
+ sleep := func(_ context.Context, d time.Duration) error { clock = clock.Add(d); return nil }
+ transient := &generationTaskRefusal{task: "/proc/9/task/9", err: os.ErrNotExist}
+
+ // A slow pass that succeeds past the deadline refuses.
+ _, _, err := generationCensusRetryFor(t.Context(), func(context.Context) (string, string, error) {
+ clock = clock.Add(25 * time.Second)
+ return "boot", "digest", nil
+ }, 20*time.Second, now, sleep)
+ if !errors.Is(err, errGenerationCensusLate) {
+ t.Fatalf("a pass that ended past the window = %v, want a late refusal", err)
+ }
+
+ // Slow refused passes: inspection time and waits share the deadline,
+ // no pass starts after it, and the census refuses inside it.
+ clock = time.Unix(0, 0)
+ var starts []time.Duration
+ _, _, err = generationCensusRetryFor(t.Context(), func(context.Context) (string, string, error) {
+ starts = append(starts, clock.Sub(time.Unix(0, 0)))
+ clock = clock.Add(6 * time.Second)
+ return "", "", transient
+ }, 20*time.Second, now, sleep)
+ if err == nil {
+ t.Fatal("persistent refusal accepted")
+ }
+ for _, s := range starts {
+ if s >= 20*time.Second {
+ t.Fatalf("a pass started at %s, past the window (%v)", s, starts)
+ }
+ }
+ if end := clock.Sub(time.Unix(0, 0)); end > 26*time.Second {
+ t.Fatalf("census ran to %s with 6 s passes; it must stop at the first pass that ends past 20 s", end)
+ }
+ if len(starts) < 2 {
+ t.Fatalf("passes started at %v: the slow passes were not retried inside the window", starts)
+ }
+
+ // A wait that overruns (a timer firing late) never starts a pass past
+ // the deadline.
+ clock, starts = time.Unix(0, 0), nil
+ oversleep := func(_ context.Context, d time.Duration) error { clock = clock.Add(d + 3*time.Second); return nil }
+ _, _, err = generationCensusRetryFor(t.Context(), func(context.Context) (string, string, error) {
+ starts = append(starts, clock.Sub(time.Unix(0, 0)))
+ return "", "", transient
+ }, 20*time.Second, now, oversleep)
+ if !errors.Is(err, errGenerationCensusLate) {
+ t.Fatalf("overrun wait = %v, want a late refusal", err)
+ }
+ for _, s := range starts {
+ if s >= 20*time.Second {
+ t.Fatalf("a pass started at %s, past the window (%v)", s, starts)
+ }
+ }
+}
+
+func TestRecoverCensusDeadlineReachesTheInspection(t *testing.T) {
+ // On the real clock: the pass's context ends at the deadline, and a pass
+ // that only returns when it does is refused, promptly.
+ start := time.Now()
+ var deadline time.Time
+ var has bool
+ _, _, err := generationCensusRetryFor(t.Context(), func(ctx context.Context) (string, string, error) {
+ deadline, has = ctx.Deadline()
+ select {
+ case <-ctx.Done():
+ case <-time.After(2 * time.Second):
+ }
+ return "boot", "digest", nil
+ }, 50*time.Millisecond, time.Now, generationSleep)
+ if !errors.Is(err, errGenerationCensusLate) {
+ t.Fatalf("a pass blocked to the deadline = %v, want a late refusal", err)
+ }
+ if !has || deadline.Sub(start) > 50*time.Millisecond+10*time.Millisecond {
+ t.Fatalf("pass context deadline = %v (set %t), want the census's", deadline.Sub(start), has)
+ }
+ if took := time.Since(start); took > 2*time.Second {
+ t.Fatalf("census took %s past a 50 ms window", took)
+ }
+ // The lookup's own blocking reads get that context: systemctl show and
+ // the image hash.
+ f := deletedExe(t)
+ var showDeadline bool
+ show := f.env.show
+ f.env.show = func(ctx context.Context, unit string) (map[string]string, error) {
+ _, showDeadline = ctx.Deadline()
+ return show(ctx, unit)
+ }
+ if _, _, err := generationCensusRetryFor(t.Context(), func(ctx context.Context) (string, string, error) {
+ return "boot", "digest", f.kernelCensus(ctx)
+ }, 20*time.Second, time.Now, generationSleep); err != nil {
+ t.Fatal(err)
+ }
+ if !showDeadline {
+ t.Fatal("systemctl show ran without the census deadline")
+ }
+ // The deadline passes after the lookup has started on the task: the
+ // image hash stops.
+ ctx, cancel := context.WithCancel(t.Context())
+ f.env.show = func(ctx context.Context, unit string) (map[string]string, error) {
+ cancel()
+ return show(ctx, unit)
+ }
+ if err := f.kernelCensus(ctx); !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "deleted exe check refused") {
+ t.Fatalf("image hash under an ended context = %v", err)
+ }
+}
+
+// guardedEdge makes the fixture's unit the guarded carlos-edge.service:
+// ExecStart is the guard, the selected program <opt>/carlos, and the task
+// is the edge's main process in its supervise/ group.
+func (f *deletedExeFixture) guardedEdge(t *testing.T) {
+ t.Helper()
+ f.unit = "carlos-edge.service"
+ f.execStart = "{ path=/opt/carlos/system/guard/carlos ; argv[]=/opt/carlos/system/guard/carlos system guard-exec edge ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=41 ; code=(null) ; status=0/0 }"
+ binary := f.env.binary
+ f.env.binary = func(ctx context.Context, opt, path string) (rootFloorBinary, error) {
+ if path == "/opt/carlos/system/guard/carlos" {
+ return rootFloorBinary{Digest: "guard"}, nil
+ }
+ b, err := binary(ctx, opt, path)
+ b.File.Inode = 7
+ return b, err
+ }
+ base := "/system.slice/carlos-edge.service"
+ generationWrite(t, f.task+"/cgroup", "0::"+base+"/supervise\n")
+ generationWrite(t, f.proc+"/41/cgroup", "0::"+base+"/supervise\n")
+ show := f.env.show
+ f.env.show = func(ctx context.Context, unit string) (map[string]string, error) {
+ props, err := show(ctx, unit)
+ if err == nil {
+ props["ControlGroup"] = base
+ }
+ return props, err
+ }
+}
+
+func TestRecoverCensusDeletedExeOfARootTask(t *testing.T) {
+ // The deleted-exe acceptance also covers a root task of a unit with a
+ // selected image (the guarded edge), and only that among root tasks.
+ // The task runs in a private mount namespace, which the strict root
+ // scan refuses: only a trusted classification lets it pass.
+ root := func(t *testing.T, f *deletedExeFixture) {
+ generationWrite(t, f.task+"/status", "Tgid:\t41\nUid:\t0\t0\t0\t0\nKthread:\t0\n")
+ generationRelink(t, "private-service", f.task+"/ns/mnt")
+ }
+ t.Run("selected-image-signed", func(t *testing.T) {
+ f := deletedExe(t)
+ f.guardedEdge(t)
+ root(t, f)
+ if err := f.census(t); err != nil {
+ t.Fatalf("census refused a root guarded-edge task on its signed deleted image: %v", err)
+ }
+ })
+ t.Run("selected-image-unsigned", func(t *testing.T) {
+ f := deletedExe(t)
+ f.guardedEdge(t)
+ root(t, f)
+ f.writeImage(t, "an image no identity signs", 0o755, true)
+ if err := f.census(t); err == nil {
+ t.Fatal("census accepted a root guarded-edge task on an unsigned deleted image")
+ }
+ })
+ t.Run("no-selected-image", func(t *testing.T) {
+ // A root task of a unit with no selected image keeps main's path
+ // walk, which a deleted exe fails, so the strict root scan applies.
+ f := deletedExe(t)
+ root(t, f)
+ if err := f.census(t); err == nil {
+ t.Fatal("census trusted a root task of an unselected unit on its deleted exe")
+ }
+ })
+ t.Run("selected-image-exec-race", func(t *testing.T) {
+ f := deletedExe(t)
+ f.guardedEdge(t)
+ root(t, f)
+ err := f.execRace(t, nil, func(_ bool, call int) {
+ if call == 1 {
+ generationRelink(t, f.unrelated, f.task+"/exe")
+ }
+ })
+ if err == nil || !strings.Contains(err.Error(), "exe binding check refused") {
+ t.Fatalf("root guarded-edge task that exec'd away = %v", err)
+ }
+ })
+}
+
+// slowImage returns one byte per read and ends ctx after the first.
+type slowImage struct {
+ cancel func()
+ reads int
+}
+
+func (s *slowImage) Read(p []byte) (int, error) {
+ s.reads++
+ if s.reads == 1 {
+ s.cancel()
+ }
+ if len(p) == 0 {
+ return 0, nil
+ }
+ p[0] = 'x'
+ return 1, nil
+}
+
+func TestSignedBinaryProofStopsWhenTheCensusDeadlineEnds(t *testing.T) {
+ // The hash: a slow image whose read outlives the deadline.
+ ctx, cancel := context.WithCancel(t.Context())
+ image := &slowImage{cancel: cancel}
+ if _, err := rootFloorDigest(ctx, image); !errors.Is(err, context.Canceled) || image.reads > 2 {
+ t.Fatalf("digest = %v after %d reads, want it stopped at the deadline", err, image.reads)
+ }
+ // The identity scan: once the deadline has passed it reads no identity.
+ reads := 0
+ ctx, cancel = context.WithCancel(t.Context())
+ cancel()
+ if _, err := rootFloorSignedIdentity(ctx, nil, rootFloorBinary{Digest: "d"}, []string{"a.json"}, func(string) ([]byte, error) { reads++; return nil, nil }); !errors.Is(err, context.Canceled) || reads != 0 {
+ t.Fatalf("identity scan after the deadline = %v after %d reads", err, reads)
+ }
+}
+
+func TestRecoverCensusDeadlineReachesTheSignedBinaryProof(t *testing.T) {
+ // The lookup hands the census's deadline context to the signed-binary
+ // proof, for the deleted image and for the guarded edge's guard and
+ // selected images: a proof that only returns when that context ends
+ // is cut off at the deadline.
+ for _, guarded := range []bool{false, true} {
+ t.Run(map[bool]string{false: "deleted-image", true: "guarded-edge"}[guarded], func(t *testing.T) {
+ f := deletedExe(t)
+ if guarded {
+ f.guardedEdge(t)
+ }
+ proofs := 0
+ f.env.binary = func(ctx context.Context, opt, path string) (rootFloorBinary, error) {
+ proofs++
+ if _, ok := ctx.Deadline(); !ok {
+ t.Error("signed-binary proof ran without the census deadline")
+ }
+ select {
+ case <-ctx.Done():
+ return rootFloorBinary{}, ctx.Err()
+ case <-time.After(5 * time.Second):
+ return rootFloorBinary{}, errors.New("proof outlived the deadline")
+ }
+ }
+ start := time.Now()
+ _, _, err := generationCensusRetryFor(t.Context(), func(ctx context.Context) (string, string, error) {
+ return "boot", "digest", f.kernelCensus(ctx)
+ }, 100*time.Millisecond, time.Now, generationSleep)
+ if !errors.Is(err, errGenerationCensusLate) {
+ t.Fatalf("census = %v, want a late refusal", err)
+ }
+ if took := time.Since(start); took > 2*time.Second {
+ t.Fatalf("census took %s past a 100 ms window: the proof ignored its context", took)
+ }
+ if proofs == 0 {
+ t.Fatal("the proof never ran")
+ }
+ })
+ }
+}
diff --git a/internal/host/rootgeneration_kernel_linux.go b/internal/host/rootgeneration_kernel_linux.go
index ea3e73966..969191b79 100644
--- a/internal/host/rootgeneration_kernel_linux.go
+++ b/internal/host/rootgeneration_kernel_linux.go
@@ -30,6 +30,10 @@ type generationKernel struct {
owner uint32
contained []release.GenerationReservation
service func(string, []byte) (bool, error)
+ // recheck holds a task service trusted to the image it was trusted for,
+ // at each use of that trust and, final, at the end of its inspection
+ // (generationServiceEnv.recheck). A trusted task needs it.
+ recheck func(path string, final bool) error
}
func generationRead(path string) ([]byte, error) {
@@ -366,9 +370,12 @@ func (k generationKernel) task(ctx context.Context, path string, b release.Gener
return "", err
}
end, err := generationStart(again)
- if err != nil || end != start {
+ if err != nil {
return "", os.ErrPermission
}
+ if end != start {
+ return "", errGenerationTaskReplaced
+ }
return start, nil
}
}
@@ -386,6 +393,24 @@ func (k generationKernel) task(ctx context.Context, path string, b release.Gener
if rootErr != nil && !trusted {
return "", rootErr
}
+ bound := func(final bool) error {
+ if !trusted {
+ return nil
+ }
+ if k.recheck == nil {
+ return fmt.Errorf("service: no executable binding: %w", os.ErrPermission)
+ }
+ if err := k.recheck(path, final); err != nil {
+ return fmt.Errorf("service: %w", err)
+ }
+ return nil
+ }
+ if trusted && k.recheck != nil {
+ defer k.recheck(path, true)
+ }
+ if err := bound(false); err != nil {
+ return "", err
+ }
if !kernel {
for name, want := range ns {
got, err := os.Readlink(path + "/ns/" + name)
@@ -415,6 +440,9 @@ func (k generationKernel) task(ctx context.Context, path string, b release.Gener
if err != nil {
return "", err
}
+ if err := bound(false); err != nil {
+ return "", err
+ }
if err := generationMounts(mounts, b, trusted); err != nil {
return "", fmt.Errorf("mounts: %w", err)
}
@@ -424,9 +452,15 @@ func (k generationKernel) task(ctx context.Context, path string, b release.Gener
return "", err
}
again, err := generationStart(raw)
- if err != nil || again != start {
+ if err != nil {
return "", os.ErrPermission
}
+ if again != start {
+ return "", errGenerationTaskReplaced
+ }
+ if err := bound(true); err != nil {
+ return "", err
+ }
return start, nil
}
func (k generationKernel) absent(ctx context.Context, r release.GenerationReservation) (string, error) {
@@ -471,7 +505,7 @@ func (k generationKernel) absent(ctx context.Context, r release.GenerationReserv
}
tasks, err := generationNames(filepath.Join(k.proc, name, "task"))
if err != nil {
- return "", err
+ return "", &generationTaskRefusal{task: filepath.Join(k.proc, name), err: err}
}
if len(tasks) == 0 {
return "", os.ErrPermission
@@ -486,7 +520,7 @@ func (k generationKernel) absent(ctx context.Context, r release.GenerationReserv
}
start, err := k.task(ctx, filepath.Join(k.proc, name, "task", task), r.Binding, ns, refs)
if err != nil {
- return "", fmt.Errorf("%s: %w", filepath.Join(k.proc, name, "task", task), err)
+ return "", &generationTaskRefusal{task: filepath.Join(k.proc, name, "task", task), err: err}
}
observations[name+"/"+task] = start
}
@@ -520,7 +554,8 @@ func productionGenerationAbsence(ctx context.Context, r release.GenerationReserv
if err != nil {
return "", err
}
- return (generationKernel{proc: "/proc", service: generationServiceReader(ctx)}).absent(ctx, current)
+ service, recheck := generationServiceReader(ctx)
+ return (generationKernel{proc: "/proc", service: service, recheck: recheck}).absent(ctx, current)
}
func generationRecordedAbsence(ctx context.Context, r release.GenerationReservation) (release.GenerationReservation, error) {
diff --git a/internal/host/rootgeneration_refusal.go b/internal/host/rootgeneration_refusal.go
new file mode 100644
index 000000000..77da17f38
--- /dev/null
+++ b/internal/host/rootgeneration_refusal.go
@@ -0,0 +1,42 @@
+package host
+
+import "errors"
+
+// generationCensusRefusal is a refusal of root's kernel generation census
+// (productionGenerationKernelCensus). Recovery with no journal open defers
+// a settlement this refused (settleWithoutJournal), and only this: an image
+// that does not verify, malformed state and a failed durable write keep
+// their own errors and still fail.
+type generationCensusRefusal struct{ err error }
+
+func (r *generationCensusRefusal) Error() string { return "generation census: " + r.err.Error() }
+
+func (r *generationCensusRefusal) Unwrap() error { return r.err }
+
+// onlyCensusRefusal reports whether every error err carries is a
+// generationCensusRefusal: a wrapped refusal is, a join with any other
+// error (a durable write that failed beside it) is not.
+func onlyCensusRefusal(err error) bool {
+ switch e := err.(type) {
+ case nil:
+ return false
+ case *generationCensusRefusal:
+ return true
+ case interface{ Unwrap() []error }:
+ found := false
+ for _, inner := range e.Unwrap() {
+ if inner == nil {
+ continue
+ }
+ if !onlyCensusRefusal(inner) {
+ return false
+ }
+ found = true
+ }
+ return found
+ }
+ if inner := errors.Unwrap(err); inner != nil {
+ return onlyCensusRefusal(inner)
+ }
+ return false
+}
diff --git a/internal/host/rootgeneration_retired_linux.go b/internal/host/rootgeneration_retired_linux.go
index ebf91d641..6b3a48c8f 100644
--- a/internal/host/rootgeneration_retired_linux.go
+++ b/internal/host/rootgeneration_retired_linux.go
@@ -243,7 +243,8 @@ func productionRetiredGenerationKernelCensus(ctx context.Context) (string, strin
}
func generationRetiredController(ctx context.Context, b release.GenerationBinding) error {
- k := generationKernel{proc: "/proc", service: generationServiceReader(ctx)}
+ service, recheck := generationServiceReader(ctx)
+ k := generationKernel{proc: "/proc", service: service, recheck: recheck}
if k.controllerGone(ctx, b) == nil {
return nil
}
diff --git a/internal/host/rootgeneration_services_linux.go b/internal/host/rootgeneration_services_linux.go
index c29da9545..9075333ef 100644
--- a/internal/host/rootgeneration_services_linux.go
+++ b/internal/host/rootgeneration_services_linux.go
@@ -4,10 +4,15 @@ package host
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
"fmt"
+ "io"
"os"
"os/exec"
"path/filepath"
+ "slices"
"strconv"
"strings"
@@ -16,23 +21,175 @@ import (
)
type generationService struct {
- group, executable string
- main int
- selected rootstate.FileIdentity
+ group, executable, selectedPath string
+ main int
+ selected rootstate.FileIdentity
}
-func generationServiceReader(ctx context.Context) func(string, []byte) (bool, error) {
+// generationServiceEnv is what the service lookup reads besides the task's
+// own /proc entries: systemd's view of the unit (show), the unit files'
+// ownership (trust), the main process's cgroup (under proc), and, for a
+// task still running a renamed-over binary, the box's signed router images
+// (binary, under opt) and the owner every trusted file must have. The
+// production values are generationServiceReader's; tests replace the parts
+// a non-root test cannot provide.
+type generationServiceEnv struct {
+ proc, opt string
+ owner uint32
+ show func(ctx context.Context, unit string) (map[string]string, error)
+ trust func(path string) error
+ binary func(ctx context.Context, opt, path string) (rootFloorBinary, error)
+}
+
+// generationServiceRefusal is a refusal of the service lookup. It names the
+// check, the unit, the task's cgroup and its exe link, so that refusals
+// that used to read as a bare "permission denied" can be told apart, and
+// it wraps the cause, so errors.Is still sees os.ErrPermission or a task
+// that exited.
+type generationServiceRefusal struct {
+ check, unit, group, exe string
+ err error
+}
+
+func (r *generationServiceRefusal) Error() string {
+ return fmt.Sprintf("%s check refused: unit %s, cgroup %s, exe %s: %v", r.check, r.unit, r.group, r.exe, r.err)
+}
+
+func (r *generationServiceRefusal) Unwrap() error { return r.err }
+
+// generationServiceReader is the production service lookup: classify a
+// task, and recheck the executable binding of a task it trusted
+// (generationKernel.recheck).
+func generationServiceReader(ctx context.Context) (func(string, []byte) (bool, error), func(string, bool) error) {
+ return generationServiceEnv{proc: "/proc", opt: "/opt/carlos", show: generationServiceShow, trust: generationServiceTrust, binary: readRootFloorBinaryContext}.reader(ctx)
+}
+
+// generationExeID is what binds a task to the image it was proven to run:
+// the device and inode of the descriptor opened through /proc/<pid>/exe,
+// the kernel's file handle for it where the filesystem gives one, and its
+// mount ID. rootstate.ObjectIdentity carries the same handle but also
+// resolves the mount in root's own mount table, which a service in a
+// private mount namespace does not share, so only its kernel facts are
+// used here.
+type generationExeID struct {
+ device, inode, mount uint64
+ handle string
+ handleType int32
+}
+
+func generationExeIdentity(f *os.File) (generationExeID, error) {
+ var st unix.Stat_t
+ if err := unix.Fstat(int(f.Fd()), &st); err != nil {
+ return generationExeID{}, err
+ }
+ var sx unix.Statx_t
+ if err := unix.Statx(int(f.Fd()), "", unix.AT_EMPTY_PATH, unix.STATX_MNT_ID, &sx); err != nil {
+ return generationExeID{}, err
+ }
+ id := generationExeID{device: uint64(st.Dev), inode: st.Ino, mount: sx.Mnt_id}
+ handle, _, err := unix.NameToHandleAt(int(f.Fd()), "", unix.AT_EMPTY_PATH)
+ if err == nil {
+ id.handle, id.handleType = hex.EncodeToString(handle.Bytes()), handle.Type()
+ } else if !errors.Is(err, unix.EOPNOTSUPP) {
+ return generationExeID{}, err
+ }
+ return id, nil
+}
+
+// generationExeBinding is a task the lookup trusted because the image it
+// ran was proven: that image's descriptor, kept open for the inspection so
+// its inode cannot be reused, and its identity.
+type generationExeBinding struct {
+ file *os.File
+ id generationExeID
+ unit, group, exe string
+}
+
+func generationServiceShow(ctx context.Context, unit string) (map[string]string, error) {
+ out, err := exec.CommandContext(ctx, "systemctl", "show", unit, "-p", "ControlGroup", "-p", "MainPID", "-p", "ExecStart", "-p", "FragmentPath", "-p", "DropInPaths").Output()
+ if err != nil {
+ return nil, err
+ }
+ props := map[string]string{}
+ for _, line := range strings.Split(string(out), "\n") {
+ if k, v, ok := strings.Cut(line, "="); ok {
+ props[k] = v
+ }
+ }
+ return props, nil
+}
+
+func generationServiceTrust(file string) error {
+ file, err := filepath.EvalSymlinks(file)
+ if err != nil {
+ return err
+ }
+ f, err := rootstate.OpenTrustedFile(file)
+ if err != nil {
+ return err
+ }
+ f.Close()
+ return nil
+}
+
+func (env generationServiceEnv) reader(ctx context.Context) (func(string, []byte) (bool, error), func(string, bool) error) {
units := map[string]generationService{}
- return func(path string, status []byte) (bool, error) {
+ bound := map[string]*generationExeBinding{}
+ var proven []string
+ var provenErr error
+ proved := false
+ // The digests of the current and the previous router binary, each
+ // proven against a signed identity the box holds, read once per census
+ // pass and only if a task's exe has been deleted.
+ prove := func() ([]string, error) {
+ if !proved {
+ proved = true
+ var errs []error
+ for _, name := range []string{"carlos", "carlos.prev"} {
+ b, err := env.binary(ctx, env.opt, filepath.Join(env.opt, name))
+ if err != nil {
+ if name == "carlos" || !errors.Is(err, os.ErrNotExist) {
+ errs = append(errs, fmt.Errorf("%s: %w", name, err))
+ }
+ continue
+ }
+ proven = append(proven, b.Digest)
+ }
+ if len(proven) == 0 {
+ provenErr = errors.Join(append(errs, os.ErrPermission)...)
+ }
+ }
+ return proven, provenErr
+ }
+ classify := func(path string, status []byte) (bool, error) {
+ var held *os.File
+ defer func() {
+ if held != nil {
+ held.Close()
+ }
+ }()
+ if old := bound[path]; old != nil {
+ old.file.Close()
+ delete(bound, path)
+ }
+ name, nameErr := os.Readlink(path + "/exe")
+ exe := name
+ if nameErr != nil {
+ exe = "unreadable"
+ }
+ unit, group := "unknown", "unknown"
+ refuse := func(check string, err error) error {
+ return &generationServiceRefusal{check: check, unit: unit, group: group, exe: exe, err: err}
+ }
raw, err := generationRead(path + "/cgroup")
if err != nil {
- return false, err
+ return false, refuse("cgroup", err)
}
- group := strings.TrimPrefix(strings.TrimSpace(string(raw)), "0::")
+ group = strings.TrimPrefix(strings.TrimSpace(string(raw)), "0::")
if !strings.HasPrefix(group, "/") || strings.Contains(group, "\n") || strings.Contains(group, "/carlos-instances.slice/") {
- return false, os.ErrPermission
+ return false, refuse("cgroup", os.ErrPermission)
}
- unit := filepath.Base(group)
+ unit = filepath.Base(group)
for _, part := range strings.Split(group, "/") {
if strings.HasSuffix(part, ".service") {
unit = part
@@ -40,91 +197,199 @@ func generationServiceReader(ctx context.Context) func(string, []byte) (bool, er
}
}
if !strings.HasSuffix(unit, ".service") && group != "/init.scope" {
- return false, os.ErrPermission
+ return false, refuse("unit", os.ErrPermission)
}
s, ok := units[unit]
if !ok {
if group == "/init.scope" {
s = generationService{group: group, main: 1}
} else {
- out, err := exec.CommandContext(ctx, "systemctl", "show", unit, "-p", "ControlGroup", "-p", "MainPID", "-p", "ExecStart", "-p", "FragmentPath", "-p", "DropInPaths").Output()
+ props, err := env.show(ctx, unit)
if err != nil {
- return false, err
- }
- props := map[string]string{}
- for _, line := range strings.Split(string(out), "\n") {
- if k, v, ok := strings.Cut(line, "="); ok {
- props[k] = v
- }
+ return false, refuse("systemctl show", err)
}
s.group, s.main = props["ControlGroup"], 0
s.main, err = strconv.Atoi(props["MainPID"])
if err != nil || s.main <= 0 || props["FragmentPath"] == "" {
- return false, os.ErrPermission
+ return false, refuse("main pid", os.ErrPermission)
}
for _, file := range append([]string{props["FragmentPath"]}, strings.Fields(props["DropInPaths"])...) {
- file, err = filepath.EvalSymlinks(file)
- if err != nil {
- return false, err
- }
- f, err := rootstate.OpenTrustedFile(file)
- if err != nil {
- return false, err
+ if err := env.trust(file); err != nil {
+ return false, refuse("unit file "+file, err)
}
- f.Close()
}
_, tail, found := strings.Cut(props["ExecStart"], "path=")
if !found {
- return false, os.ErrPermission
+ return false, refuse("exec start", os.ErrPermission)
}
s.executable, _, _ = strings.Cut(tail, " ;")
if unit == "carlos-edge.service" && s.executable == "/opt/carlos/system/guard/carlos" {
if !generationGuardEdgeStart(unit, props["ExecStart"]) {
- return false, os.ErrPermission
+ return false, refuse("guard start", os.ErrPermission)
}
- if _, err := readRootFloorBinary("/opt/carlos", s.executable); err != nil {
- return false, err
+ if _, err := env.binary(ctx, env.opt, s.executable); err != nil {
+ return false, refuse("guard binary", err)
}
- selected, err := readRootFloorBinary("/opt/carlos", "/opt/carlos/carlos")
+ selected, err := env.binary(ctx, env.opt, filepath.Join(env.opt, "carlos"))
if err != nil {
- return false, err
+ return false, refuse("selected binary", err)
}
- s.selected = selected.File
+ s.selected, s.selectedPath = selected.File, filepath.Join(env.opt, "carlos")
}
}
units[unit] = s
}
supervisor := generationServiceSupervisor(unit, group, s.group, status, s.main)
if !generationServiceGroup(group, s.group) && !supervisor {
- return false, os.ErrPermission
+ return false, refuse("group", os.ErrPermission)
}
- name, err := os.Readlink(path + "/exe")
- if err != nil {
- return false, err
- }
- actual, err := generationServiceExecutable(path, name)
- if err != nil {
- return false, fmt.Errorf("%s: %w", name, err)
+ if nameErr != nil {
+ return false, refuse("exe link", nameErr)
}
// Non-root services may execute only the program selected by their
// root-owned unit. UID and a service-looking cgroup are insufficient.
- if _, err := generationRootTask(status); err != nil || s.selected != (rootstate.FileIdentity{}) {
- want, e := generationIdentity(s.executable)
+ _, rootErr := generationRootTask(status)
+ checked := rootErr != nil || s.selected != (rootstate.FileIdentity{})
+ if program, deleted := strings.CutSuffix(name, " (deleted)"); deleted && checked {
+ want := s.executable
if s.selected != (rootstate.FileIdentity{}) {
- want, e = s.selected, nil
+ want = s.selectedPath
+ }
+ image, err := env.deletedProgram(ctx, path, program, want, prove)
+ if err != nil {
+ return false, refuse("deleted exe", err)
}
- if e != nil || want != actual {
- return false, os.ErrPermission
+ held = image
+ } else {
+ actual, err := generationServiceExecutable(path, name)
+ if err != nil {
+ return false, refuse("exe", fmt.Errorf("%s: %w", name, err))
+ }
+ if checked {
+ want, e := generationIdentity(s.executable)
+ if s.selected != (rootstate.FileIdentity{}) {
+ want, e = s.selected, nil
+ }
+ if e != nil || want != actual {
+ return false, refuse("selected program", os.ErrPermission)
+ }
}
}
- mainGroup, err := generationRead("/proc/" + strconv.Itoa(s.main) + "/cgroup")
+ mainGroup, err := generationRead(env.proc + "/" + strconv.Itoa(s.main) + "/cgroup")
mainPath := strings.TrimPrefix(strings.TrimSpace(string(mainGroup)), "0::")
if err != nil || (!generationServiceGroup(mainPath, s.group) && !(supervisor && mainPath == group)) {
- return false, os.ErrPermission
+ if err == nil {
+ err = os.ErrPermission
+ }
+ return false, refuse("main cgroup", err)
+ }
+ if held != nil {
+ id, err := generationExeIdentity(held)
+ if err != nil {
+ return false, refuse("exe binding", err)
+ }
+ bound[path] = &generationExeBinding{file: held, id: id, unit: unit, group: group, exe: exe}
+ held = nil
}
return true, nil
}
+ return classify, func(path string, final bool) error { return env.recheck(bound, path, final) }
+}
+
+// recheck holds a trusted task to the image it was proven to run, at each
+// later use of that trust and at the end of its inspection (final).
+// /proc/<pid>/exe, opened again, must be the very inode that was hashed (device, inode, mount and file handle),
+// still a root-owned regular file with no links left and no group or other
+// write. A task that exec'd anything else since, the same PID with the same
+// start time, refuses; a later census pass may classify it afresh. The
+// bytes are not hashed again: the identity is the hashed descriptor's, which
+// stays open, and the kernel refuses writes to an image being executed
+// (ETXTBSY). final releases the binding whatever the result. A task the
+// lookup trusted without a proven image (its live exe was the unit's
+// selected program, as on main) has no binding and passes.
+func (env generationServiceEnv) recheck(bound map[string]*generationExeBinding, path string, final bool) error {
+ b := bound[path]
+ if b == nil {
+ return nil
+ }
+ if final {
+ defer func() {
+ b.file.Close()
+ delete(bound, path)
+ }()
+ }
+ refuse := func(err error) error {
+ return &generationServiceRefusal{check: "exe binding", unit: b.unit, group: b.group, exe: b.exe, err: err}
+ }
+ f, err := os.Open(path + "/exe")
+ if err != nil {
+ return refuse(err)
+ }
+ defer f.Close()
+ id, err := generationExeIdentity(f)
+ if err != nil {
+ return refuse(err)
+ }
+ var st unix.Stat_t
+ if err := unix.Fstat(int(f.Fd()), &st); err != nil {
+ return refuse(err)
+ }
+ if id != b.id || st.Uid != env.owner || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Mode&0022 != 0 || st.Nlink != 0 {
+ return refuse(fmt.Errorf("the task's image is no longer the one proven: %w", os.ErrPermission))
+ }
+ return nil
+}
+
+// deletedProgram accepts a non-root service task still running a carlos
+// binary that an apply has since renamed over, whose exe link reads
+// "<program> (deleted)". The program must be the one the unit selects and
+// the box's router binary, <opt>/carlos. The task's own image, opened
+// through /proc/<pid>/exe, must be a root-owned regular file with no links
+// left that no group or other can write. Its bytes must hash to the router
+// artifact of the signed identity of the current or the previous binary
+// (<opt>/carlos, <opt>/carlos.prev), proven by readRootFloorBinary, the
+// proof the guard makes before it execs one. Anything else refuses, and
+// the census's reference checks still run on an accepted task.
+//
+// On success it returns the image's descriptor, open: the lookup keeps it
+// to bind the task to that image for the rest of its inspection (recheck).
+func (env generationServiceEnv) deletedProgram(ctx context.Context, task, program, want string, prove func() ([]string, error)) (*os.File, error) {
+ if program != want || want != filepath.Join(env.opt, "carlos") {
+ return nil, fmt.Errorf("%s is not this unit's router binary: %w", program, os.ErrPermission)
+ }
+ f, err := os.Open(task + "/exe")
+ if err != nil {
+ return nil, err
+ }
+ keep := false
+ defer func() {
+ if !keep {
+ f.Close()
+ }
+ }()
+ var st unix.Stat_t
+ if err := unix.Fstat(int(f.Fd()), &st); err != nil {
+ return nil, err
+ }
+ if st.Uid != env.owner || st.Mode&unix.S_IFMT != unix.S_IFREG || st.Mode&0022 != 0 || st.Nlink != 0 || st.Size > 512<<20 {
+ return nil, fmt.Errorf("image is not an unlinked root-owned file: %w", os.ErrPermission)
+ }
+ sum := sha256.New()
+ if n, err := io.Copy(sum, generationContextReader{ctx: ctx, r: io.LimitReader(f, 512<<20+1)}); err != nil {
+ return nil, err
+ } else if n > 512<<20 {
+ return nil, os.ErrPermission
+ }
+ digests, err := prove()
+ if err != nil {
+ return nil, err
+ }
+ if !slices.Contains(digests, hex.EncodeToString(sum.Sum(nil))) {
+ return nil, fmt.Errorf("image is neither the current nor the previous signed router build: %w", os.ErrPermission)
+ }
+ keep = true
+ return f, nil
}
func generationGuardEdgeStart(unit, start string) bool {
diff --git a/internal/host/rootmovesfloor_refresh.go b/internal/host/rootmovesfloor_refresh.go
index 2dfd7b464..5b080c09c 100644
--- a/internal/host/rootmovesfloor_refresh.go
+++ b/internal/host/rootmovesfloor_refresh.go
@@ -2,6 +2,7 @@ package host
import (
"context"
+ "crypto/ed25519"
"crypto/sha256"
"encoding/hex"
"encoding/json"
@@ -149,6 +150,14 @@ func rootFloorAdvertisementFor(public rootstate.Store, p rootFloorProjection) (r
// A binary's signed identity, not its mutable version sidecar, supplies its
// release and protocol. The cache is root-owned; each envelope is reverified.
func readRootFloorBinary(opt, path string) (rootFloorBinary, error) {
+ return readRootFloorBinaryContext(context.Background(), opt, path)
+}
+
+// readRootFloorBinaryContext is readRootFloorBinary under ctx: the hash of
+// the image stops between reads, and the scan of the identity cache before
+// each identity, once ctx ends. The generation census's service lookup
+// proves signed images with it under the census deadline.
+func readRootFloorBinaryContext(ctx context.Context, opt, path string) (rootFloorBinary, error) {
var b rootFloorBinary
f, err := rootstate.OpenAuthorityFile(path)
if err != nil {
@@ -166,11 +175,13 @@ func readRootFloorBinary(opt, path string) (rootFloorBinary, error) {
if st.Size() > 512<<20 || st.Mode()&0111 == 0 {
return b, os.ErrPermission
}
- sum := sha256.New()
- if _, err := io.Copy(sum, f); err != nil {
+ sum, err := rootFloorDigest(ctx, f)
+ if err != nil {
return b, err
}
- b.Digest = hex.EncodeToString(sum.Sum(nil))
+ // One single-valued hex write, as before: the moves copy gate derives the
+ // Digest that reaches a seal's Reason from here (approved_moves_copy_test).
+ b.Digest = hex.EncodeToString(sum)
pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
if err != nil {
return b, err
@@ -184,8 +195,28 @@ func readRootFloorBinary(opt, path string) (rootFloorBinary, error) {
if err != nil {
return b, err
}
+ return rootFloorSignedIdentity(ctx, pub, b, names, identities.Read)
+}
+
+// rootFloorDigest is the sha256 of an image, read through ctx: the raw sum,
+// which the caller hex-encodes in place.
+func rootFloorDigest(ctx context.Context, image io.Reader) ([]byte, error) {
+ sum := sha256.New()
+ if _, err := io.Copy(sum, generationContextReader{ctx: ctx, r: image}); err != nil {
+ return nil, err
+ }
+ return sum.Sum(nil), nil
+}
+
+// rootFloorSignedIdentity finds the cached signed identity whose router
+// artifact for this platform is b's digest, verifying each envelope, and
+// stops before the next identity once ctx ends.
+func rootFloorSignedIdentity(ctx context.Context, pub ed25519.PublicKey, b rootFloorBinary, names []string, read func(string) ([]byte, error)) (rootFloorBinary, error) {
for _, name := range names {
- raw, err := identities.Read(name)
+ if err := ctx.Err(); err != nil {
+ return b, err
+ }
+ raw, err := read(name)
if err != nil {
return b, err
}
@@ -201,6 +232,20 @@ func readRootFloorBinary(opt, path string) (rootFloorBinary, error) {
return b, errRootFloorHistory
}
+// generationContextReader stops a long read, an image hash, once its
+// context ends.
+type generationContextReader struct {
+ ctx context.Context
+ r io.Reader
+}
+
+func (c generationContextReader) Read(p []byte) (int, error) {
+ if err := c.ctx.Err(); err != nil {
+ return 0, err
+ }
+ return c.r.Read(p)
+}
+
func rootFloorDowngradeIdle(ctx context.Context) error {
l, held, err := rootparticipation.TransactionLedger(ctx)
if err != nil {
diff --git a/internal/host/systemapply.go b/internal/host/systemapply.go
index 9a959dadf..da966c116 100644
--- a/internal/host/systemapply.go
+++ b/internal/host/systemapply.go
@@ -24,6 +24,7 @@ import (
"github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/cliflag"
"github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostwrite"
"github.com/carlosframework/platform/internal/layout"
"github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/provision"
@@ -52,13 +53,22 @@ const (
// design §5). It execs /opt/carlos/carlos and stays up for weeks once
// the first wake activated it, so every install that changes the
// binary restarts it (restartInstanceControl; discussion #256).
- systemControlUnit = "carlos-instance-control.service"
- systemLockFile = "lock"
- systemMaxRequest = 4 << 10
- systemGateWait = 20 * time.Second
- systemGateSoak = 15 * time.Second
- systemVersionWait = 30 * time.Second
- systemBootstrapWait = 120 * time.Second
+ systemControlUnit = "carlos-instance-control.service"
+ systemLockFile = "lock"
+ systemMaxRequest = 4 << 10
+ systemGateWait = 20 * time.Second
+ // systemClaimsGateWait is the gate's window on a box running with
+ // CARLOS_HOST_CLAIM on. There the agent finishes its whole claim
+ // startup (assembly, wiring, the inline activator and duties) before
+ // the edge's listener binds, which takes about 22 s on a small
+ // deployment and grows with the deployment's app count. A fixed 20 s
+ // rolled every update back, and its rollback too. The longer window
+ // comes with fail-fast (gateFailFast), so a build that crashes still
+ // rolls back in seconds. Claims-off boxes keep systemGateWait exactly.
+ systemClaimsGateWait = 120 * time.Second
+ systemGateSoak = 15 * time.Second
+ systemVersionWait = 30 * time.Second
+ systemBootstrapWait = 120 * time.Second
// How long --recover waits for the apply lock, and how often it asks.
// See waitForLock.
@@ -116,11 +126,12 @@ func RunSystemApply(args []string) {
if invalidChannel != "" {
log.Printf("system apply: %s=%q is not a channel; this box follows %s", systemChannelEnv, invalidChannel, channel)
}
+ claims, gateWait := systemApplyGate(os.Getenv)
a := &systemApplier{
rootMoves: rootMovesFloorIdentity, ops: productionSysOps(), openStore: openSystemStore, channel: channel,
optDir: release.OptDir(), dataDir: dataDir(), etcDir: "/etc", build: version.Build,
- platform: runtime.GOOS + "-" + runtime.GOARCH, gateWait: systemGateWait, soak: systemGateSoak,
- regionHA: regionHAEnabled(os.Getenv),
+ platform: runtime.GOOS + "-" + runtime.GOARCH, gateWait: gateWait, soak: systemGateSoak,
+ regionHA: regionHAEnabled(os.Getenv), claims: claims,
}
a.floorHook = installFloorHook // decided under the apply lock, in Run
runErr := a.runWithRelease(ctx, *recover)
@@ -249,6 +260,12 @@ type systemApplier struct {
// refuses an update while this box takes part in a claim operation
// (region HA spec 1 §5.10; systemparticipation.go).
regionHA bool
+ // claims is CARLOS_HOST_CLAIM, read with hostwrite.Enabled from the
+ // unit's environment (carlos-system-update.service and
+ // carlos-system-recover.service both load host.env). It changes only the
+ // gate: the claims window and fail-fast (systemGateWaitFor,
+ // gateFailFast). Off, the gate is exactly the 20 s one.
+ claims bool
// store is the deploy store verifyPointer opened, kept for armFloor.
// Nil on --recover, which reads no store.
store release.Store
@@ -311,9 +328,15 @@ func (a *systemApplier) Run(ctx context.Context, recover bool) (retErr error) {
return err
}
}
- // Rejected floor recovery must retain staging and the journal.
+ // Rejected floor recovery must retain staging and the journal. A
+ // settlement deferred with no journal open (errSettlementDeferred) lets
+ // --recover go on and exit 0; it is not tried again at the end.
+ deferred := false
if err := a.checkGuardRecovery(ctx); err != nil {
- return err
+ if !a.deferSettlement(recover, err) {
+ return err
+ }
+ deferred = true
}
defer func() {
active, err := a.activeFloor()
@@ -322,16 +345,20 @@ func (a *systemApplier) Run(ctx context.Context, recover bool) (retErr error) {
return
}
if !active {
- a.settleUpdateBegun(ctx)
+ if !deferred {
+ a.settleMarker(ctx, recover, &retErr)
+ }
_ = os.RemoveAll(a.stagingDir())
return
}
- if j, e := readSystemJournal(a.optDir); e == nil && j == nil {
+ if j, e := readSystemJournal(a.optDir); e == nil && j == nil && !deferred {
if err := a.checkGuardRecovery(ctx); err != nil {
- retErr = errors.Join(retErr, err)
+ if !a.deferSettlement(recover, err) {
+ retErr = errors.Join(retErr, err)
+ }
return
}
- a.settleUpdateBegun(ctx)
+ a.settleMarker(ctx, recover, &retErr)
_ = os.RemoveAll(a.stagingDir())
}
}()
@@ -342,9 +369,15 @@ func (a *systemApplier) Run(ctx context.Context, recover bool) (retErr error) {
// main leaves it, and the triggers wait for the run that gets past it.
a.rearmPending()
+ // A settlement the census refused after recovery durably removed the
+ // journal is deferred like one with no journal open: --recover exits 0
+ // and does not try it again at the end of the run.
recovered, err := a.recoverIfNeeded(ctx)
if err != nil {
- return err
+ if !a.deferSettlement(recover, err) {
+ return err
+ }
+ deferred = true
}
if recover {
return nil
@@ -1479,7 +1512,7 @@ func (a *systemApplier) restartRelay() {
func (a *systemApplier) gate(ctx context.Context, target string) error {
wait, soak := a.gateWait, a.soak
if wait <= 0 {
- wait = systemGateWait
+ wait = systemGateWaitFor(a.claims)
}
if soak <= 0 {
soak = systemGateSoak
@@ -1487,10 +1520,16 @@ func (a *systemApplier) gate(ctx context.Context, target string) error {
deadline := a.ops.now().Add(wait)
var lastErr error
for {
- lastErr = a.gateOnce(ctx, target)
+ var state string
+ state, lastErr = a.gateCheck(ctx, target)
if lastErr == nil {
break
}
+ if a.claims {
+ if err := a.gateFailFast(state, lastErr); err != nil {
+ return err
+ }
+ }
if !a.ops.now().Before(deadline) {
return lastErr
}
@@ -1517,6 +1556,49 @@ func (a *systemApplier) gate(ctx context.Context, target string) error {
return a.gateOnce(ctx, target)
}
+// systemApplyGate is how RunSystemApply reads claims mode for the gate: the
+// unit's CARLOS_HOST_CLAIM through hostwrite.Enabled, the test the agent
+// and the claim wiring make, and the window that follows from it.
+func systemApplyGate(getenv func(string) string) (bool, time.Duration) {
+ claims := hostwrite.Enabled(getenv)
+ return claims, systemGateWaitFor(claims)
+}
+
+// systemGateWaitFor is the gate's window: systemClaimsGateWait with
+// CARLOS_HOST_CLAIM on, systemGateWait without. The install's gate and the
+// rollback's gate both use it, so a claims-on box can roll back to a
+// previous build that also needs more than 20 s to listen.
+func systemGateWaitFor(claims bool) time.Duration {
+ if claims {
+ return systemClaimsGateWait
+ }
+ return systemGateWait
+}
+
+// gateFailFast is what makes the claims window "wait while the same process
+// is still starting" rather than "wait 120 s for anything". After a failed
+// check it returns an error, ending the gate before its deadline, when the
+// edge unit reports a failed or inactive state (from the is-active the
+// check just made) or NRestarts above 0 (from the same systemctl show the
+// soak reads). A build that crashes rolls back in seconds; one that is
+// still starting, active with no restart, gets the whole window. A show
+// that cannot be read says nothing either way, so the wait goes on to the
+// deadline. Claims-off boxes never call it.
+func (a *systemApplier) gateFailFast(state string, cause error) error {
+ switch state {
+ case "failed", "inactive":
+ return fmt.Errorf("%w (failing fast: %s is %s)", cause, systemEdgeUnit, state)
+ }
+ rs, err := a.edgeRunState()
+ if err != nil {
+ return nil
+ }
+ if err := rs.steady(); err != nil {
+ return fmt.Errorf("%w (failing fast: %v)", cause, err)
+ }
+ return nil
+}
+
// edgeRunState is the edge's NRestarts and ActiveEnterTimestampMonotonic,
// in that order — the order `show -p` is asked for them.
type edgeRunState struct{ restarts, activeEnter string }
@@ -1542,19 +1624,29 @@ func (a *systemApplier) edgeRunState() (edgeRunState, error) {
}
func (a *systemApplier) gateOnce(ctx context.Context, target string) error {
- if out, err := a.ops.systemctl("is-active", systemEdgeUnit); err != nil || strings.TrimSpace(out) != "active" {
+ _, err := a.gateCheck(ctx, target)
+ return err
+}
+
+// gateCheck is gateOnce that also returns what is-active said about the
+// edge, trimmed, for gateFailFast. The checks, their order and their
+// errors are gateOnce's.
+func (a *systemApplier) gateCheck(ctx context.Context, target string) (string, error) {
+ out, err := a.ops.systemctl("is-active", systemEdgeUnit)
+ state := strings.TrimSpace(out)
+ if err != nil || state != "active" {
if err != nil {
- return fmt.Errorf("%s is %s: %w", systemEdgeUnit, strings.TrimSpace(out), err)
+ return state, fmt.Errorf("%s is %s: %w", systemEdgeUnit, state, err)
}
- return fmt.Errorf("%s is %s", systemEdgeUnit, strings.TrimSpace(out))
+ return state, fmt.Errorf("%s is %s", systemEdgeUnit, state)
}
if err := a.ops.probeEdge(); err != nil {
- return fmt.Errorf("edge probe: %w", err)
+ return state, fmt.Errorf("edge probe: %w", err)
}
if got := a.onDiskBuild(ctx); got != target {
- return fmt.Errorf("installed binary reports %q, not %s", got, target)
+ return state, fmt.Errorf("installed binary reports %q, not %s", got, target)
}
- return nil
+ return state, nil
}
// arm enables every unit in the plan and starts the ones the plan says to
diff --git a/internal/host/systemapply_claimsgate_test.go b/internal/host/systemapply_claimsgate_test.go
new file mode 100644
index 000000000..e4de9a444
--- /dev/null
+++ b/internal/host/systemapply_claimsgate_test.go
@@ -0,0 +1,240 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "os"
+ "strings"
+ "testing"
+ "time"
+)
+
+// The update gate on a claims-on box (CARLOS_HOST_CLAIM). There the edge's
+// listener binds only after the agent's whole claim startup, about 22 s
+// after the restart, so a fixed 20 s window rolled every update back and
+// the rollback too. With claims on the window is 120 s and the gate fails
+// fast on a crash; with claims off it is exactly the 20 s gate it was.
+
+// slowListener makes the fake edge refuse the probe for listenAfter after
+// every restart, as a claims-on edge does while the claim assembly runs.
+// It returns the times at which each restart was issued.
+func slowListener(f *applyFixture, listenAfter time.Duration) *[]time.Time {
+ var restarts []time.Time
+ orig := f.a.ops.systemctl
+ f.a.ops.systemctl = func(args ...string) (string, error) {
+ if args[0] == "restart" && len(args) > 1 && args[1] == systemEdgeUnit {
+ restarts = append(restarts, f.now)
+ }
+ return orig(args...)
+ }
+ f.a.ops.probeEdge = func() error {
+ if len(restarts) == 0 || f.now.Sub(restarts[len(restarts)-1]) < listenAfter {
+ return errors.New("dial tcp 127.0.0.1:80: connect: connection refused")
+ }
+ return nil
+ }
+ return &restarts
+}
+
+func TestSystemGateWaitIsKeyedOnClaimsMode(t *testing.T) {
+ if got := systemGateWaitFor(false); got != 20*time.Second {
+ t.Fatalf("claims-off gate window = %s, want the 20 s it always was", got)
+ }
+ if got := systemGateWaitFor(true); got != 120*time.Second {
+ t.Fatalf("claims-on gate window = %s, want 120 s", got)
+ }
+}
+
+func TestSystemApplyReadsClaimsModeFromTheUnitEnvironment(t *testing.T) {
+ for _, tc := range []struct {
+ value string
+ claims bool
+ wait time.Duration
+ }{{"", false, 20 * time.Second}, {" ", false, 20 * time.Second}, {"1", true, 120 * time.Second}, {"on", true, 120 * time.Second}} {
+ claims, wait := systemApplyGate(func(k string) string {
+ if k == "CARLOS_HOST_CLAIM" {
+ return tc.value
+ }
+ return ""
+ })
+ if claims != tc.claims || wait != tc.wait {
+ t.Fatalf("CARLOS_HOST_CLAIM=%q: claims=%t wait=%s, want %t %s", tc.value, claims, wait, tc.claims, tc.wait)
+ }
+ }
+}
+
+func TestClaimsOffGateIsUnchangedByAnEdgeThatListensAt22s(t *testing.T) {
+ // Ruling 6: a claims-off box sees no change. The same 22 s edge rolls
+ // back, and its 22 s previous build fails its gate too, as on main.
+ f, req := applyHappy(t)
+ slowListener(f, 22*time.Second)
+ if err := f.a.Run(context.Background(), false); err == nil {
+ t.Fatal("a claims-off 20 s gate passed an edge that listened at 22 s")
+ }
+ r := f.result()
+ if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) {
+ t.Fatalf("result = %+v, want rollback-failed as on main", r)
+ }
+}
+
+func TestClaimsOffGateWaitsItsFullWindowThroughACrash(t *testing.T) {
+ // Claims off never fails fast: a crash-restarting edge still gets the
+ // full 20 s, and the gate's systemctl calls are only is-active.
+ f := newApplyFixture(t, "stable", "v1")
+ f.edgeUp = false
+ f.nrestarts = "3"
+ start := f.now
+ if err := f.a.gate(context.Background(), "v1"); err == nil {
+ t.Fatal("gate passed a failed edge")
+ }
+ if waited := f.now.Sub(start); waited != 20*time.Second {
+ t.Fatalf("claims-off gate gave up after %s, want exactly 20 s", waited)
+ }
+ for _, c := range f.calls {
+ if strings.HasPrefix(c, "systemctl show") {
+ t.Fatalf("claims-off gate read %q inside its wait; it must not", c)
+ }
+ }
+}
+
+func TestClaimsOnGatePassesAnEdgeThatListensAt22s(t *testing.T) {
+ f, req := applyHappy(t)
+ f.a.claims = true
+ restarts := slowListener(f, 22*time.Second)
+ if err := f.a.Run(context.Background(), false); err != nil {
+ t.Fatalf("claims-on gate failed a still-starting edge: %v", err)
+ }
+ r := f.result()
+ if r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v2", req.PromotedAt) {
+ t.Fatalf("result = %+v", r)
+ }
+ if len(*restarts) != 1 {
+ t.Fatalf("edge restarted %d times, want 1 (no rollback)", len(*restarts))
+ }
+}
+
+func TestClaimsOnGateWaitsUpTo120s(t *testing.T) {
+ f := newApplyFixture(t, "stable", "v1")
+ f.a.claims = true
+ f.probeErr = errors.New("connection refused")
+ start := f.now
+ err := f.a.gate(context.Background(), "v1")
+ if err == nil || !strings.Contains(err.Error(), "connection refused") {
+ t.Fatalf("gate = %v, want the probe's error at the deadline", err)
+ }
+ if waited := f.now.Sub(start); waited != 120*time.Second {
+ t.Fatalf("claims-on gate gave up after %s, want 120 s", waited)
+ }
+}
+
+func TestClaimsOnGateFailsFastOnACrashRestart(t *testing.T) {
+ f := newApplyFixture(t, "stable", "v1")
+ f.a.claims = true
+ f.probeErr = errors.New("connection refused")
+ f.nrestarts = "1"
+ start := f.now
+ err := f.a.gate(context.Background(), "v1")
+ if err == nil || !strings.Contains(err.Error(), "NRestarts=1") {
+ t.Fatalf("gate = %v, want a fail-fast on NRestarts", err)
+ }
+ if waited := f.now.Sub(start); waited != 0 {
+ t.Fatalf("claims-on gate waited %s on a crash-restarted edge, want no wait", waited)
+ }
+}
+
+func TestClaimsOnGateFailsFastOnAFailedOrInactiveEdge(t *testing.T) {
+ for _, state := range []string{"failed", "inactive"} {
+ t.Run(state, func(t *testing.T) {
+ f := newApplyFixture(t, "stable", "v1")
+ f.a.claims = true
+ orig := f.a.ops.systemctl
+ f.a.ops.systemctl = func(args ...string) (string, error) {
+ if args[0] == "is-active" {
+ return state + "\n", errors.New("exit status 3")
+ }
+ return orig(args...)
+ }
+ start := f.now
+ err := f.a.gate(context.Background(), "v1")
+ if err == nil || !strings.Contains(err.Error(), "failing fast: carlos-edge is "+state) {
+ t.Fatalf("gate = %v, want a fail-fast on %s", err, state)
+ }
+ if waited := f.now.Sub(start); waited != 0 {
+ t.Fatalf("claims-on gate waited %s on a %s edge", waited, state)
+ }
+ })
+ }
+}
+
+func TestClaimsOnGateKeepsWaitingWhileActivating(t *testing.T) {
+ // "activating" with no restart is a process still starting, not a
+ // crash: no fail-fast, the gate passes once it answers.
+ f := newApplyFixture(t, "stable", "v1")
+ f.a.claims = true
+ start := f.now
+ orig := f.a.ops.systemctl
+ f.a.ops.systemctl = func(args ...string) (string, error) {
+ if args[0] == "is-active" && f.now.Sub(start) < 30*time.Second {
+ return "activating\n", errors.New("exit status 3")
+ }
+ return orig(args...)
+ }
+ if err := f.a.gate(context.Background(), "v1"); err != nil {
+ t.Fatalf("gate = %v, want a pass once the edge is active", err)
+ }
+}
+
+func TestClaimsOnRollbackGateUsesTheSameRule(t *testing.T) {
+ // v2 crashes on start; v1, the previous build, needs 22 s to listen.
+ // The install's gate fails fast, and the rollback's gate gives v1 the
+ // claims window, so the rollback succeeds instead of rollback-failed.
+ f, req := applyHappy(t)
+ f.a.claims = true
+ restarts := slowListener(f, 22*time.Second)
+ orig := f.a.ops.systemctl
+ f.a.ops.systemctl = func(args ...string) (string, error) {
+ if args[0] == "is-active" {
+ if b, _ := os.ReadFile(systemBinaryPath(f.a.optDir)); strings.Contains(string(b), "carlos v2") {
+ return "failed\n", errors.New("exit status 3")
+ }
+ }
+ return orig(args...)
+ }
+ err := f.a.Run(context.Background(), false)
+ if err == nil {
+ t.Fatal("a failed gate must exit non-zero")
+ }
+ r := f.result()
+ if r == nil || r.Outcome != systemOutcomeRolledBack || !r.matches("stable", "v2", req.PromotedAt) {
+ t.Fatalf("result = %+v, want rolled-back", r)
+ }
+ if !strings.Contains(r.Reason, "failing fast") {
+ t.Fatalf("reason = %q, want the install gate's fail-fast", r.Reason)
+ }
+ if len(*restarts) != 2 {
+ t.Fatalf("edge restarted %d times, want install + rollback", len(*restarts))
+ }
+ if d := (*restarts)[1].Sub((*restarts)[0]); d > 5*time.Second {
+ t.Fatalf("install gate took %s to give up on a failed edge, want fail-fast", d)
+ }
+}
+
+func TestClaimsOnRollbackGateFailsFastToo(t *testing.T) {
+ // Both builds crash: the rollback gate fails fast as the install's did,
+ // rather than waiting 120 s on a dead previous build.
+ f, req := applyHappy(t)
+ f.a.claims = true
+ f.nrestarts = "2"
+ f.probeErr = errors.New("connection refused")
+ start := f.now
+ if err := f.a.Run(context.Background(), false); err == nil {
+ t.Fatal("a failed gate must exit non-zero")
+ }
+ r := f.result()
+ if r == nil || r.Outcome != systemOutcomeRollbackFailed || !r.matches("stable", "v2", req.PromotedAt) {
+ t.Fatalf("result = %+v, want rollback-failed", r)
+ }
+ if waited := f.now.Sub(start); waited >= 20*time.Second {
+ t.Fatalf("install and rollback gates took %s between them, want fail-fast on both", waited)
+ }
+}
diff --git a/internal/host/systemparticipation.go b/internal/host/systemparticipation.go
index 268a66731..a067c70ae 100644
--- a/internal/host/systemparticipation.go
+++ b/internal/host/systemparticipation.go
@@ -146,16 +146,31 @@ func (a *systemApplier) clearLegacyUpdateBegun() error {
// got past its first step. A kept journal (a rollback that could not
// finish) keeps the marker with it. Run with or without CARLOS_REGION_HA,
// so a marker left behind after the flag is turned off still goes.
-func (a *systemApplier) settleUpdateBegun(ctx context.Context) {
+//
+// On a census box the settlement goes through settleWithoutJournal (Astra
+// round 3): a legacy marker with no update record and no journal has
+// clearUpdateBegun begin root's update record, and a census refusal there
+// withdrew it and left acquisitions open while the error was only logged.
+// Now a census refusal comes back as errSettlementDeferred with the
+// "recover-legacy" barrier held, and a barrier that could not be written as
+// errAcquisitionBarrier; the caller decides what each means for the run.
+// Every other failure is logged and swallowed, as on main.
+func (a *systemApplier) settleUpdateBegun(ctx context.Context) error {
if active, err := a.activeFloor(); err != nil || active {
- return
+ return nil
}
if j, err := readSystemJournal(a.optDir); err != nil || j != nil {
- return
+ return nil
+ }
+ err := a.settleWithoutJournal(ctx, "recover-legacy", a.clearUpdateBegun(ctx))
+ if err == nil {
+ return nil
}
- if err := a.clearUpdateBegun(ctx); err != nil {
- a.ops.logf("system apply: %v", err)
+ if errors.Is(err, errSettlementDeferred) || errors.Is(err, errAcquisitionBarrier) {
+ return err
}
+ a.ops.logf("system apply: %v", err)
+ return nil
}
// closeGuardRollback is closeGuardJournal for the backward paths: rollback
@@ -225,6 +240,9 @@ func (a *systemApplier) checkUnarmedParticipationRecovery(ctx context.Context) e
}
if recover {
if err := a.controllersRecovered(ctx); err != nil {
+ if j == nil {
+ err = a.settleWithoutJournal(ctx, "recover-system", err)
+ }
return err
}
}
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go
index 495304a7c..0aad151ea 100644
--- a/internal/provision/plan14_root_reads_test.go
+++ b/internal/provision/plan14_root_reads_test.go
@@ -1118,6 +1118,27 @@ func TestPlan14RootReadCensus(t *testing.T) {
t.Fatal(err)
}
seal = "33a8ff46dd9fcf85664c42c42eaf01585af6c92806cb7830fa8f30fd0950c430"
+ // Census overlay claims-on-gate-recover (a claims-on box's update gate
+ // waits 120 s and fails fast; carlos-system-recover defers a refused
+ // census with no journal open, accepts a service still running the
+ // signed current or previous router build it was renamed over, retries
+ // task-level refusals for at most 20 s and names each refusal's check,
+ // unit, cgroup and exe), loaded last. PROPOSED-claims-on-gate-recover,
+ // to be asked of the census owner on #280
+ // (docs/superpowers/notes/census-proposal-claims-on-gate-recover.md).
+ // Changed authority units re-pinned at their class and task; the
+ // service reader's reads move into three new units at its class (E2);
+ // the deleted-exe image proof is a new root read at F,J4, as the
+ // guard's own image proof. No legacy row; the seal stands.
+ if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-claims-on-gate-recover.tsv"), &inv, true); err != nil {
+ t.Fatal(err)
+ }
+ // Census overlay claims-on-gate-recover-merge-main (the branch merged
+ // with main at 521c6ee9), loaded last: envelope only, internal/host
+ // re-pinned over both branches' code. The seal stands.
+ if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-claims-on-gate-recover-merge-main.tsv"), &inv, true); err != nil {
+ t.Fatal(err)
+ }
errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
errs = append(errs, checkRootReadDeclarations(got, inv)...)
if len(errs) > 0 {
diff --git a/internal/provision/testdata/plan14-root-reads-claims-on-gate-recover-merge-main.tsv b/internal/provision/testdata/plan14-root-reads-claims-on-gate-recover-merge-main.tsv
new file mode 100644
index 000000000..e5f2bb08f
--- /dev/null
+++ b/internal/provision/testdata/plan14-root-reads-claims-on-gate-recover-merge-main.tsv
@@ -0,0 +1,2 @@
+# Census overlay claims-on-gate-recover-merge-main (claims-on-gate-recover merged with main at 521c6ee9: changelog-v0.28.7, securelisten-337, console-invariant-parallel and the rest since 3d444a1b): envelope only. internal/host's envelope is re-pinned over both branches' code; main's overlays load before this branch's, unchanged. No unit, class or read changes; the seal stands at main's.
+@coverage internal/host fe12f161972b1f74afbb4850717f345cc9bf94ad8244bb6d571d59c2ae70841a
diff --git a/internal/provision/testdata/plan14-root-reads-claims-on-gate-recover.tsv b/internal/provision/testdata/plan14-root-reads-claims-on-gate-recover.tsv
new file mode 100644
index 000000000..0ce4b12c0
--- /dev/null
+++ b/internal/provision/testdata/plan14-root-reads-claims-on-gate-recover.tsv
@@ -0,0 +1,33 @@
+# Census overlay claims-on-gate-recover (the claims-on update gate and carlos-system-recover's generation census, on main 3d444a1b), loaded LAST, after scan-demand-revival. PROPOSED, not ruled: rev3=PROPOSED-claims-on-gate-recover, to be asked of the census owner on #280 (docs/superpowers/notes/census-proposal-claims-on-gate-recover.md).
+# (a) RunSystemApply re-pinned: the gate window follows CARLOS_HOST_CLAIM. The gate itself (systemApplier.gate, gateCheck, gateFailFast) has no root read and no row.
+# (b) checkGuardRecovery, Run and checkUnarmedParticipationRecovery re-pinned for the no-journal deferral; generationKernel.absent and .task re-pinned for typed task refusals; generationCensusRetry re-pinned for the bounded backoff; generationServiceReader re-pinned as a binding, its reads moved into three new E2 units at its class; generationServiceEnv.deletedProgram is a new root read at F,J4 like the guard's own image proof.
+# Every changed unit is authority: no legacy row, no frozen declaration, so the seal stands. internal/host's envelope is re-pinned.
+# Fix round 1 (Astra): checkGuardRecovery, checkUnarmedParticipationRecovery, generationKernel.task, generationCensusRetry, generationServiceReader, generationServiceEnv.reader and .deletedProgram re-pinned; productionGenerationKernelCensus, productionGenerationAbsence, type generationKernel and generationRetiredController re-pinned at main's class (E2); generationServiceEnv.recheck, generationExeIdentity and generationContextReader.Read new at F,J4 with the image proof they serve. Still no legacy row; the seal stands.
+# Fix round 2 (Astra round 2): checkGuardRecovery, systemApplier.Run and checkUnarmedParticipationRecovery re-pinned for the post-journal deferral and the acquisition barrier; Ledger.HoldUpdate new at its siblings' class (D); the signed-image proof moves under the census deadline (readRootFloorBinary re-pinned as a no-read wrapper, readRootFloorBinaryContext at its class with p3's reads, generationContextReader.Read moved beside it); generationServiceReader, .reader and .deletedProgram bases corrected (sampled identity checks, not execution exclusion; the deadline's coverage and its gaps). internal/rootparticipation's envelope re-pinned. Still no legacy row; the seal stands.
+# Fix round 3 (Astra round 3): systemApplier.Run re-pinned and settleUpdateBegun pinned as a declaration for the legacy marker's settlement through settleWithoutJournal (barrier held, barrier failure fatal); generationKernel.task, type generationKernel, generationServiceEnv.recheck and generationCensusRetry bases qualified (sampled identity checks at observation points; cooperative cancellation at checkpoints, with the uncancelled work named). Still no legacy row; the seal stands.
+# Fix round 4 (Astra round 4): settleWithoutJournal classifies a failed ParticipationRequired check as an acquisition-barrier failure (both errors kept), so the legacy marker's settlement cannot log it and exit 0; no row changes (settleWithoutJournal has no read of its own); internal/host envelope re-pinned. The seal stands.
+internal/host/guardtransaction.go systemApplier.checkGuardRecovery authority D rev3=PROPOSED-claims-on-gate-recover;identity=root system update/recovery uid 0 (carlos-system-update.service, carlos-system-recover.service);basis=re-pinned at its class and task; its reads are unchanged (activation and marker records). Fix round 1: every no-journal settlement (the controllers census, the completion or withdrawal, and the census again inside EndParticipation, in the marker, no-marker and already-completed branches) goes through settleWithoutJournal, which defers only a result that is a census refusal and nothing else (onlyCensusRefusal), on a census box: Run under --recover then exits 0 with the reservation, marker and update record kept. An image that does not verify, malformed or foreign state, a live or unreadable controller and a failed durable write fail as on main; so do the journal-open branch, the apply and no-census boxes (ruling 6). An unfinished transaction's marker is removed inside the settlement, after its census. Claims-on or retained root census uses root exclusion; claims-off with no census preserves main participation and recovery behavior. TestRecoverDefersACensusRefusalWithNoJournalOpen, TestRecoverDefersACensusRefusalOnlyAtTheEndOfParticipation, TestRecoverDefersACensusRefusalOfAnUnfinishedTransaction, TestRecoverDefersACensusRefusalOfAnAlreadyCompletedTransaction, TestRecoverStillFailsSettlementErrorsThatAreNotCensusRefusals, TestRuling6NoCensusRecoverIsUnchanged. Fix round 2: once the journal is durably removed, closeGuardJournal's settlement (endGuard, clearUpdateBegun) also goes through settleWithoutJournal, forward and backward, guarded and unarmed; before it defers a transaction with no update record, settleWithoutJournal holds acquisitions refused under the transaction lock with Ledger.HoldUpdate (root's update record kept, or written under the transaction's id, with no census of its own); a barrier that cannot be written leaves the refusal fatal; an already-completed transaction with no record left replays at once with no census. TestRecoverDefersARefusalAfterTheJournalIsRemoved, TestRecoverStillFailsANonCensusErrorAfterTheJournalIsRemoved, TestGuardedRollbackCloseDefersARefusalAfterTheJournalIsRemoved, TestRecoverHoldsAcquisitionsWhenNoUpdateRecordWasBegun, TestRecoverReplaysASettledTransactionWithoutACensus, TestRecoverDoesNotDeferWithoutABarrier. file:ReadFile@433,file:ReadFile@660 98821263c0f79caf845c2627e4ee5e0535e7e83b5a6211b32d23c71837ad09e8
+internal/host/systemapply.go systemApplier.Run authority D,F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root apply/recovery uid 0;basis=re-pinned at its class and task; its lock-file read is unchanged. A checkGuardRecovery error wrapping errSettlementDeferred under --recover is logged and not fatal, and the end-of-run settlement is not retried after a deferral; every other refusal returns as before. root-input;kept=D,F;scope=protected-decision-only. TestRecoverDefersACensusRefusalWithNoJournalOpen, TestApplyStillFailsACensusRefusalWithNoJournalOpen. Fix round 2: a deferral that recoverIfNeeded returns after the journal was removed is consumed the same way, with no fatal error kept and no settlement retried. TestRecoverDefersARefusalAfterTheJournalIsRemoved. Fix round 3: the end-of-run marker settlement (settleMarker, settleUpdateBegun) also goes through settleWithoutJournal, so a legacy update-begun marker with no update record and no journal defers a census refusal with the recover-legacy barrier held; a barrier that cannot be written joins the run's error in every mode; every other marker failure is logged as on main. TestRecoverHoldsAcquisitionsForALegacyMarker, TestRecoverFailsALegacyMarkerWhoseBarrierCannotBeWritten. file:OpenFile@483 7d9c6997336322ee4c4a9feacc8312c4c7b6175393a26932246a32ce77fe5c55
+internal/host/systemparticipation.go systemApplier.settleUpdateBegun authority D rev3=PROPOSED-claims-on-gate-recover;identity=root apply/recovery uid 0;basis=pinned as a declaration (no read of its own) for fix round 3: the end-of-run marker settlement on a census box goes through settleWithoutJournal, returning a census refusal as errSettlementDeferred with the recover-legacy barrier held, a barrier that could not be written as errAcquisitionBarrier, and logging everything else as on main; clearUpdateBegun's reads are unchanged. TestRecoverHoldsAcquisitionsForALegacyMarker, TestRecoverFailsALegacyMarkerWhoseBarrierCannotBeWritten. b36cd2040f531d903a50300dd040efc8f2cdc1cadf00626365b539cdd4a47a3f
+internal/host/systemapply.go RunSystemApply authority D,F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root apply/recovery uid 0;basis=re-pinned at its class and task, no read of its own: reads CARLOS_HOST_CLAIM from the unit environment through systemApplyGate (hostwrite.Enabled, as the agent does) to pick the update gate window, 120 s with fail-fast claims-on and 20 s unchanged claims-off. Registers local verifier and invokes census-gated release wrapper; absent census executes main apply directly even with leftover release state. TestSystemApplyReadsClaimsModeFromTheUnitEnvironment, TestClaimsOffGateIsUnchangedByAnEdgeThatListensAt22s. d563475f2ac28b1d1bc0bae86fa326ccaae3df1efb71928f7efd69a644a4e960
+internal/host/systemparticipation.go systemApplier.checkUnarmedParticipationRecovery authority G,F rev3=PROPOSED-claims-on-gate-recover;identity=root system update/recovery uid 0;basis=re-pinned at its class and task, no read of its own: with no journal open a failed controllers settlement goes through settleWithoutJournal, so only a census refusal defers (transaction and update record kept, recover exits 0); with a journal open, and for every other failure, it refuses as before. Pending acquisition triggers production controller settlement even without an update journal; normal ledger admission stays refused until verified recovery clears the marker. TestRecoverDefersACensusRefusalWithoutAnArmedFloor, TestRecoverWithoutAnArmedFloorKeepsALateRefusalNonFatal. Fix round 2: the unarmed path's post-journal settlement defers the same way, and holds acquisitions refused first when no update record exists. 48f3d01af2c4badeff7ba4526304121aa75a2efb37032880f7de8914369d9c6b
+internal/host/rootgeneration_kernel_linux.go generationKernel.absent authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task; its reads are unchanged. A task refusal is now typed (generationTaskRefusal, same text: the /proc path and the cause) so the census retry can tell task-level refusals from structural ones; bounded task and persistent generation inspection retains boot and process start checks. file:Readlink@312 b758554c182902d80a255122404d8418ec78a9d49ed32c78b9116493ab137b5a
+internal/host/rootgeneration_kernel_linux.go generationKernel.task authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task; its reads are unchanged. A start time that moves during inspection refuses with errGenerationTaskReplaced (still os.ErrPermission). Fix round 1: a task the service lookup trusted is rechecked against the image it was trusted for (sampled identity checks, not execution exclusion) before each use of that trust (the namespace compare, the mount check) and at the end of the inspection: a replacement visible at a recheck refuses; one after the final open, or undone between rechecks, is not seen until the next pass, and a kernel with a lookup but no recheck refuses every trusted task. Every accepted service, including a deleted-exe task, still has exe, cwd, root, descriptors, maps and mounts checked against retired generations and its start rechecked. TestRecoverCensusRefusesATaskThatExecsAfterItsImageWasProven, TestRecoverCensusNeedsABindingForATrustedTask, TestRecoverCensusStillChecksReferencesOfAnAcceptedDeletedExe, TestPlan14GenerationServiceReferences. file:Readlink@1657 a6e618451bb716139bf3c97d01ed82c00e2d1f450937e3bf325c6589942a0c71
+internal/host/rootgeneration_census_linux.go generationCensusRetry authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task, no read of its own: runs whole independent census passes through generationCensusRetryFor. Every pass runs under one 20 s deadline from the census's start, as cooperative cancellation at the implemented checkpoints plus rejection of late results: systemctl show through exec.CommandContext, the deleted-image hash between reads and, since fix round 2, the signed-image proof (readRootFloorBinaryContext: the hash of <opt>/carlos and <opt>/carlos.prev between reads, and the identity-cache scan before each identity). A pass that ends past the deadline refuses even if it succeeded, and none starts after it. Between checkpoints the work is not cancelled: the opens and metadata operations, the identity cache's directory enumeration (Names, a ReadDir), a whole identity record's read (io.ReadAll), a systemctl show already started, one Read of an image, and the versions-tree walk and mount checks before the first pass; the boot-delay figures in the census note are estimates built on this, not bounds. Task-level refusals a later pass can clear (exited, replaced, refused by the service lookup) retry with backoff from 250 ms inside that deadline, then the last refusal is returned unchanged; other missing-process errors retry at most three times as before; reference, namespace, mount and structural refusals refuse immediately; no partial census is accepted. TestRecoverCensusDeadlineCountsInspectionTime, TestRecoverCensusDeadlineReachesTheInspection, TestRecoverCensusRetriesATransientRefusalWithinTheWindow, TestRecoverCensusRetriesOnlyTaskLevelTransients, TestPlan14GenerationCensusRetriesWholeObservation. f779c207011954e9ca7a8c98b83ac11e6d6926b3a480eaf0547b22507a23daf5
+internal/host/rootgeneration_services_linux.go generationServiceReader authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=re-pinned at its class and task, no read of its own: binds the production generationServiceEnv (/proc, /opt/carlos, owner uid 0, systemctl show, trusted unit files, readRootFloorBinaryContext, the signed-image proof under the census deadline since fix round 2) and returns its lookup, classify and recheck; the reads moved into generationServiceEnv.reader, generationServiceShow and generationServiceTrust unchanged. ad2cc68e7792e9c0af9242531a5cfc642d59d9436a46588ebc91eba27635f0f9
+internal/host/rootgeneration_services_linux.go generationServiceEnv.reader authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=new unit, generationServiceReader's body at its class: classifies independently verified platform and system services by trusted unit files, systemd main PID, cgroup and protected executable; only edge main threads may use its delegated supervise subgroup; exact guard-exec edge verifies guard and selected current against signed protected identities. The exe Readlink moves before the cgroup checks for diagnostics only (a failure still refuses at its old place); every refusal names check, unit, cgroup and exe and wraps its cause. A non-root task, or a root task of a unit with a selected image (the guarded edge), whose exe reads <path> (deleted) goes to deletedProgram instead of the path walk; root tasks of units with no selected image keep the walk. Fix round 1: a task accepted on a proven deleted image keeps the hashed descriptor open, with its device, inode, mount ID and file handle recorded, and is rechecked against it by sampled identity checks at observation points (before each use of the trust and at the end of the inspection): a replacement visible at a recheck refuses; an exec after the final /proc/<pid>/exe open is not seen until the next pass, the race main's live-exe path (selected-program and non-root services) already has, so this is not execution exclusion. The binding applies only to deleted-image acceptance; live-image rechecks return at once. The lookup is built on each census pass's deadline context, which systemctl show, the image hash and (fix round 2) the signed-image proof obey. TestRecoverCensusRefusalsNameCheckUnitCgroupAndExe, TestRecoverCensusAcceptsTheEdgesDeletedExeInSupervise. file:Readlink@1270 96a04fe5667537f9193daef824964e9b4823266c5dd8ced3cf657b2b4b9014e3
+internal/host/rootgeneration_services_linux.go generationServiceShow authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=new unit, the systemctl show of ControlGroup, MainPID, ExecStart, FragmentPath and DropInPaths moved out of generationServiceReader unchanged, at its class. file:CommandContext@103 26d82849d8e6d22eb35b0d9befd0b5c02d81cec9dc3bfaf4b684504c16c4d0cd
+internal/host/rootgeneration_services_linux.go generationServiceTrust authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=new unit, the unit-file EvalSymlinks and rootstate.OpenTrustedFile check moved out of generationServiceReader unchanged, at its class. file:EvalSymlinks@63 07fe0cf4c6a8b17eaa223ec37b44818152ac1e70c8fe3832bedcb99c61cf24a1
+internal/host/rootgeneration_services_linux.go generationServiceEnv.deletedProgram authority F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0 (carlos-system-recover, carlos-system-update, instance-control start, guard-exec prelude);basis=NEW root read, classified as guard-exec's equivalent (hostfloor OpenProtected, Disk.VerifyFile and ExecutingHash, F,J4 root-input) because it is the same proof on the same kind of input: it opens a service task's running image through /proc/<pid>/exe, requires a regular file owned by root with no links left and no group or other write, hashes its bytes (at most 512 MiB) and accepts only a digest equal to the signed router build of <opt>/carlos or <opt>/carlos.prev as readRootFloorBinaryContext proves it from the local identity cache, under the census deadline (fix round 2). It applies to a non-root task, and to a root task of a unit with a selected image (the guarded edge, whose selected program is <opt>/carlos), when the unit selects <opt>/carlos and the exe link reads <opt>/carlos (deleted); a root task of a unit with no selected image keeps main's path walk, and anything else refuses as before. The hashed descriptor stays open and the task is rechecked against it at observation points (generationServiceEnv.recheck): sampled identity checks, not execution exclusion; an exec after the final open shares main's live-exe race. Acceptance only lets the census reach the reference checks. Fix round 1: on success it returns the hashed descriptor, open, to bind the task to it (recheck), and the hash stops when the census deadline passes (generationContextReader, now in rootmovesfloor_refresh.go with the proof). TestRecoverCensusAcceptsADeletedExeOfTheCurrentOrPreviousBuild, TestRecoverCensusRefusesOtherDeletedExes, TestRecoverCensusRefusesADeletedExeNotOwnedByRoot. file:Fstat@481,file:Open@323 9b912bec1e9cecf3caff27e8742b84501b82d4b7c0b29f6aa96843d8c43952b7
+internal/host/rootgeneration_census_linux.go productionGenerationKernelCensus authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task, no read of its own changed: every refusal is now typed generationCensusRefusal, so a no-journal recovery can defer it and nothing else, and each pass builds the service lookup on the census deadline context and wires its recheck. Checks protected staged tree and hardlink policy; actual systemd services pass only through unit, cgroup and executable verification while recorded live tenants use independent containment. TestProductionGenerationCensusRefusalsAreTyped. 0da7bd4e8f5c68bd7f30c3c8b3650eb32db5ff54995be293b0f0469d27cc515b
+internal/host/rootgeneration_kernel_linux.go productionGenerationAbsence authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task, no read of its own: wires the service lookup's recheck beside its classify. Checks recorded launch targets and controller absence then inspects all tasks with independently verified service classification. 189c4b2c5d7d47c91814d2788fc04150eb6c0145ca7b76b7fcb7ecde81028ffb
+internal/host/rootgeneration_kernel_linux.go type generationKernel authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task: gains the recheck field that rechecks a trusted task against its proven image at observation points (sampled identity checks). Root kernel generation verifier carries service classification and exact retained containment; all observations are independently checked before certification. 40aa7e14fbbc5da15d951f0de1bd93e2f1f2fb3b60591592e264b211a1029e6f
+internal/host/rootgeneration_retired_linux.go generationRetiredController authority E2 rev3=PROPOSED-claims-on-gate-recover;identity=root claim/adopt/update controller uid 0;basis=re-pinned at its class and task, no read of its own: wires the service lookup's recheck beside its classify. For an exact retired isolated installation, an old controller may still serve another host only after its boot/start identity and all thread references to this target generation are independently checked. d074043a9cfcfb1d4317388dab298915c38fd5b3b7ffcc8dedf4824c206bce0d
+internal/host/rootgeneration_services_linux.go generationServiceEnv.recheck authority F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=new root read at its caller deletedProgram's class (the guard's image proof, F,J4): re-opens a trusted task's /proc/<pid>/exe at each use of the trust and at the end of its inspection, and requires the very inode that was hashed (generationExeIdentity: device, inode, mount ID, file handle), still root-owned, unlinked and not group or other writable; it does not hash again, the hashed descriptor stays open and the kernel refuses writes to an executing image (ETXTBSY). A replacement visible at an observation point refuses, transient; a replacement after the final open, or one undone between observations, is not seen until the next pass, the race main's live-exe path shares. Final releases the descriptor. TestRecoverCensusRefusesATaskThatExecsAfterItsImageWasProven, TestRecoverCensusDeletedExeOfARootTask. file:Fstat@589,file:Open@399 3fa2ccce10f6389a4d177aa0420f9b8c86aa25b2068934aa2e5059318a1edcfd
+internal/host/rootgeneration_services_linux.go generationExeIdentity authority F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=new unit at its callers' class (deletedProgram's binding and recheck, F,J4): Fstat, Statx(STATX_MNT_ID) and name_to_handle_at of an already open image descriptor, giving the device, inode, mount ID and file handle that bind a task to its proven image; a filesystem without handles gives none (EOPNOTSUPP). rootstate.ObjectIdentity is not used because it resolves the mount in root's own mount table, which a service in a private mount namespace does not share. file:Fstat@97,file:Statx@207 90589ce72ed160c94a5d155157d3f93ea1e9b4850992f2aacfc63dd88e29a828
+internal/host/rootmovesfloor_refresh.go generationContextReader.Read authority F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root generation kernel reader uid 0;basis=moved beside the signed-image proof in fix round 2, same class as its callers deletedProgram and readRootFloorBinaryContext (F,J4): the hash's reader over an already open, already checked image (the deleted image, <opt>/carlos or <opt>/carlos.prev); it stops once the census deadline passes and opens nothing. TestRecoverCensusDeadlineReachesTheInspection. file:Read@127,file:Read@33 d20afc659f08ab69244381b3b677a44e51467ad813b188aeaad1d00c278cf53b
+internal/host/rootmovesfloor_refresh.go readRootFloorBinary authority F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root adopt refresh, root system update/recovery and root guard floor readers, uid 0;basis=re-pinned at its class and task, no read of its own since fix round 2: readRootFloorBinaryContext under context.Background(), for the callers that have no deadline; behaviour unchanged. 8ac4df055cc5f91436f65929bea964152009d120d98e6ee5069a63ed406242e6
+internal/host/rootmovesfloor_refresh.go readRootFloorBinaryContext authority F,J4 rev3=PROPOSED-claims-on-gate-recover;identity=root adopt refresh, root system update/recovery, root guard floor readers and the root generation kernel reader, uid 0;basis=readRootFloorBinary's body at its class (p3: verifies checked executable and signed system manifest; reads signed moves and seeds protocol numbers rather than version sidecar), now under a context: the image hash stops between reads (generationContextReader) and the identity-cache scan stops before each identity (rootFloorSignedIdentity) once the context ends; the reads are p3's. Fix round 2, for the census deadline. TestSignedBinaryProofStopsWhenTheCensusDeadlineEnds (the two helpers), TestRecoverCensusDeadlineReachesTheSignedBinaryProof (the lookup hands the deadline to the proof; the production plumbing from this function into the helpers is not witnessed by a non-root test). After CI: rootFloorDigest returns the raw sum and the hex encode is one single-valued write to Digest again, so the moves copy gate can derive the digest that reaches a seal's Reason (TestMovesSinksAreSentences, TestMovesStringsAreRecorded). file:Open@846,file:Read@1111,file:Stat@304 92c74898bfc43ac45d774ca766c5b4be474e58ffe60a8ac2737a32655240a304
+internal/rootparticipation/ledger.go Ledger.HoldUpdate authority D rev3=PROPOSED-claims-on-gate-recover;identity=root system recover uid 0 (settleWithoutJournal, under the transaction lock);basis=new unit at its siblings' class (Ledger.BeginUpdateWith, Ledger.EndUpdateWith, D): the recovery acquisition barrier. Under the same-ledger transaction exclusion it reads the recovery census and update.json and, when no acquisition is pending and no update record exists, writes root's update record under the deferring transaction's id, so Acquire stays refused while recover defers a census refusal; it runs no census of its own (EndUpdateWith, with its census, still removes the record). Fix round 2 (Astra finding 1b). TestHoldUpdateRefusesAcquisitionsWithoutACensus, TestHoldUpdateLeavesAPendingAcquisitionCensusAlone, TestRecoverHoldsAcquisitionsWhenNoUpdateRecordWasBegun, TestRecoverDoesNotDeferWithoutABarrier. file:Read@319 e5cfaa26ccf5241507b41920d64e1ccf7db2e6435d071a9226246fac1a346645
+@coverage internal/host d92e492a30973333e8c6df8a939a347dbb96a30a0e404327737f502232376268
+@coverage internal/rootparticipation 6c6d03dec45fe2fca86efc0fcb7615a4ab531c1da0a3a10adab36932d9d81a5a
diff --git a/internal/rootparticipation/holdupdate_test.go b/internal/rootparticipation/holdupdate_test.go
new file mode 100644
index 000000000..fd30aec17
--- /dev/null
+++ b/internal/rootparticipation/holdupdate_test.go
@@ -0,0 +1,68 @@
+package rootparticipation
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "testing"
+)
+
+func TestHoldUpdateRefusesAcquisitionsWithoutACensus(t *testing.T) {
+ l := censusFixture(t)
+ refused := errors.New("census refused")
+ l.VerifyLaunches = func(context.Context, uint64) error { return refused }
+ // BeginUpdate cannot hold: its census refuses and it withdraws the record.
+ if err := l.BeginUpdate(t.Context(), "tx-1"); !errors.Is(err, refused) {
+ t.Fatalf("BeginUpdate = %v", err)
+ }
+ if err := l.Acquire(t.Context(), func() error { return nil }); err != nil {
+ t.Fatalf("fixture: acquisition refused before the hold: %v", err)
+ }
+ if err := l.HoldUpdate(t.Context(), "tx-1"); err != nil {
+ t.Fatal(err)
+ }
+ if id, err := l.UpdateID(); err != nil || id != "tx-1" {
+ t.Fatalf("held record = %q, %v", id, err)
+ }
+ if err := l.Acquire(t.Context(), func() error { return nil }); !errors.Is(err, ErrUnavailable) {
+ t.Fatalf("acquisition under the hold = %v, want refused", err)
+ }
+ // A second hold keeps the record as it is; only EndUpdateWith, with its
+ // census, removes it.
+ if err := l.HoldUpdate(t.Context(), "tx-2"); err != nil {
+ t.Fatal(err)
+ }
+ if id, _ := l.UpdateID(); id != "tx-1" {
+ t.Fatalf("a second hold replaced the record with %q", id)
+ }
+ if err := l.EndUpdateWith(t.Context(), "tx-1", nil); !errors.Is(err, refused) {
+ t.Fatalf("EndUpdateWith under a refusing census = %v", err)
+ }
+ l.VerifyLaunches = func(context.Context, uint64) error { return nil }
+ if err := l.EndUpdateWith(t.Context(), "tx-1", nil); err != nil {
+ t.Fatal(err)
+ }
+ if err := l.Acquire(t.Context(), func() error { return nil }); err != nil {
+ t.Fatalf("acquisition after settlement = %v", err)
+ }
+ if err := l.HoldUpdate(t.Context(), "bad id"); !errors.Is(err, ErrUnavailable) {
+ t.Fatalf("invalid id = %v", err)
+ }
+}
+
+func TestHoldUpdateLeavesAPendingAcquisitionCensusAlone(t *testing.T) {
+ l := censusFixture(t)
+ b, _ := json.Marshal(Census{Schema: 1, Box: "box", Enrollment: "en", Migration: 1, Revision: 2, Complete: true, LaunchRevision: 1, Pending: "acquisition"})
+ if err := l.State.Write("census.json", b); err != nil {
+ t.Fatal(err)
+ }
+ if err := l.HoldUpdate(t.Context(), "tx-1"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := l.State.Read("update.json"); err == nil {
+ t.Fatal("hold wrote a record over a pending acquisition")
+ }
+ if err := l.Acquire(t.Context(), func() error { return nil }); err == nil {
+ t.Fatal("a pending acquisition census accepted a new acquisition")
+ }
+}
diff --git a/internal/rootparticipation/ledger.go b/internal/rootparticipation/ledger.go
index 66617f575..87d55d7aa 100644
--- a/internal/rootparticipation/ledger.go
+++ b/internal/rootparticipation/ledger.go
@@ -215,6 +215,40 @@ func (l Ledger) EndUpdateWith(ctx context.Context, id string, commit func() erro
return l.State.Remove("update.json")
}
+// HoldUpdate is the acquisition barrier recovery leaves when it defers a
+// settlement its census refused: root's update record under id, written
+// under the transaction lock, which refuses every new acquisition. Unlike
+// BeginUpdate it runs no idle check, since it only adds a refusal and
+// starts nothing; EndUpdateWith, with its census, is still the only way
+// the record goes. An existing record is kept as it is, and a census with
+// an acquisition pending already refuses acquisitions, so neither writes.
+func (l Ledger) HoldUpdate(ctx context.Context, id string) error {
+ if l.State == nil || !validID(id) {
+ return ErrUnavailable
+ }
+ unlock, e := l.lockTransaction(ctx)
+ if e != nil {
+ return e
+ }
+ defer unlock()
+ c, e := l.RecoveryCensus()
+ if e != nil {
+ return e
+ }
+ if c.Pending != "" {
+ return nil
+ }
+ if _, e = l.State.Read("update.json"); !errors.Is(e, os.ErrNotExist) {
+ return e
+ }
+ raw, _ := json.Marshal(struct {
+ Schema int
+ ID string
+ Revision uint64
+ }{1, id, c.Revision})
+ return l.State.Write("update.json", raw)
+}
+
func (l Ledger) UpdateID() (string, error) {
if l.State == nil {
return "", ErrUnavailable