amadan
WIP claim Plan 14 track E checkpoint 8 (not for merge): E census rows in their own overlay, recovery under merge ruling 5, E1 reconstruction status projection and signed replacement workflow
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
11 files changed,
+532
−15
internal/host/legacyrestart.go+1 −1internal/host/namedrecovery_active.go+46 −4internal/host/plan14_active_recovery_test.go+2 −1internal/host/plan14_e8_outcome_test.go+113 −0internal/host/recoverysettle.go+37 −0internal/host/roothistory.go+13 −2internal/host/roothistory_recovery.go+1 −1internal/host/rootrestart.go+23 −0internal/provision/testdata/plan14-root-reads-e.tsv+9 −6internal/release/plan14_history_status_test.go+122 −0internal/release/roothistory_status.go+165 −0
CI passed — run details
CI log
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok amadan.net/carlos/platform.git/launchctx 0.011s
? amadan.net/carlos/platform.git/launchctx/controlname [no test files]
? amadan.net/carlos/platform.git/launchctx/wireid [no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok github.com/carlosframework/platform 348.307s
? github.com/carlosframework/platform/cmd/changelog [no test files]
ok github.com/carlosframework/platform/cmd/console 2.923s
ok github.com/carlosframework/platform/cmd/moveprobe 92.382s
ok github.com/carlosframework/platform/internal/activator 227.676s
ok github.com/carlosframework/platform/internal/agentsock 0.021s
ok github.com/carlosframework/platform/internal/alerting 0.005s
ok github.com/carlosframework/platform/internal/analytics 0.019s
ok github.com/carlosframework/platform/internal/appconfig 2.329s
ok github.com/carlosframework/platform/internal/apperr 0.071s
ok github.com/carlosframework/platform/internal/applog 0.132s
ok github.com/carlosframework/platform/internal/awsinv 0.003s
ok github.com/carlosframework/platform/internal/boxchoice 0.004s
ok github.com/carlosframework/platform/internal/boxid 0.695s
? github.com/carlosframework/platform/internal/boxid/boxidtest [no test files]
? github.com/carlosframework/platform/internal/boxid/internal/signtest [no test files]
ok github.com/carlosframework/platform/internal/boxvolumes 0.009s
ok github.com/carlosframework/platform/internal/caps 0.012s
ok github.com/carlosframework/platform/internal/certwarm 0.002s
ok github.com/carlosframework/platform/internal/cgroup 0.198s
ok github.com/carlosframework/platform/internal/changelog 0.290s
ok github.com/carlosframework/platform/internal/clamd 0.054s
? github.com/carlosframework/platform/internal/cliflag [no test files]
ok github.com/carlosframework/platform/internal/console 368.306s
? github.com/carlosframework/platform/internal/controlname [no test files]
? github.com/carlosframework/platform/internal/controlsock [no test files]
ok github.com/carlosframework/platform/internal/dbdoor 6.034s
ok github.com/carlosframework/platform/internal/dbdoor/dbdoortest 0.009s
ok github.com/carlosframework/platform/internal/dbverify 0.004s
ok github.com/carlosframework/platform/internal/dnsrec 0.003s
ok github.com/carlosframework/platform/internal/docsite 0.005s
ok github.com/carlosframework/platform/internal/domains 2.101s
ok github.com/carlosframework/platform/internal/doorkey 0.006s
ok github.com/carlosframework/platform/internal/economics 0.004s
ok github.com/carlosframework/platform/internal/edge 26.858s
? github.com/carlosframework/platform/internal/edgeproof [no test files]
ok github.com/carlosframework/platform/internal/edgesfile 0.007s
ok github.com/carlosframework/platform/internal/fleetapi 0.120s
ok github.com/carlosframework/platform/internal/gate 1.655s
ok github.com/carlosframework/platform/internal/genselect 0.015s
ok github.com/carlosframework/platform/internal/geo 0.018s
ok github.com/carlosframework/platform/internal/geoip 0.029s
ok github.com/carlosframework/platform/internal/host 99.141s
ok github.com/carlosframework/platform/internal/hostactors 0.660s
ok github.com/carlosframework/platform/internal/hostbox 2.163s
ok github.com/carlosframework/platform/internal/hostclaim 0.529s
ok github.com/carlosframework/platform/internal/hostevidence 0.011s
ok github.com/carlosframework/platform/internal/hostexec 5.590s
ok github.com/carlosframework/platform/internal/hostfence 0.030s
ok github.com/carlosframework/platform/internal/hostfiles 11.098s
ok github.com/carlosframework/platform/internal/hostfloor 0.012s
ok github.com/carlosframework/platform/internal/hostguard 0.971s
ok github.com/carlosframework/platform/internal/hostops 1.430s
? github.com/carlosframework/platform/internal/hostops/fencerelease [no test files]
ok github.com/carlosframework/platform/internal/hostops/internal/fencerelease 0.011s
ok github.com/carlosframework/platform/internal/hostquarantine 0.011s
ok github.com/carlosframework/platform/internal/hostrelocate 0.468s
ok github.com/carlosframework/platform/internal/hostresolve 0.133s
ok github.com/carlosframework/platform/internal/hostwrite 0.011s
ok github.com/carlosframework/platform/internal/infraview 0.011s
ok github.com/carlosframework/platform/internal/instancectl 1.719s
ok github.com/carlosframework/platform/internal/instanceexec 2.743s
ok github.com/carlosframework/platform/internal/instancefs 0.011s
ok github.com/carlosframework/platform/internal/instancekey 0.011s
ok github.com/carlosframework/platform/internal/instanceupdates 0.017s
ok github.com/carlosframework/platform/internal/layout 1.423s
ok github.com/carlosframework/platform/internal/lease 9.511s
ok github.com/carlosframework/platform/internal/lease/leasetest 0.015s
ok github.com/carlosframework/platform/internal/ledger 0.010s
ok github.com/carlosframework/platform/internal/mail 0.168s
ok github.com/carlosframework/platform/internal/maintenance 0.095s
ok github.com/carlosframework/platform/internal/maintenanceexec 5.570s
ok github.com/carlosframework/platform/internal/maintenanceguard 0.073s
ok github.com/carlosframework/platform/internal/metrics 0.004s
ok github.com/carlosframework/platform/internal/moves 21.710s
ok github.com/carlosframework/platform/internal/moves/archive 0.771s
ok github.com/carlosframework/platform/internal/objstore 0.176s
ok github.com/carlosframework/platform/internal/ops 0.173s
ok github.com/carlosframework/platform/internal/park 0.221s
ok github.com/carlosframework/platform/internal/privatefile 0.005s
ok github.com/carlosframework/platform/internal/provision 15.526s
ok github.com/carlosframework/platform/internal/publicrelease 0.007s
? github.com/carlosframework/platform/internal/regionname [no test files]
ok github.com/carlosframework/platform/internal/registry 4.123s
ok github.com/carlosframework/platform/internal/relay 7.267s
ok github.com/carlosframework/platform/internal/release 22.261s
ok github.com/carlosframework/platform/internal/rollout 3.974s
ok github.com/carlosframework/platform/internal/rolloutwords 0.003s
ok github.com/carlosframework/platform/internal/rootaws 0.008s
ok github.com/carlosframework/platform/internal/rootparticipation 0.008s
ok github.com/carlosframework/platform/internal/rootstate 0.025s
ok github.com/carlosframework/platform/internal/s3compat 0.005s
ok github.com/carlosframework/platform/internal/safefile 0.006s
ok github.com/carlosframework/platform/internal/schedule 2.330s
ok github.com/carlosframework/platform/internal/sim/hostlife 13.165s
ok github.com/carlosframework/platform/internal/sim/litestreamfacts 4.177s
ok github.com/carlosframework/platform/internal/status 0.041s
ok github.com/carlosframework/platform/internal/steering 0.006s
ok github.com/carlosframework/platform/internal/stopevidence 0.241s
ok github.com/carlosframework/platform/internal/svcrole 0.002s
ok github.com/carlosframework/platform/internal/telemetry 0.004s
ok github.com/carlosframework/platform/internal/termfence 0.004s
ok github.com/carlosframework/platform/internal/tunnelclient 0.004s
ok github.com/carlosframework/platform/internal/unixdial 0.003s
ok github.com/carlosframework/platform/internal/updatewords 0.004s
ok github.com/carlosframework/platform/internal/usage 0.004s
ok github.com/carlosframework/platform/internal/version 0.001s
ok github.com/carlosframework/platform/internal/vet 1.548s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
minio: cached, digest verified
mc: cached, digest verified
minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
buckets and scoped user ready
=== RUN TestDeliveredContractMinio
=== RUN TestDeliveredContractMinio/put
=== RUN TestDeliveredContractMinio/get_round-trips_the_body
=== RUN TestDeliveredContractMinio/list_shows_the_key
=== RUN TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN TestDeliveredContractMinio/delete_removes_it
=== RUN TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
--- PASS: TestDeliveredContractMinio/put (0.00s)
--- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
--- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
--- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
--- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
--- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
--- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok github.com/carlosframework/platform/internal/objstore 0.021s
=== RUN TestT4StorageProbeMinio
2026/10/13 16:46:58 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DE24AAF725E0B9, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.00s)
PASS
ok github.com/carlosframework/platform/internal/host 0.020s
minio contract: PASS
diff --git a/internal/host/legacyrestart.go b/internal/host/legacyrestart.go| index be4a5d070..55092d1b3 100644 |
| --- a/internal/host/legacyrestart.go |
| +++ b/internal/host/legacyrestart.go |
| @@ -40,7 +40,7 @@ func (c rootRestartChecks) noteLegacy() { |
| func restartEffectFailed(err error) bool { |
| var notUp *release.UnitNotUpError |
| - return errors.As(err, ¬Up) && !errors.Is(err, errRestartBinding) |
| + return errors.As(err, ¬Up) |
| } |
| // This is main's stamped-unit path, including its fallback on Show or |
diff --git a/internal/host/namedrecovery_active.go b/internal/host/namedrecovery_active.go| index 0bc854bc5..cb1719d2b 100644 |
| --- a/internal/host/namedrecovery_active.go |
| +++ b/internal/host/namedrecovery_active.go |
| @@ -18,6 +18,7 @@ import ( |
| ) |
| func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootRestartChecks, getenv func(string) string, reg *registry.Registry) (handled bool, result error) { |
| + c.preserveOutcome = true |
| legacy, err := c.allowLegacy() |
| if err != nil { |
| return true, err |
| @@ -27,10 +28,11 @@ func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootResta |
| if err != nil || rt == nil || rt.Unit == "" || rt.Unit == activator.UnitName(host) { |
| return false, nil |
| } |
| + c.recoveryOutcome = &recoveryRestartOutcome{route: *rt} |
| } |
| // An optional binding cannot veto the no-census restart path. |
| defer func() { |
| - if legacy && result != nil && !restartEffectFailed(result) { |
| + if legacy && !c.recoveryOutcome.effect && result != nil && !restartEffectFailed(result) { |
| handled, result = false, nil |
| } |
| }() |
| @@ -128,8 +130,23 @@ func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootResta |
| if err := restartBoundUnit(n.source.sys, n.source.optDir, getenv, old, false, nil, c); err != nil { |
| return true, err |
| } |
| + if c.recoveryOutcome != nil && c.recoveryOutcome.unbound { |
| + return true, effect(func() error { |
| + current, err := reg.Get(host) |
| + if err != nil || current == nil { |
| + return errors.Join(err, os.ErrPermission) |
| + } |
| + if current.Version == pointer.Version { |
| + return nil |
| + } |
| + return reg.SetVersions(host, pointer.Version, current.Version) |
| + }) |
| + } |
| return true, effect(func() error { |
| if err := validate(); err != nil { |
| + if legacy { |
| + return nil |
| + } |
| return err |
| } |
| current, err := reg.Get(host) |
| @@ -198,24 +215,49 @@ func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootResta |
| if err != nil { |
| return true, err |
| } |
| - c.identity, c.effect = id, effect |
| + c.identity, c.effect, c.targetDigest = id, effect, digest |
| if err := restartBoundUnit(n.source.sys, n.source.optDir, getenv, old, false, nil, c); err != nil { |
| - if legacy && (errors.Is(err, errRestartBinding) || errors.Is(err, errUnitStopped)) { |
| + if legacy && !c.recoveryOutcome.effect && !restartEffectFailed(err) && (errors.Is(err, errRestartBinding) || errors.Is(err, errUnitStopped)) { |
| return false, settleRollbackAttempt(s, attempt) |
| } |
| + if legacy && errors.Is(err, errUnitStopped) { |
| + return true, settleRollbackAttempt(s, attempt) |
| + } |
| attempt.Failed = true |
| if writeErr := writeRollbackState(s, rollbackJournalKey(host), attempt); writeErr != nil { |
| return true, errors.Join(err, writeErr) |
| } |
| + if legacy && restartEffectFailed(err) { |
| + attempt.Recovering = true |
| + rollbackErr := effect(func() error { |
| + if err := writeRollbackState(s, rollbackJournalKey(host), attempt); err != nil { |
| + return err |
| + } |
| + if err := release.PointLive(n.source.optDir, host, rec.Account, old.Version, rec.App); err != nil { |
| + return err |
| + } |
| + if err := legacyRollbackRestart(n.source.sys, n.source.optDir, getenv, c.recoveryOutcome.route, c); err != nil { |
| + return err |
| + } |
| + return reg.SetVersions(host, old.Version, "") |
| + }) |
| + return true, errors.Join(err, rollbackErr, settleRollbackAttempt(s, attempt)) |
| + } |
| return true, errors.Join(err, resumeHistoryRestart(ctx, n.source.sys, n.source.optDir, getenv, c, n.writes, s, attempt)) |
| } |
| - attempt.Verified = true |
| + attempt.Verified = c.recoveryOutcome == nil || !c.recoveryOutcome.unbound |
| if err := settleRollbackAttempt(s, attempt); err != nil { |
| return true, err |
| } |
| + if !attempt.Verified { |
| + return true, effect(func() error { return reg.SetVersions(host, pointer.Version, old.Version) }) |
| + } |
| return true, effect(func() error { |
| current, err := c.host(ctx, host) |
| if err != nil || current.Source != old.Source || current.Version != pointer.Version || current.Digest != digest { |
| + if legacy { |
| + return reg.SetVersions(host, pointer.Version, old.Version) |
| + } |
| return errors.Join(err, os.ErrPermission) |
| } |
| return reg.SetVersions(host, pointer.Version, old.Version) |
diff --git a/internal/host/plan14_active_recovery_test.go b/internal/host/plan14_active_recovery_test.go| index 722217a1d..18b8173cd 100644 |
| --- a/internal/host/plan14_active_recovery_test.go |
| +++ b/internal/host/plan14_active_recovery_test.go |
| @@ -191,6 +191,7 @@ func TestPlan14SignedFixSupersedesOlderAttempt(t *testing.T) { |
| func TestPlan14ERecoveryBindingRefusalKeepsPromotedFix(t *testing.T) { |
| e, u, c, source, pub := signedRecoveryFixture(t) |
| c.legacy = func() (bool, error) { return true, nil } |
| + c.legacyReady = c.ready |
| // Refuse the final readiness binding after the process effect. |
| original := c.host |
| c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| @@ -211,7 +212,7 @@ func TestPlan14ERecoveryBindingRefusalKeepsPromotedFix(t *testing.T) { |
| if readErr != nil || attempt.Failed || !attempt.Settled { |
| t.Fatalf("binding refusal poisoned promotion: %+v %v", attempt, readErr) |
| } |
| - if handled || err != nil || liveErr != nil || live != "v2" || len(u.effects) != 1 { |
| + if !handled || err != nil || liveErr != nil || live != "v2" || len(u.effects) != 1 { |
| t.Fatalf("binding refusal undid promotion: handled=%v err=%v live=%s/%v effects=%v", handled, err, live, liveErr, u.effects) |
| } |
| } |
diff --git a/internal/host/plan14_e8_outcome_test.go b/internal/host/plan14_e8_outcome_test.go| new file mode 100644 |
| index 000000000..c52fafe86 |
| --- /dev/null |
| +++ b/internal/host/plan14_e8_outcome_test.go |
| @@ -0,0 +1,113 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +type e8ExeUnits struct{ *plan14RestartUnits } |
| + |
| +func (u e8ExeUnits) Exe(pid int) (string, error) { |
| + if len(u.effects) > 0 { |
| + return "", os.ErrPermission |
| + } |
| + return u.plan14RestartUnits.Exe(pid) |
| +} |
| + |
| +func TestPlan14ERecoveryRestartOutcome(t *testing.T) { |
| + for _, fault := range []string{"binding", "final binding", "exe", "failed", "auto-restart", "not ready"} { |
| + t.Run(fault, func(t *testing.T) { |
| + e, u, c, source, pub := signedRecoveryFixture(t) |
| + u.afterRestart = func() { u.props.NRestarts = 0 } |
| + c.legacy = func() (bool, error) { return true, nil } |
| + c.legacyReady = func(registry.Route) func() error { |
| + return func() error { |
| + if fault == "not ready" && len(u.effects) == 1 { |
| + return os.ErrDeadlineExceeded |
| + } |
| + return nil |
| + } |
| + } |
| + orig := c.host |
| + calls := 0 |
| + c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| + if len(u.effects) == 1 { |
| + calls++ |
| + if fault != "final binding" || calls >= 2 { |
| + if fault == "failed" { |
| + u.props.ActiveState, u.props.MainPID = "failed", 0 |
| + } |
| + if fault == "auto-restart" { |
| + u.props.ActiveState, u.props.SubState, u.props.NRestarts = "activating", "auto-restart", 1 |
| + } |
| + return proofBinding{}, os.ErrPermission |
| + } |
| + } |
| + return orig(ctx, host) |
| + } |
| + if fault == "exe" { |
| + source.sys = e8ExeUnits{u} |
| + } |
| + n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} |
| + fallbacks := 0 |
| + err := release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, func(host string) error { |
| + rt, e := e.root.reg.Get(host) |
| + if e != nil { |
| + return e |
| + } |
| + fallbacks++ |
| + return legacyCheckedUnitRestart(u, u.opt, func(string) string { return "5s" }, *rt, nil, c) |
| + }, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRecovery(func(ctx context.Context, rt registry.Route) (bool, error) { |
| + return n.recover(ctx, rt.Host, c, func(string) string { return "5s" }, e.root.reg) |
| + }, c.allowLegacy)) |
| + failed := fault == "failed" || fault == "auto-restart" || fault == "not ready" |
| + want, count := "v2", 1 |
| + if failed { |
| + want, count = "v1", 3 |
| + } |
| + live, liveErr := release.LiveVersion(u.opt, observedHost) |
| + if liveErr != nil || live != want || len(u.effects) != count || fallbacks != 0 || (err != nil) != failed { |
| + t.Fatalf("outcome: live=%s/%v effects=%v fallback=%d err=%v", live, liveErr, u.effects, fallbacks, err) |
| + } |
| + stamp, stampErr := e.root.reg.Get(observedHost) |
| + if stampErr != nil || stamp.Version != want { |
| + t.Fatalf("accepted outcome not stamped: %+v %v", stamp, stampErr) |
| + } |
| + disk, _ := c.attempts() |
| + a, readErr := readRollbackAttempt(disk, observedHost) |
| + if readErr != nil || !a.Settled || a.Failed != failed || (!failed && a.Verified) { |
| + t.Fatalf("mint evidence: %+v %v", a, readErr) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestPlan14ERecoveryNotUpPrecedesBinding(t *testing.T) { |
| + err := errors.Join(errRestartBinding, &release.UnitNotUpError{Unit: "console.service", Reason: "failed"}) |
| + if !restartEffectFailed(err) { |
| + t.Fatal("binding masked real not-up result") |
| + } |
| +} |
| + |
| +func TestPlan14ERecoveryStrictNotUpPrecedesBinding(t *testing.T) { |
| + e, u, c, source, pub := signedRecoveryFixture(t) |
| + original := c.host |
| + c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| + if len(u.effects) == 1 { |
| + u.props.ActiveState, u.props.MainPID = "failed", 0 |
| + return proofBinding{}, os.ErrPermission |
| + } |
| + return original(ctx, host) |
| + } |
| + n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} |
| + handled, err := n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg) |
| + var notUp *release.UnitNotUpError |
| + if !handled || !errors.As(err, ¬Up) || len(u.effects) != 3 { |
| + t.Fatalf("strict outcome lost: handled=%v err=%v effects=%v", handled, err, u.effects) |
| + } |
| +} |
diff --git a/internal/host/recoverysettle.go b/internal/host/recoverysettle.go| new file mode 100644 |
| index 000000000..508ef4ab0 |
| --- /dev/null |
| +++ b/internal/host/recoverysettle.go |
| @@ -0,0 +1,37 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func settleLegacyRecovery(sys unitSystem, optDir string, getenv func(string) string, binding proofBinding, rollback bool, before, want string, ready func() error, c rootRestartChecks) error { |
| + opts := legacySettleOptions(optDir, getenv, c.recoveryOutcome.route, want, c) |
| + settle := func() error { return settleUnit(sys, binding.Route.Unit, before, opts) } |
| + if err := settle(); err != nil { |
| + return err |
| + } |
| + c.recoveryOutcome.unbound = true |
| + observe := func() (proofBinding, error) { |
| + if rollback && c.recovered != nil { |
| + return c.recovered(context.Background(), binding) |
| + } |
| + return c.host(context.Background(), binding.Route.Host) |
| + } |
| + first, err := observe() |
| + if err != nil || first.Source != binding.Source || want != release.VersionPath(optDir, first.Route.Account, first.Version, first.Route.App) || |
| + (c.targetDigest != "" && first.Digest != c.targetDigest) || |
| + (rollback && (first.Version != binding.Version || first.Digest != binding.Digest)) { |
| + return settle() |
| + } |
| + if err := ready(); err != nil { |
| + return settle() |
| + } |
| + last, err := observe() |
| + if err != nil || last.Source != first.Source || last.Version != first.Version || last.Digest != first.Digest || last.Invocation != first.Invocation || last.PID != first.PID || last.Start != first.Start { |
| + return settle() |
| + } |
| + c.recoveryOutcome.unbound = false |
| + return nil |
| +} |
diff --git a/internal/host/roothistory.go b/internal/host/roothistory.go| index fb842c344..b73cb0b3e 100644 |
| --- a/internal/host/roothistory.go |
| +++ b/internal/host/roothistory.go |
| @@ -6,6 +6,7 @@ import ( |
| "debug/buildinfo" |
| "encoding/hex" |
| "io" |
| + "log" |
| "os" |
| "time" |
| @@ -26,7 +27,7 @@ func openRootHistory(store release.Store, optDir string) (*release.RootHistoryPa |
| if err != nil { |
| return err |
| } |
| - return h.History.ReceiveReconstruction(ctx, store, pub, b, time.Now(), func(want release.HistoryBinding) (string, error) { |
| + resolve := func(want release.HistoryBinding) (string, error) { |
| got, err := source.forHost(ctx, want.Host) |
| if err != nil { |
| return "", err |
| @@ -35,7 +36,17 @@ func openRootHistory(store release.Store, optDir string) (*release.RootHistoryPa |
| return "", os.ErrPermission |
| } |
| return got.Source, nil |
| - }) |
| + } |
| + err = h.History.ReceiveReconstruction(ctx, store, pub, b, time.Now(), resolve) |
| + projection, statusErr := rootstate.Open(release.RootHistoryStatusDir, true, true) |
| + if statusErr == nil { |
| + statusErr = h.History.ProjectMigrationStatus(projection, b, resolve) |
| + projection.Close() |
| + } |
| + if statusErr != nil { |
| + log.Print(statusErr) |
| + } |
| + return err |
| } |
| return h, nil |
| } |
diff --git a/internal/host/roothistory_recovery.go b/internal/host/roothistory_recovery.go| index fc3de6962..25eab4b6a 100644 |
| --- a/internal/host/roothistory_recovery.go |
| +++ b/internal/host/roothistory_recovery.go |
| @@ -198,7 +198,7 @@ func resumeHistoryRestart(ctx context.Context, sys unitSystem, optDir string, ge |
| } |
| return errors.Join(err, settleRollbackAttempt(s, a)) |
| } |
| - a.Verified = true |
| + a.Verified = checks.recoveryOutcome == nil || !checks.recoveryOutcome.unbound |
| if err := settleRollbackAttempt(s, a); err != nil { |
| return err |
| } |
diff --git a/internal/host/rootrestart.go b/internal/host/rootrestart.go| index 86893ed7f..8bc64065e 100644 |
| --- a/internal/host/rootrestart.go |
| +++ b/internal/host/rootrestart.go |
| @@ -21,7 +21,15 @@ import ( |
| "github.com/carlosframework/platform/internal/rootstate" |
| ) |
| +type recoveryRestartOutcome struct { |
| + route registry.Route |
| + effect bool |
| + unbound bool |
| +} |
| + |
| type rootRestartChecks struct { |
| + recoveryOutcome *recoveryRestartOutcome |
| + preserveOutcome bool |
| history func() (*release.RootHistoryPass, error) |
| readLive func(string) (string, error) |
| legacy func() (bool, error) |
| @@ -299,6 +307,9 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string, |
| if err := sys.ResetFailed(rt.Unit); err != nil { |
| return err |
| } |
| + if c.recoveryOutcome != nil { |
| + c.recoveryOutcome.effect = true |
| + } |
| if err := sys.Restart(rt.Unit); err != nil { |
| return err |
| } |
| @@ -320,8 +331,14 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string, |
| if err := sys.ResetFailed(rt.Unit); err != nil { |
| return err |
| } |
| + if c.recoveryOutcome != nil { |
| + c.recoveryOutcome.effect = true |
| + } |
| return sys.Restart(rt.Unit) |
| } |
| + if c.recoveryOutcome != nil { |
| + c.recoveryOutcome.effect = true |
| + } |
| if err := sys.TryRestart(rt.Unit); err != nil { |
| return err |
| } |
| @@ -330,6 +347,9 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string, |
| }); err != nil { |
| return err |
| } |
| + if c.recoveryOutcome != nil { |
| + return settleLegacyRecovery(sys, optDir, getenv, binding, rollback, before.InvocationID, want, ready, c) |
| + } |
| var observed proofBinding |
| observe := func() (proofBinding, error) { |
| if rollback && c.recovered != nil { |
| @@ -369,6 +389,9 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string, |
| }) |
| if err != nil { |
| if bindingErr != nil { |
| + if c.preserveOutcome { |
| + return errors.Join(err, bindingErr) |
| + } |
| return bindingErr |
| } |
| return err |
diff --git a/internal/provision/testdata/plan14-root-reads-e.tsv b/internal/provision/testdata/plan14-root-reads-e.tsv| index 76136057a..c59897302 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-e.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-e.tsv |
| @@ -2,7 +2,7 @@ |
| internal/host/agent.go runAdoptUnderRootRelease authority B3,E1 Root retains independent recovery results and includes exact failed target digests in proof mint refusals. Pending, malformed or unsynced evidence refuses minting; recovery remains independent. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14RecoveryMintEvidence, TestPlan14SignedActiveFixIgnoresHistory. E7 applies merge ruling 4: unbound no-census routes use the legacy pass and restart; ordinary candidates make no binding store call. Fixtures: TestR2UnguardedERecoveryUpdate, TestR2UnguardedERecoveryIncidentRecipe, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix. registry:Host@14260,whole:Route@10431,whole:Route@17915,whole:Route@17951,whole:Route@7852 f24943a35599f7a46117aff91d3d3c11507757c7f0688f6f831fecee29a88f84 |
| internal/host/agent.go runAdoptWithEnsureSystemUnits authority B3,C,D,E1,E2,E3,E4,F,J1 E1 selects the checked private history, canonical named-unit provider and root mint inventory under release exclusion with claims off or on; reconstruction and restart consumption remain incomplete. file:Open@191 9cdd0a958f5924c9c8d8c5613fd665f2c96a194cd11322adb6f0050fb183b541 |
| internal/host/namedrecovery.go namedUnitRecovery.repair authority E1,J1 Signed channel target, canonical deployment record, checked named unit and root-owned live pointer authorize stopped-unit pointer repair. Rechecks source, claim selection, pointer and artifact at the effect; does not start a process or change grants. Fixtures: TestPlan14Incident20261006WithLedger and TestPlan14SignedNamedRecoveryAuthority. Uses account/app from the same canonical read as the unit binding. Fixture: TestPlan14SignedNamedRecoveryUsesOneCanonicalRecord. file:Open@2565,registry:Channel@1814,whole:Route@1060,whole:Route@1822 8a42a6a27537be49d30d32891f407789449f9b77d37674fea5cb288d908adf6a |
| -internal/host/namedrecovery_active.go namedUnitRecovery.recover authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. E7 applies merge ruling 4: unbound no-census routes use the legacy pass and restart; ordinary candidates make no binding store call. Fixtures: TestR2UnguardedERecoveryUpdate, TestR2UnguardedERecoveryIncidentRecipe, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix. file:Open@5410,registry:Channel@2666,registry:Unit@316,registry:Unit@333,registry:Version@3655,registry:Version@3761,whole:Route@1586,whole:Route@2668,whole:Route@303,whole:Route@3581 95e34cf07d344a1dd85d6c7da36d80c45d996cbe6dfb241d7d477eacb2019192 |
| +internal/host/namedrecovery_active.go namedUnitRecovery.recover authority B3,E1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding, TestPlan14ERecoveryStrictNotUpPrecedesBinding. file:Open@5928,registry:Channel@2779,registry:Unit@342,registry:Unit@359,registry:Version@3777,registry:Version@3886,registry:Version@4173,registry:Version@4279,whole:Route@1699,whole:Route@2781,whole:Route@329,whole:Route@3700,whole:Route@4099,whole:Route@468,whole:Route@476,whole:Route@7138 1b2296098659ba83def943d8cbb75cf537fad2ef97b78c093262ff3586c3728a |
| internal/host/proofbinding.go proofBinding.authorizedRecovery authority B3,E1 Independent signed recovery authority and retained attempt outcomes. Fixtures: TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14ActiveRecoveryProcessCrash. a7a7cc9edf8437ed2a1fd4d4ed3d54e9ffab78b21ed3d2867a2254c7836e4f13 |
| internal/host/proofbinding.go proofBindingSource.binding authority B3,J1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@1176,file:EvalSymlinks@1485,file:EvalSymlinks@518,file:EvalSymlinks@757,registry:Unit@3161,whole:Route@265,whole:Route@3658,whole:Route@3665 56edaf01411de3813fe327973302b1df35e403d2870625fc341355b5bc5cfa0b |
| internal/host/proofbinding.go proofBindingSource.recovered authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Host@123,whole:Route@125 9ddd5ae1e5b981654e47a0f97cba43b93e32617feddf57918136c57c9943f8ae |
| @@ -11,15 +11,15 @@ internal/host/proofroot.go proofRoot.mint authority B3,J1,J4,E1 E1 production us |
| internal/host/proofroot.go proofRoot.observeAndRecord authority B3,J1,J4,E1 E1 production uses canonical observed programs and complete root refusal inventory; legacy diagnostics do not grant root mint authority. file:Readlink@108,registry:Account@443,registry:App@468,registry:Host@151,registry:Host@657,whole:Route@186,whole:Route@453,whole:Route@478 88065f1cfe7ad31178ba65c2672d8dd1e25fab26820290368fc063907f7b3b98 |
| internal/host/recoveryrefusals.go rootRestartChecks.mintRefusals authority B3,E1 Root retains independent recovery results and includes exact failed target digests in proof mint refusals. Pending, malformed or unsynced evidence refuses minting; recovery remains independent. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14RecoveryMintEvidence, TestPlan14SignedActiveFixIgnoresHistory. file:Read@616 1bb27522e17f4c16f7c09390f2a132ce6a1050e73fc807179d1fa9f4ff71278e |
| internal/host/roothistory.go boundRootHistory authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. file:Open@1061,file:Read@1379,registry:Account@613,registry:App@635,registry:Channel@456,registry:Channel@470,registry:Host@590,whole:Route@1469,whole:Route@377,whole:Route@472,whole:Route@592,whole:Route@615,whole:Route@637,whole:Route@846 fe46269101116374044658bbb87c1bd44317525dcadf227d4515b47845238be2 |
| -internal/host/roothistory.go openRootHistory authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. file:Open@104,registry:Account@800,registry:App@837,whole:Route@804,whole:Route@841 77b7ee45bd93e0e5d0dfd4ea444cb9df293461875ed0a42f8c177b84c65ed317 |
| +internal/host/roothistory.go openRootHistory authority E1 E1 reconstruction status reads exact private category revisions and canonical sources twice; public projection grants nothing. Fixtures: TestPlan14HistoryMigrationStatus, TestPlan14HistoryPendingStatus, TestPlan14HistoryStatusReplacementDelivery. Recovery outcome integration remains incomplete. file:Open@104,file:Open@980,registry:Account@740,registry:App@777,whole:Route@744,whole:Route@781 6615b63fd53196099dc7ced185ad25bf941c201dba4078481ac8e8e6b3aa854d |
| internal/host/roothistory.go recordRootProgram authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. whole:Route@151,whole:Route@297,whole:Route@398 fbbc9451c39309090580486d75da6120b02e1ba054157159846bea057ae0c80f |
| internal/host/roothistory_recovery.go historyRestartAttempt authority B3,E1 Captured root capability and exact process/source identity authorize restart independently of release history availability. Fixture: TestPlan14HistoryUnavailableAfterCaptureCannotBlockRestart. registry:Host@187,registry:Host@361,whole:Route@189,whole:Route@363 1c2af7b90cb56053c1d58c10ccaa851c3aa25b0be9e595bf8e90a013ad0bae66 |
| internal/host/roothistory_recovery.go linkRootHistory authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Prepare@121,registry:Account@374,registry:App@412,whole:Route@376,whole:Route@414 a60324796d2b25b3b130bb0439bbd9a415b679e6b51bc92a44f23f77e2d93e1e |
| -internal/host/roothistory_recovery.go resumeHistoryRestart authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Account@1025,registry:Account@2194,registry:App@1057,registry:App@2226,registry:Unit@824,whole:Route@1029,whole:Route@1061,whole:Route@2198,whole:Route@2230,whole:Route@828 7e11454ad9a1adf74bf437a54626674c285aa5e3f3eb3f37a0301daa2410ba71 |
| +internal/host/roothistory_recovery.go resumeHistoryRestart authority B3,E1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding. registry:Account@1025,registry:Account@2194,registry:App@1057,registry:App@2226,registry:Unit@824,whole:Route@1029,whole:Route@1061,whole:Route@2198,whole:Route@2230,whole:Route@828 79d86ba8876ab6cdeced4f27b0bf1f16097f2d56da1c08ae4839b5fc4143ec47 |
| internal/host/roothistory_recovery.go rootRestartChecks.settleHistory authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Account@580,registry:App@1351,registry:App@627,whole:Route@1353,whole:Route@582,whole:Route@629,whole:Route@845,whole:Route@991 ea977169cb738ba5e3e9bb83f86fbe0fca6e66ec6fcd9416dae94e8f39bc47fc |
| internal/host/rootrestart.go checkedUnitRestart authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@1798,registry:Account@1367,registry:Account@2166,registry:App@1399,registry:App@2198,registry:Host@1069,registry:Host@1854,registry:Host@2741,registry:Host@2963,registry:Host@3606,registry:Host@510,registry:Unit@3488,registry:Unit@711,whole:Route@1375,whole:Route@1407,whole:Route@2174,whole:Route@2206,whole:Route@3496,whole:Route@638,whole:Route@719 3a29634563392118e927685e478c8265916fbe89d922b9dde4c08aa5178c1c3c |
| internal/host/rootrestart.go productionRootRestartChecks authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Open@1356,whole:Route@1189,whole:Route@1990,whole:Route@2257 921445f1e11f9bfd202e309a0b0be4864f057a0907164f32ec8603f6949f4bdd |
| -internal/host/rootrestart.go restartBoundUnit authority B3,J1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Account@2482,registry:Account@3896,registry:App@2518,registry:App@3936,registry:Host@1585,registry:Host@2301,registry:Host@650,registry:Unit@1071,registry:Unit@1139,registry:Unit@1894,registry:Unit@1962,registry:Unit@2004,registry:Unit@3299,registry:Unit@395,whole:Route@198,whole:Route@2488,whole:Route@2524,whole:Route@317,whole:Route@3904,whole:Route@3944,whole:Route@4155 018a4502e3b1d18fde5f53d49805246b4aa83d74934d5ee0bcc7a5227ac4be5e |
| +internal/host/rootrestart.go restartBoundUnit authority B3,J1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding, TestPlan14ERecoveryStrictNotUpPrecedesBinding. registry:Account@2848,registry:Account@4333,registry:App@2884,registry:App@4373,registry:Host@1659,registry:Host@2667,registry:Host@650,registry:Unit@1071,registry:Unit@1213,registry:Unit@1968,registry:Unit@2113,registry:Unit@2229,registry:Unit@3665,registry:Unit@395,whole:Route@198,whole:Route@2854,whole:Route@2890,whole:Route@317,whole:Route@4341,whole:Route@4381,whole:Route@4592 123efb827a00d4350aa69ab03560cc69b460e3b5cf7f7daed181346efe784fae |
| internal/host/rootrestart.go rollbackUnitRestart authority B3,J1,E1 Captured canonical rollback authority is unchanged. Durable verified outcome permits later history settlement without another restart. Fixtures: TestPlan14HistoryRollbackLink and TestPlan14HistoryUnavailableCannotBlockCapturedRollback. file:Read@1458,registry:Host@1750,registry:Unit@1852,registry:Unit@2153,registry:Unit@435,registry:Unit@692,whole:Route@1758,whole:Route@1860,whole:Route@2161,whole:Route@422,whole:Route@774 4d02504999ba2746f3815cba0e3ca7fe8723cdf2528e57d3fa78b5acc9b90998 |
| internal/host/rootrestart.go rootRestartChecks.live authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@158 28721577488e77d229d3a5cbe6eb5806a4af0696d350a144c5ed2e4d10c36828 |
| internal/host/rootrollback.go captureRollback authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Read@704,registry:Account@1191,registry:App@1227,registry:Host@1154,registry:Host@341,registry:Host@823,whole:Route@1156,whole:Route@1193,whole:Route@1229,whole:Route@343,whole:Route@825 6d3ec51488ef1be9dac2d5fe168cb38e72614242fd18064ba441273b14af6efa |
| @@ -54,6 +54,9 @@ internal/release/roothistory_replace.go replacementReconstruction authority E1 S |
| internal/release/signedhead.go StageSignedHead authority E1 Canonical route is supplied by the root unit verifier. Verifies signed channel, digest ratchet and manifest/artifact before staging; unavailable pin census never authorizes restaging differing bytes. Fixtures: TestPlan14Incident20261006WithLedger and TestPlan14SignedNamedRecoveryAuthority. registry:Account@202,registry:Account@302,registry:Account@403,registry:App@214,registry:App@415,registry:Channel@222,registry:Host@434 ca02f04e55556ba0233dfa0848b0c533a9d9aa06cad6abd33cfd2968b3bd838a |
| internal/rootstate/link_linux.go ReadTrustedLink authority A,E1 Checks the full root-owned parent chain before descriptor-based live-link reading. Fixture: TestPlan14TrustedLiveLink; privileged acceptance remains required. file:Open@156 90f8e8de4e2c71f07b5d749cbe4bb1c1dfa426576a82138823449f56fc9ec5ab |
| internal/rootstate/link_linux.go directory.readLink authority A,E1 Pins the symlink inode with O_PATH and checks owner/type/link count before readlinkat. Fixture: TestPlan14TrustedLiveLink. file:Fstat@356,file:Openat@183,file:Readlinkat@554 a8559fe63dce48830916004ecfe7fbbe5d419568c7daf7a3073cf1b5bb12c3d7 |
| -@coverage internal/host d4f20f60919fd0c9e9228087c591e969dcb3ae7bb3b98f450d8a462f76f2efb1 |
| -@coverage internal/release 9837d8dab6911a5d922389195b3b06849cea63a8ed92fa2b9ba1c6b0509f4a86 |
| +@coverage internal/host f0cd6de14ac64636c123adde18a0b69ea46332ac6a075da5958a65e428e4432e |
| +@coverage internal/release 8d2ad3c9529ccf8a2da259399b7bb1de77b17a4914fda46c2c54c94779c3a617 |
| @coverage internal/rootstate 6338f74a6b2059cf77cf478bd6111b37b441eb9d65a309464c378359d4bdac05 |
| +internal/release/roothistory_status.go RootHistory.MigrationStatus authority E1 E1 reconstruction status reads exact private category revisions and canonical sources twice; public projection grants nothing. Fixtures: TestPlan14HistoryMigrationStatus, TestPlan14HistoryPendingStatus, TestPlan14HistoryStatusReplacementDelivery. Recovery outcome integration remains incomplete. file:Read@1062 c4dfc98bf7722bf88302eca9806027ad08eb446f0a0c8561bf926592665683a6 |
| +internal/host/legacyrestart.go restartEffectFailed authority B3 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding. b68bb65884a14fe0cf78a65f56ded4ea41584bba99af9195827d1ea191b340a9 |
| +internal/host/recoverysettle.go settleLegacyRecovery authority E1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding, TestPlan14ERecoveryStrictNotUpPrecedesBinding. registry:Account@752,registry:App@788,registry:Host@616,registry:Unit@320,whole:Route@255,whole:Route@328,whole:Route@624,whole:Route@758,whole:Route@794 65785c42a5e21854873d8d9ea952b6754cecbb766495768ef64edc3da4a446a6 |
diff --git a/internal/release/plan14_history_status_test.go b/internal/release/plan14_history_status_test.go| new file mode 100644 |
| index 000000000..db6d4cf2d |
| --- /dev/null |
| +++ b/internal/release/plan14_history_status_test.go |
| @@ -0,0 +1,122 @@ |
| +package release |
| + |
| +import ( |
| + "os" |
| + "path/filepath" |
| + "testing" |
| + "time" |
| +) |
| + |
| +func TestPlan14HistoryMigrationStatus(t *testing.T) { |
| + for _, fault := range []string{"valid", "missing", "changed source", "malformed", "unsynced"} { |
| + t.Run(fault, func(t *testing.T) { |
| + h, s, b := rootHistoryFixture(t) |
| + if fault == "missing" { |
| + os.Remove(filepath.Join(s.dir, historyKey(b))) |
| + } |
| + if fault == "malformed" { |
| + s.Write(historyKey(b), []byte("{")) |
| + } |
| + if fault == "unsynced" { |
| + s.fail = 1 |
| + } |
| + calls := 0 |
| + view, err := h.MigrationStatus(b, func(HistoryBinding) (string, error) { |
| + calls++ |
| + if fault == "changed source" && calls > 1 { |
| + return "new", nil |
| + } |
| + return "canonical", nil |
| + }) |
| + if fault == "valid" || fault == "missing" { |
| + if err != nil || view.Identity == "" || view.Revision != 1 || len(view.Hosts) != 1 || view.Hosts[0].Source != "canonical" { |
| + t.Fatalf("view=%+v err=%v", view, err) |
| + } |
| + if fault == "missing" && (view.Hosts[0].Revision != 0 || view.Hosts[0].Programs.Complete || view.Hosts[0].RefusalStatus.Complete) { |
| + t.Fatal("absence became completeness") |
| + } |
| + if fault == "valid" && view.Hosts[0].Revision != 1 { |
| + t.Fatal("wrong category revision") |
| + } |
| + } else if err == nil { |
| + t.Fatalf("%s projected usable status: %+v", fault, view) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestPlan14HistoryPendingStatus(t *testing.T) { |
| + h, s, b, pub, wire, now := reconstructionCrashFixture(t) |
| + h.Store = &reconstructionCrashStore{healthyRetryStore: s, stage: "intent", fail: true} |
| + resolve := func(HistoryBinding) (string, error) { return "canonical", nil } |
| + if err := h.Reconstruct(pub, wire, b.Account, b.App, now, resolve); err == nil { |
| + t.Fatal("no crash") |
| + } |
| + h.Store = s |
| + view, err := h.MigrationStatus(b, resolve) |
| + if err != nil || !view.Pending || view.PendingDigest == "" || view.PendingGrant == nil || view.PendingGrant.Operation != "crash" { |
| + t.Fatalf("pending status: %+v %v", view, err) |
| + } |
| + if _, err := h.Refusals(b); err == nil { |
| + t.Fatal("status removed pending barrier") |
| + } |
| + projection := &healthyRetryStore{dir: t.TempDir(), fault: os.ErrPermission} |
| + if err := h.ProjectMigrationStatus(projection, b, resolve); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, err := projection.Read(HistoryStatusKey(b.Account, b.App)) |
| + if err != nil || len(raw) == 0 { |
| + t.Fatalf("projection missing: %v", err) |
| + } |
| + projection.Write(HistoryStatusKey(b.Account, b.App), []byte(`{"complete":true}`)) |
| + if _, err := h.Refusals(b); err == nil { |
| + t.Fatal("projection supplied authority") |
| + } |
| +} |
| + |
| +func TestPlan14HistoryStatusReplacementDelivery(t *testing.T) { |
| + h, s, b, pub, wire, now := reconstructionCrashFixture(t) |
| + // Retain a real interrupted intent, then change its canonical source. |
| + h.Store = &reconstructionCrashStore{healthyRetryStore: s, stage: "intent", fail: true} |
| + if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err == nil { |
| + t.Fatal("no interruption") |
| + } |
| + h.Store = s |
| + resolve := func(HistoryBinding) (string, error) { return "new-source", nil } |
| + view, err := h.MigrationStatus(b, resolve) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + g := *view.PendingGrant |
| + g.Operation = "replacement-from-status" |
| + g.Replaces = view.PendingDigest |
| + g.AppRevision, g.BakeRevision = view.AppRevision, view.BakeRevision |
| + g.Hosts = nil |
| + for _, host := range view.Hosts { |
| + g.Hosts = append(g.Hosts, host.HistoryReconstructionHost) |
| + } |
| + // The deployment signer signs the exact revisions and sources from status. |
| + priv, pub := testKeys(t) |
| + st := NewDirStore(t.TempDir()) |
| + if err := PublishHistoryReconstruction(t.Context(), st, RootSigner(priv), g); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := h.ReceiveReconstruction(t.Context(), st, pub, b, now, resolve); err != nil { |
| + t.Fatal(err) |
| + } |
| + after, err := h.MigrationStatus(b, resolve) |
| + if err != nil || after.Pending || after.Revision != view.Revision+1 || !after.AppComplete || after.BakeComplete == 0 { |
| + t.Fatalf("replacement not complete: %+v %v", after, err) |
| + } |
| + if after.Clock.Holds[channelClockKey(b.Account, b.App, "stable")].Window != time.Hour { |
| + t.Fatal("replacement lost intended hold") |
| + } |
| + g.Operation = "stale-revision" |
| + stale, err := SignRootEnvelope(RootSigner(priv), g) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := h.Reconstruct(pub, stale, b.Account, b.App, now, resolve); err == nil { |
| + t.Fatal("stale projected revision accepted") |
| + } |
| +} |
diff --git a/internal/release/roothistory_status.go b/internal/release/roothistory_status.go| new file mode 100644 |
| index 000000000..da1c8f7ad |
| --- /dev/null |
| +++ b/internal/release/roothistory_status.go |
| @@ -0,0 +1,165 @@ |
| +package release |
| + |
| +import ( |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "slices" |
| + "strings" |
| + |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +const RootHistoryStatusDir = "/var/lib/carlos-release/history-status" |
| + |
| +type HistoryHostStatus struct { |
| + HistoryReconstructionHost |
| + Programs historyPrograms `json:"programs"` |
| + RefusalStatus historyRefusals `json:"refusal_status"` |
| + Attempt *HistoryAttempt `json:"attempt,omitempty"` |
| +} |
| + |
| +type HistoryMigrationStatus struct { |
| + HistoryReconstructionStatus |
| + Schema int `json:"schema"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + AppRevision uint64 `json:"app_revision"` |
| + AppComplete bool `json:"app_complete"` |
| + BakeRevision uint64 `json:"bake_revision"` |
| + BakeComplete uint64 `json:"bake_complete"` |
| + Hosts []HistoryHostStatus `json:"hosts"` |
| + Clock FirstSeen `json:"clock"` |
| + PendingDigest string `json:"pending_digest,omitempty"` |
| + PendingGrant *HistoryReconstruction `json:"pending_grant,omitempty"` |
| +} |
| + |
| +func HistoryStatusKey(acct, app string) string { return historyAppKey(acct, app) + ".status" } |
| + |
| +func (h *RootHistory) MigrationStatus(b HistoryBinding, resolve func(HistoryBinding) (string, error)) (HistoryMigrationStatus, error) { |
| + var out HistoryMigrationStatus |
| + if !b.valid() || resolve == nil { |
| + return out, os.ErrPermission |
| + } |
| + status, err := h.ReconstructionStatus() |
| + if err != nil { |
| + return out, err |
| + } |
| + out = HistoryMigrationStatus{HistoryReconstructionStatus: status, Schema: 1, Account: b.Account, App: b.App} |
| + state, err := h.reconstructionState() |
| + if err != nil { |
| + return out, err |
| + } |
| + hosts := []string{b.Host} |
| + if state.Pending != nil { |
| + raw, err := json.Marshal(state.Pending.Grant) |
| + if err != nil || reconstructionDigest(raw) != state.Pending.Digest { |
| + return out, os.ErrPermission |
| + } |
| + if state.Pending.Grant.Account == b.Account && state.Pending.Grant.App == b.App { |
| + out.PendingDigest = state.Pending.Digest |
| + grant := state.Pending.Grant |
| + out.PendingGrant = &grant |
| + for _, host := range grant.Hosts { |
| + hosts = append(hosts, host.Host) |
| + } |
| + } |
| + } |
| + read := func(key string, v any, valid func() bool) error { |
| + raw, err := h.Store.Read(key) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return nil |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + if len(raw) > rootstate.MaxRecord || strictCanonicalJSON(raw, v) != nil || !valid() { |
| + return os.ErrPermission |
| + } |
| + return h.Store.Write(key, raw) |
| + } |
| + var app historyApp |
| + if err := read(historyAppKey(b.Account, b.App), &app, func() bool { return validHistoryApp(app) && app.Account == b.Account && app.App == b.App }); err != nil { |
| + return out, err |
| + } |
| + out.AppRevision, out.AppComplete = app.Revision, app.Complete |
| + hosts = append(hosts, app.Hosts...) |
| + var bake rootBakeRecord |
| + if err := read(rootBakeKey, &bake, func() bool { return validRootBake(bake) }); err != nil { |
| + return out, err |
| + } |
| + out.BakeRevision, out.BakeComplete = bake.Revision, bake.Complete[prodKey(b.Account, b.App)] |
| + out.Clock = FirstSeen{Production: map[string]*prodMemory{}, Holds: map[string]*holdMemory{}} |
| + if p := bake.Clock.Production[prodKey(b.Account, b.App)]; p != nil { |
| + out.Clock.Production[prodKey(b.Account, b.App)] = p |
| + } |
| + for key, hold := range bake.Clock.Holds { |
| + if strings.HasPrefix(key, holdKeyPrefix(b.Account, b.App)) { |
| + out.Clock.Holds[key] = hold |
| + } |
| + } |
| + names, err := h.Store.Names() |
| + if err != nil { |
| + return out, err |
| + } |
| + for _, name := range names { |
| + if name == rootBakeKey || !strings.HasSuffix(name, ".json") { |
| + continue |
| + } |
| + var record historyRecord |
| + if err := read(name, &record, func() bool { return validHistory(record, record.Binding) && name == historyKey(record.Binding) }); err != nil { |
| + return out, err |
| + } |
| + if record.Binding.Account == b.Account && record.Binding.App == b.App { |
| + hosts = append(hosts, record.Binding.Host) |
| + } |
| + } |
| + slices.Sort(hosts) |
| + hosts = slices.Compact(hosts) |
| + if len(hosts) > historyLimit { |
| + return out, os.ErrPermission |
| + } |
| + for _, host := range hosts { |
| + binding := HistoryBinding{Host: host, Account: b.Account, App: b.App} |
| + source, err := resolve(binding) |
| + if err != nil { |
| + return out, err |
| + } |
| + if source == "" { |
| + return out, os.ErrPermission |
| + } |
| + record, err := h.readRecord(binding) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return out, err |
| + } |
| + out.Hosts = append(out.Hosts, HistoryHostStatus{HistoryReconstructionHost: HistoryReconstructionHost{Host: host, Source: source, Revision: record.Revision, Refusals: record.Refusals.Values}, Programs: record.Programs, RefusalStatus: record.Refusals, Attempt: record.Attempt}) |
| + } |
| + for _, host := range out.Hosts { |
| + source, err := resolve(HistoryBinding{Host: host.Host, Account: b.Account, App: b.App}) |
| + if err != nil { |
| + return out, err |
| + } |
| + if source != host.Source { |
| + return out, os.ErrPermission |
| + } |
| + } |
| + return out, nil |
| +} |
| + |
| +func (h *RootHistory) ProjectMigrationStatus(dst rootstate.Store, b HistoryBinding, resolve func(HistoryBinding) (string, error)) error { |
| + if dst == nil { |
| + return os.ErrPermission |
| + } |
| + view, err := h.MigrationStatus(b, resolve) |
| + if err != nil { |
| + return err |
| + } |
| + raw, err := json.Marshal(view) |
| + if err != nil { |
| + return err |
| + } |
| + if len(raw) > rootstate.MaxRecord { |
| + return os.ErrPermission |
| + } |
| + return dst.Write(HistoryStatusKey(b.Account, b.App), raw) |
| +} |