amadan
WIP claim Plan 14 track E checkpoint 10 (not for merge): merge Part 1 rounds 4-6 (15fe8df1); E1 signed retirement of settled recovery outcomes
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
41 files changed,
+2501
−260
internal/host/agent.go+21 −22internal/host/legacyrestart.go+25 −7internal/host/namedrecovery_pass.go+32 −0internal/host/plan14_backend_source_test.go+10 −2internal/host/plan14_constructors_test.go+1 −1internal/host/plan14_e9_census_test.go+1 −1internal/host/plan14_fix2_index_test.go+3 −3internal/host/plan14_fix2_live_linux_test.go+24 −7internal/host/plan14_fix3_live_linux_test.go+76 −56internal/host/plan14_fix4_adopt_linux_test.go+100 −0internal/host/plan14_fix4_live_linux_test.go+548 −0internal/host/plan14_fix5_live_linux_test.go+305 −0internal/host/plan14_fix5_main_linux_test.go+97 −0internal/host/plan14_history_retirement_test.go+227 −0internal/host/plan14_recovery_pass_test.go+60 −0internal/host/proofbinding.go+1 −0internal/host/proofnamed.go+5 −20internal/host/proofobserve.go+4 −0internal/host/proofroot.go+3 −2internal/host/recoveryrefusals.go+36 −13internal/host/releaseinit_linux_test.go+4 −1internal/host/roothistory.go+10 −30internal/host/roothistory_maintenance.go+63 −0internal/host/roothistory_recovery.go+3 −0internal/host/rootrestart.go+54 −45internal/host/rootrollback_resume.go+10 −0internal/provision/plan14_e_write_census_test.go+1 −0internal/provision/plan14_legacy_identity_test.go+43 −0internal/provision/plan14_root_reads_test.go+12 −0internal/provision/testdata/plan14-root-reads-e.tsv+54 −26internal/provision/testdata/plan14-root-reads-fix4.tsv+15 −0internal/provision/testdata/plan14-root-reads-fix5.tsv+11 −0internal/provision/testdata/plan14-root-reads-fix6.tsv+3 −0internal/release/plan14_history_retirement_process_test.go+108 −0internal/release/plan14_history_retirement_test.go+225 −0internal/release/roothistory.go+1 −1internal/release/roothistory_delivery.go+69 −0internal/release/roothistory_outcomes.go+12 −2internal/release/roothistory_retirement.go+152 −0internal/release/roothistory_status.go+65 −18internal/release/versions.go+7 −3
diff --git a/internal/host/agent.go b/internal/host/agent.go| index df7b8ebc6..7548c9bbb 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -4343,7 +4343,9 @@ func runAdoptUnderRootRelease(args []string, buildClaims ClaimFactory, ensureUni |
| // settled: D1's settle passing records the build in the healthy history |
| // (D3, proofobserve.go). Root is the process that can hash the unit's |
| // executable. |
| - settled := recordSettled(systemdUnits{}, procExePath, release.OptDir(), release.RootHealthyDir, time.Now) |
| + observer := productionProofObserver(release.OptDir()) |
| + observer.resolve = restartChecks.resolve |
| + settled := observer.recordSettled(release.RootHealthyDir) |
| restart := rootRestartRouteWith(reg, systemdUnits{}, release.OptDir(), os.Getenv, settled, restartChecks) |
| // The restart-request half, and the ONE place it dispatches from — |
| // including for exec-backed routes, whose children belong to the agent: |
| @@ -4475,9 +4477,7 @@ func runAdoptUnderRootRelease(args []string, buildClaims ClaimFactory, ensureUni |
| linkRootHistory(history, restartChecks, release.OptDir(), os.Getenv) |
| } |
| recovery := productionNamedUnitRecovery(store, pub, release.OptDir(), claims.Writes) |
| - passOpts := []release.PassOption{release.WithRootHistory(history), release.WithRecovery(func(ctx context.Context, rt registry.Route) (bool, error) { |
| - return recovery.recover(ctx, rt.Host, restartChecks, os.Getenv, reg) |
| - }, restartChecks.allowLegacy), release.WithCapabilities(caps.Ops{}), release.BakeDir(dataDir()), release.WithRestartFailurePropagation(), release.WithRestartAttemptSettled(restartChecks.finish), |
| + passOpts := []release.PassOption{release.WithRootHistory(history), recovery.passOption(reg, restartChecks, os.Getenv), release.WithCapabilities(caps.Ops{}), release.BakeDir(dataDir()), release.WithRestartAttemptSettled(restartChecks.finish), |
| release.WithRunningVersion(restartChecks.runningVersion), |
| release.WithSelectedRestarts( |
| rootSelectedRestart(reg, systemdUnits{}, release.OptDir(), os.Getenv, settled, restartChecks, false), |
| @@ -4501,27 +4501,26 @@ func runAdoptUnderRootRelease(args []string, buildClaims ClaimFactory, ensureUni |
| // adopt a proven head over it (final review H4). After it, mint on a |
| // proving box and, on the hourly pass, say when a requirement is off; |
| // a route the pass changed is looked at again first. |
| - pass := func() error { return release.Pass(ctx, reg, store, pub, release.OptDir(), restart, passOpts...) } |
| + pr := proofRoot{ |
| + obs: observer, |
| + reg: reg, |
| + store: store, |
| + bakeDir: dataDir(), |
| + healthyDir: release.RootHealthyDir, |
| + policyPath: release.ProofPolicyPath, |
| + readKey: readProvingKey, |
| + hourly: !*stageOnly, |
| + logf: log.Printf, |
| + } |
| if history != nil { |
| - pr := proofRoot{ |
| - recordProgram: func(b liveBuild) error { return recordRootProgram(history, b) }, |
| - refusals: func(b release.HistoryBinding) ([]release.Refusal, error) { |
| - return restartChecks.mintRefusals(history.History, b) |
| - }, |
| - obs: productionProofObserver(release.OptDir()), |
| - reg: reg, |
| - store: store, |
| - bakeDir: dataDir(), |
| - healthyDir: release.RootHealthyDir, |
| - policyPath: release.ProofPolicyPath, |
| - readKey: readProvingKey, |
| - hourly: !*stageOnly, |
| - logf: log.Printf, |
| + pr.recordProgram = func(b liveBuild) error { return recordRootProgram(history, b) } |
| + pr.refusals = func(b release.HistoryBinding) ([]release.Refusal, error) { |
| + return restartChecks.mintRefusals(history.History, b) |
| } |
| - ordinary := pass |
| - pass = func() error { return pr.around(ctx, ordinary) } |
| } |
| - passErr := pass() |
| + passErr := pr.around(ctx, func() error { |
| + return release.Pass(ctx, reg, store, pub, release.OptDir(), restart, passOpts...) |
| + }) |
| if passErr != nil { |
| log.Printf("adopt: %v", passErr) |
| } |
diff --git a/internal/host/legacyrestart.go b/internal/host/legacyrestart.go| index 888d586fb..dcbcc3afe 100644 |
| --- a/internal/host/legacyrestart.go |
| +++ b/internal/host/legacyrestart.go |
| @@ -2,6 +2,7 @@ package host |
| import ( |
| "errors" |
| + "fmt" |
| "log" |
| "os" |
| "path/filepath" |
| @@ -16,6 +17,7 @@ var errRestartBinding = errors.Join(errors.New("named-unit binding changed"), os |
| // A retained census keeps the strict path even after the flag is disabled. |
| // An unreadable census is never evidence that this is a legacy box. |
| +// The claims flag alone cannot select bound effects without a census. |
| func legacyRootRestart() (bool, error) { |
| l, required, err := rootparticipation.OpenForUpdate() |
| if l.State != nil { |
| @@ -43,11 +45,19 @@ func restartEffectFailed(err error) bool { |
| } |
| // This is main's stamped-unit path, including its fallback on Show or |
| -// ExecStart mismatch. Binding is optional until a root census exists. |
| +// ExecStart mismatch. Binding cannot drive effects without a root census. |
| func legacyCheckedUnitRestart(sys unitSystem, optDir string, getenv func(string) string, rt registry.Route, settled func(registry.Route) error, c rootRestartChecks) error { |
| before, err := sys.Show(rt.Unit) |
| link := release.LiveLink(optDir, rt.Host) |
| - if err != nil || before.ExecPath != link || before.ActiveState != "active" { |
| + if err != nil { |
| + log.Printf("adopt: %s: cannot read %s (%v); restart not checked", rt.Host, rt.Unit, err) |
| + return sys.TryRestart(rt.Unit) |
| + } |
| + if before.ExecPath != link { |
| + log.Printf("adopt: binding: %s runs %s, but the unit stamp is on %s; restart not checked", rt.Unit, before.ExecPath, rt.Host) |
| + return sys.TryRestart(rt.Unit) |
| + } |
| + if before.ActiveState != "active" { |
| return sys.TryRestart(rt.Unit) |
| } |
| want, err := filepath.EvalSymlinks(link) |
| @@ -59,11 +69,15 @@ func legacyCheckedUnitRestart(sys unitSystem, optDir string, getenv func(string) |
| } |
| err = settleUnit(sys, rt.Unit, before.InvocationID, legacySettleOptions(optDir, getenv, rt, want, c)) |
| if errors.Is(err, errUnitStopped) { |
| + log.Printf("adopt: %s: %s was stopped during settle; not judged, not restarted", rt.Host, rt.Unit) |
| return nil |
| } |
| - if err == nil && settled != nil { |
| - if e := settled(rt); e != nil { |
| - log.Print(e) |
| + if err == nil { |
| + log.Printf("adopt: %s: %s settled on %s", rt.Host, rt.Unit, filepath.Dir(want)) |
| + if settled != nil { |
| + if e := settled(rt); e != nil { |
| + log.Print(e) |
| + } |
| } |
| } |
| return err |
| @@ -72,7 +86,7 @@ func legacyCheckedUnitRestart(sys unitSystem, optDir string, getenv func(string) |
| func legacyRollbackRestart(sys unitSystem, optDir string, getenv func(string) string, rt registry.Route, c rootRestartChecks) error { |
| before, _ := sys.Show(rt.Unit) |
| if err := sys.ResetFailed(rt.Unit); err != nil { |
| - log.Print(err) |
| + log.Printf("adopt: %s: reset-failed %s: %v (continuing)", rt.Host, rt.Unit, err) |
| } |
| if err := sys.Restart(rt.Unit); err != nil { |
| return err |
| @@ -83,9 +97,13 @@ func legacyRollbackRestart(sys unitSystem, optDir string, getenv func(string) st |
| } |
| err = settleUnit(sys, rt.Unit, before.InvocationID, legacySettleOptions(optDir, getenv, rt, want, c)) |
| if errors.Is(err, errUnitStopped) { |
| + log.Printf("adopt: %s: %s was stopped during the rollback's settle; not judged, not restarted", rt.Host, rt.Unit) |
| return nil |
| } |
| - return err |
| + if err != nil { |
| + return fmt.Errorf("rollback did not come up either: %w", err) |
| + } |
| + return nil |
| } |
| func legacySettleOptions(optDir string, getenv func(string) string, rt registry.Route, want string, c rootRestartChecks) settleOpts { |
diff --git a/internal/host/namedrecovery_pass.go b/internal/host/namedrecovery_pass.go| new file mode 100644 |
| index 000000000..fb57ae96f |
| --- /dev/null |
| +++ b/internal/host/namedrecovery_pass.go |
| @@ -0,0 +1,32 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "log" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func (n namedUnitRecovery) passOption(reg *registry.Registry, checks rootRestartChecks, getenv func(string) string) release.PassOption { |
| + remainingNamed := false |
| + return release.WithRecovery(func(ctx context.Context, rt registry.Route) (bool, error) { |
| + done, err := n.recover(ctx, rt.Host, checks, getenv, reg) |
| + if err != nil { |
| + log.Printf("adopt: %v", err) |
| + return true, nil |
| + } |
| + if !done && (release.Adoptable(rt) || release.SidecarAdoptable(rt)) && rt.Unit != "" && rt.Unit != activator.UnitName(rt.Host) { |
| + remainingNamed = true |
| + } |
| + return done, nil |
| + }, func() (bool, error) { |
| + named := remainingNamed |
| + remainingNamed = false |
| + if !named { |
| + return true, nil |
| + } |
| + return checks.allowLegacy() |
| + }) |
| +} |
diff --git a/internal/host/plan14_backend_source_test.go b/internal/host/plan14_backend_source_test.go| index 473015710..69c8ee102 100644 |
| --- a/internal/host/plan14_backend_source_test.go |
| +++ b/internal/host/plan14_backend_source_test.go |
| @@ -8,6 +8,7 @@ import ( |
| "testing" |
| "time" |
| + "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/domains" |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -74,6 +75,9 @@ func TestPlan14OrdinaryDispatchSourceGap(t *testing.T) { |
| if err != nil || need { |
| t.Fatalf("%s ordinary record dispatched: need=%v err=%v", forge, need, err) |
| } |
| + if err := reg.Delete(candidate.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| if err := reg.Upsert(candidate); err != nil { |
| t.Fatal(err) |
| } |
| @@ -82,8 +86,12 @@ func TestPlan14OrdinaryDispatchSourceGap(t *testing.T) { |
| if err := rootRestartRouteWith(reg, u, "", nil, nil, checks)(record.Host); err != nil { |
| t.Fatal(err) |
| } |
| - if len(u.effects) != 0 { |
| - t.Fatalf("ordinary restart: %v", u.effects) |
| + if candidate.Unit == "" { |
| + if len(u.effects) != 1 || u.effects[0] != activator.UnitName(record.Host) { |
| + t.Fatalf("ordinary restart: %v", u.effects) |
| + } |
| + } else if len(u.effects) != 0 { |
| + t.Fatalf("forged named restart: %v", u.effects) |
| } |
| } |
diff --git a/internal/host/plan14_constructors_test.go b/internal/host/plan14_constructors_test.go| index 2ee90571a..d2af5b7f9 100644 |
| --- a/internal/host/plan14_constructors_test.go |
| +++ b/internal/host/plan14_constructors_test.go |
| @@ -45,7 +45,7 @@ func TestPlan14ProductionProofConstructors(t *testing.T) { |
| } |
| } |
| if call, ok := n.(*ast.CallExpr); ok { |
| - if fn, ok := call.Fun.(*ast.Ident); ok && fn.Name == "recordSettled" && len(call.Args) == 5 && isRootHealthy(call.Args[3]) { |
| + if fn, ok := call.Fun.(*ast.SelectorExpr); ok && fn.Sel.Name == "recordSettled" && len(call.Args) == 1 && isRootHealthy(call.Args[0]) { |
| settledWriter++ |
| } |
| } |
diff --git a/internal/host/plan14_e9_census_test.go b/internal/host/plan14_e9_census_test.go| index afd417d9a..a735ca3b3 100644 |
| --- a/internal/host/plan14_e9_census_test.go |
| +++ b/internal/host/plan14_e9_census_test.go |
| @@ -66,7 +66,7 @@ func TestPlan14ERecoveryNoCensusResume(t *testing.T) { |
| h := seedHostRootHistory(t, *rt) |
| c.history = func() (*release.RootHistoryPass, error) { return &release.RootHistoryPass{History: h}, nil } |
| c.legacy = func() (bool, error) { return true, nil } |
| - if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil { |
| + if err := resumeHistoryRestart(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}, disk, a); err != nil { |
| t.Fatal(err) |
| } |
| after, err := disk.Read(rollbackJournalKey(a.Host)) |
diff --git a/internal/host/plan14_fix2_index_test.go b/internal/host/plan14_fix2_index_test.go| index 2bb2f39a0..b62e6e309 100644 |
| --- a/internal/host/plan14_fix2_index_test.go |
| +++ b/internal/host/plan14_fix2_index_test.go |
| @@ -33,13 +33,13 @@ func (s *fix2IndexStore) Get(ctx context.Context, key string) ([]byte, string, e |
| } |
| return s.Store.Get(ctx, key) |
| } |
| -func TestPlan14Fix2RecordIndexRetriesAndRefreshesBodies(t *testing.T) { |
| +func TestPlan14Fix2RecordIndexCachesDiscoveryAndRefreshesBodies(t *testing.T) { |
| e, rt, rec, source := plan14ProofFixture(t) |
| - store := &fix2IndexStore{Store: e.store, accountFailure: true} |
| + store := &fix2IndexStore{Store: e.store} |
| source.store = store |
| source.cacheIndex = true |
| if _, err := source.resolve(t.Context(), rt); err != nil { |
| - t.Fatal("partial listing was refused", err) |
| + t.Fatal("record discovery failed", err) |
| } |
| lists := store.lists |
| rec.RecordSeq++ |
diff --git a/internal/host/plan14_fix2_live_linux_test.go b/internal/host/plan14_fix2_live_linux_test.go| index f58a79c8c..2ab363b71 100644 |
| --- a/internal/host/plan14_fix2_live_linux_test.go |
| +++ b/internal/host/plan14_fix2_live_linux_test.go |
| @@ -20,7 +20,9 @@ func TestPlan14Fix2LiveSafety(t *testing.T) { |
| } |
| for _, scenario := range []string{"console update with empty history", "two consecutive updates", "incident recipe", "failed forward fix", "crash rollback", "successful restart crash before settle"} { |
| t.Run(scenario, func(t *testing.T) { |
| - e, rt, u, c, _ := plan14AdoptionFixture(t) |
| + e, rt, base, c, _ := plan14AdoptionFixture(t) |
| + u := &r2Units{plan14RestartUnits: base} |
| + c.legacy, c.legacyReady = legacyRootRestart, c.ready |
| if err := e.store.Delete(t.Context(), provision.RecordKey(rt.Account, rt.App, rt.Host)); err != nil { |
| t.Fatal(err) |
| } |
| @@ -113,17 +115,32 @@ func TestPlan14Fix2LiveSafety(t *testing.T) { |
| return release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, restart, release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollback)) |
| } |
| err = run() |
| - if scenario == "crash rollback" { |
| - if err == nil { |
| - t.Fatal("failed update succeeded") |
| - } |
| - } else if err != nil { |
| + if err != nil { |
| t.Fatal(err) |
| } |
| want := "v2" |
| - if scenario == "crash rollback" || scenario == "incident recipe" { |
| + if scenario == "incident recipe" { |
| want = "v1" |
| } |
| + if scenario == "crash rollback" || scenario == "incident recipe" || scenario == "failed forward fix" { |
| + // Main cannot settle a guard ExecStart against the live-link |
| + // stamp. A stopped unit needs the operator's reset and start. |
| + if u.props.MainPID != 0 { |
| + t.Fatal("main path woke a stopped guard unit", u.effects) |
| + } |
| + if live, err := release.LiveVersion(u.opt, rt.Host); err != nil || live != want { |
| + t.Fatal("main pointer result", live, err) |
| + } |
| + if scenario == "crash rollback" { |
| + return |
| + } |
| + if err := u.ResetFailed(rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := u.Start(rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| if u.props.MainPID <= 0 || u.exe != release.VersionPath(u.opt, "", want, "console") { |
| t.Fatalf("not serving %s: %+v", want, u) |
| } |
diff --git a/internal/host/plan14_fix3_live_linux_test.go b/internal/host/plan14_fix3_live_linux_test.go| index a481be21c..d0a99c5b3 100644 |
| --- a/internal/host/plan14_fix3_live_linux_test.go |
| +++ b/internal/host/plan14_fix3_live_linux_test.go |
| @@ -30,6 +30,7 @@ type r2Units struct { |
| showErr error |
| tryErr error |
| tries []string |
| + starts []string |
| } |
| func (u *r2Units) Show(unit string) (unitProps, error) { return u.props, u.showErr } |
| @@ -51,6 +52,14 @@ func (u *r2Units) ResetFailed(unit string) error { |
| return nil |
| } |
| +func (u *r2Units) Start(unit string) error { |
| + u.starts = append(u.starts, unit) |
| + if u.props.ActiveState == "active" { |
| + return nil |
| + } |
| + return u.Restart(unit) |
| +} |
| + |
| type r2Live struct { |
| e *proofRootEnv |
| rt registry.Route |
| @@ -131,7 +140,9 @@ func (f *r2Live) restart() func(string) error { |
| return rootRestartRouteWith(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, nil, f.c) |
| } |
| func (f *r2Live) pass(t *testing.T, extra ...release.PassOption) error { |
| - opts := []release.PassOption{release.WithRunningVersion(f.c.runningVersion), release.WithRestartFailurePropagation(), release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)), release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c))} |
| + start := f.c |
| + start.legacyStart = f.u.Start |
| + opts := []release.PassOption{release.WithRunningVersion(f.c.runningVersion), release.WithRestartFailurePropagation(), release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, start)), release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c))} |
| return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, f.restart(), append(opts, extra...)...) |
| } |
| func (f *r2Live) wantLive(t *testing.T, v string) { |
| @@ -236,7 +247,6 @@ func TestR2UnguardedIncidentRecipe(t *testing.T) { |
| if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { |
| t.Fatal("PID-zero binding rejected", err) |
| } |
| - f.c.legacy = nil |
| } |
| f.promote(t, "v1") |
| if err := f.pass(t); err != nil { |
| @@ -267,15 +277,15 @@ func TestR2GuardedAutoRestartFixForward(t *testing.T) { |
| for _, state := range []unitProps{{ActiveState: "activating", SubState: "auto-restart"}, {ActiveState: "activating", SubState: "start"}} { |
| t.Run(state.SubState, func(t *testing.T) { |
| f := r2Fixture(t, "guard") |
| - f.c.legacy = nil |
| f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = state.ActiveState, state.SubState, 0 |
| if err := f.pass(t); err != nil { |
| t.Fatal(err) |
| } |
| f.wantLive(t, "v2") |
| f.wantRunning(t, "v2") |
| - if len(f.u.effects) != 2 || f.u.effects[0] != "reset console.service" { |
| - t.Fatal(f.u.effects) |
| + // Without a census, main try-restarts the activating unit once. |
| + if len(f.u.effects) != 1 || len(f.u.tries) != 1 || f.u.tries[0] != "console.service" { |
| + t.Fatal(f.u.effects, f.u.tries) |
| } |
| }) |
| } |
| @@ -441,64 +451,74 @@ func TestR2CrashBeforeRestartCapture(t *testing.T) { |
| if !plan14RootFixture(t) { |
| return |
| } |
| - for _, fail := range []bool{false, true} { |
| - t.Run(map[bool]string{false: "forward", true: "rollback"}[fail], func(t *testing.T) { |
| - f := r2Fixture(t, "list") |
| - crashed := errors.New("crash after repoint before restart capture") |
| - func() { |
| - defer func() { |
| - if recover() != crashed { |
| - t.Fatal("missing crash") |
| - } |
| - }() |
| - _ = release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, func(string) error { panic(crashed) }, release.WithRunningVersion(f.c.runningVersion)) |
| - }() |
| - f.wantLive(t, "v2") |
| - f.wantRunning(t, "v1") |
| - state, err := f.c.attempts() |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - if _, err := readRollbackAttempt(state, f.rt.Host); !os.IsNotExist(err) { |
| - t.Fatal("attempt already captured", err) |
| - } |
| - m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| - if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil { |
| - t.Fatal(err) |
| - } |
| - f.promote(t, "v3") |
| - // A new resolver and restart callback model the next root process. |
| - f.source.index = nil |
| - if fail { |
| - f.u.afterEffect = func() { |
| - if f.u.exe == release.VersionPath(f.u.opt, "", "v3", f.rt.App) { |
| - f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 |
| - } |
| + // Merge ruling 6: without a census the rollback target is main's cur |
| + // (the interrupted repoint's v2); only a census-backed box retains the |
| + // verified running build (v1) as its rollback target. |
| + for _, mode := range []string{"no census", "census"} { |
| + for _, fail := range []bool{false, true} { |
| + t.Run(mode+"/"+map[bool]string{false: "forward", true: "rollback"}[fail], func(t *testing.T) { |
| + f := r2Fixture(t, "list") |
| + if mode == "census" { |
| + f.c.legacy = func() (bool, error) { return false, nil } |
| } |
| - } |
| - err = f.pass(t) |
| - if fail { |
| - if err == nil { |
| - t.Fatal("failed build accepted") |
| - } |
| - f.wantLive(t, "v1") |
| + // Main's rollback target, and the retained one with a census. |
| + target := map[string]string{"no census": "v2", "census": "v1"}[mode] |
| + crashed := errors.New("crash after repoint before restart capture") |
| + func() { |
| + defer func() { |
| + if recover() != crashed { |
| + t.Fatal("missing crash") |
| + } |
| + }() |
| + _ = release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, func(string) error { panic(crashed) }, release.WithRunningVersion(f.c.runningVersion)) |
| + }() |
| + f.wantLive(t, "v2") |
| f.wantRunning(t, "v1") |
| - } else { |
| + state, err := f.c.attempts() |
| if err != nil { |
| t.Fatal(err) |
| } |
| - f.wantLive(t, "v3") |
| - f.wantRunning(t, "v3") |
| - prior, err := os.Readlink(filepath.Join(f.u.opt, "previous", f.rt.Host)) |
| - if err != nil || prior != release.VersionPath(f.u.opt, "", "v1", f.rt.App) { |
| - t.Fatal("lost running authority", prior, err) |
| + if _, err := readRollbackAttempt(state, f.rt.Host); !os.IsNotExist(err) { |
| + t.Fatal("attempt already captured", err) |
| } |
| - pins, err := release.LivePins(f.u.opt) |
| - if err != nil || !pins["v1"] { |
| - t.Fatal("rollback build not pinned", pins, err) |
| + m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| + if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil { |
| + t.Fatal(err) |
| } |
| - } |
| - }) |
| + f.promote(t, "v3") |
| + // A new resolver and restart callback model the next root process. |
| + f.source.index = nil |
| + if fail { |
| + f.u.afterEffect = func() { |
| + if f.u.exe == release.VersionPath(f.u.opt, "", "v3", f.rt.App) { |
| + f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 |
| + } |
| + } |
| + } |
| + err = f.pass(t) |
| + if fail { |
| + if err == nil { |
| + t.Fatal("failed build accepted") |
| + } |
| + f.wantLive(t, target) |
| + f.wantRunning(t, target) |
| + } else { |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantLive(t, "v3") |
| + f.wantRunning(t, "v3") |
| + prior, err := os.Readlink(filepath.Join(f.u.opt, "previous", f.rt.Host)) |
| + if err != nil || prior != release.VersionPath(f.u.opt, "", target, f.rt.App) { |
| + t.Fatal("previous pointer", prior, err) |
| + } |
| + pins, err := release.LivePins(f.u.opt) |
| + if err != nil || !pins[target] { |
| + t.Fatal("rollback build not pinned", pins, err) |
| + } |
| + } |
| + }) |
| + } |
| } |
| } |
diff --git a/internal/host/plan14_fix4_adopt_linux_test.go b/internal/host/plan14_fix4_adopt_linux_test.go| new file mode 100644 |
| index 000000000..b32a88e31 |
| --- /dev/null |
| +++ b/internal/host/plan14_fix4_adopt_linux_test.go |
| @@ -0,0 +1,100 @@ |
| +package host |
| + |
| +import ( |
| + "encoding/base64" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +func TestPlan14Fix4AdoptPerHostFailureExitsZero(t *testing.T) { |
| + if os.Getenv("PLAN14_FIX4_ADOPT_CHILD") == "1" { |
| + args := []string{"-registry", os.Getenv("PLAN14_FIX4_REGISTRY")} |
| + if os.Getenv("PLAN14_FIX4_STAGE") == "1" { |
| + args = append(args, "-stage-only") |
| + } |
| + runAdoptWithEnsureSystemUnits(args, nil, func() {}, func() (rootstate.Store, error) { return &rollbackMemory{files: map[string][]byte{}}, nil }) |
| + return |
| + } |
| + if os.Geteuid() == 0 { |
| + t.Skip("fixture exercises the local registry as an unprivileged user") |
| + } |
| + for _, stage := range []bool{false, true} { |
| + for _, crash := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "hourly", true: "stage"}[stage]+map[bool]string{false: "/restart-error", true: "/rollback-error"}[crash], func(t *testing.T) { |
| + e, rt, u, _, pub := plan14AdoptionFixture(t) |
| + rt.Unit = activator.UnitName(rt.Host) |
| + if err := e.root.reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + dir := t.TempDir() |
| + if err := os.Mkdir(filepath.Join(dir, "data"), 0755); err != nil { |
| + t.Fatal(err) |
| + } |
| + bucket := filepath.Join(dir, "bucket") |
| + store := release.NewDirStore(bucket) |
| + keys, err := e.store.List(t.Context(), "") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, key := range keys { |
| + raw, _, err := e.store.Get(t.Context(), key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := store.Put(t.Context(), key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + stub := `#!/bin/sh |
| +case "$1" in |
| + show) |
| + if [ -f "$PLAN14_FIX4_EFFECT" ]; then |
| + printf 'ActiveState=failed\nSubState=failed\nMainPID=0\n' |
| + else |
| + printf 'Id=console.service\nActiveState=active\nSubState=running\nMainPID=1\nInvocationID=old\nExecStart={ path=%s ; }\n' "$PLAN14_FIX4_LIVE" |
| + fi ;; |
| + try-restart) |
| + if [ "$PLAN14_FIX4_CRASH" = 1 ]; then : > "$PLAN14_FIX4_EFFECT"; exit 0; fi |
| + exit 1 ;; |
| + start|restart) exit 1 ;; |
| + *) exit 0 ;; |
| +esac |
| +` |
| + if err := os.WriteFile(filepath.Join(dir, "systemctl"), []byte(stub), 0755); err != nil { |
| + t.Fatal(err) |
| + } |
| + cmd := exec.Command(os.Args[0], "-test.run=^TestPlan14Fix4AdoptPerHostFailureExitsZero$", "-test.v") |
| + cmd.Dir = dir |
| + bit := func(v bool) string { |
| + if v { |
| + return "1" |
| + } |
| + return "0" |
| + } |
| + cmd.Env = isolatedAdoptOrderEnv(t, |
| + "PATH="+dir+":"+os.Getenv("PATH"), "PLAN14_FIX4_ADOPT_CHILD=1", "PLAN14_FIX4_STAGE="+bit(stage), "PLAN14_FIX4_CRASH="+bit(crash), |
| + "PLAN14_FIX4_REGISTRY="+e.root.reg.Path(), "PLAN14_FIX4_EFFECT="+filepath.Join(dir, "effect"), "PLAN14_FIX4_LIVE="+release.LiveLink(u.opt, rt.Host), |
| + "CARLOS_DATA_DIR="+filepath.Join(dir, "data"), "CARLOS_OPT_DIR="+u.opt, "CARLOS_DEPLOYMENT_DIR="+bucket, "CARLOS_DEPLOYMENT_BUCKET=", |
| + "CARLOS_HOST_CLAIM=", "CARLOS_REGION_HA=", "CARLOS_MOVES_PHASE1=", "CARLOS_SECRETS_KEY=", "CARLOS_RELEASE_PUBKEY="+base64.StdEncoding.EncodeToString(pub), |
| + ) |
| + out, err := cmd.CombinedOutput() |
| + if err != nil || !strings.Contains(string(out), "update to v2 failed") || !strings.Contains(string(out), "may be serving nothing") { |
| + t.Fatalf("per-host failure exit=%v\n%s", err, out) |
| + } |
| + if crash && !strings.Contains(string(out), "did not come up") { |
| + t.Fatal("crash did not take D1 rollback", string(out)) |
| + } |
| + if live, err := release.LiveVersion(u.opt, rt.Host); err != nil || live != "v1" { |
| + t.Fatal(live, err) |
| + } |
| + }) |
| + } |
| + } |
| +} |
diff --git a/internal/host/plan14_fix4_live_linux_test.go b/internal/host/plan14_fix4_live_linux_test.go| new file mode 100644 |
| index 000000000..18ecef2ed |
| --- /dev/null |
| +++ b/internal/host/plan14_fix4_live_linux_test.go |
| @@ -0,0 +1,548 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rootparticipation" |
| +) |
| + |
| +func (f *r2Live) productionPass(t *testing.T) error { |
| + t.Helper() |
| + start := f.c |
| + start.legacyStart = f.u.Start |
| + observer := f.e.root.obs |
| + observer.resolve, observer.sys = f.c.resolve, f.u |
| + observer.procExe = func(int) string { return f.u.exe } |
| + pr := f.e.root |
| + pr.obs = observer |
| + return pr.around(t.Context(), func() error { |
| + restart := rootRestartRouteWith(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, observer.recordSettled(f.e.bake), f.c) |
| + return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, restart, |
| + release.WithRunningVersion(f.c.runningVersion), release.WithRestartAttemptSettled(f.c.finish), |
| + release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, start)), |
| + release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c))) |
| + }) |
| +} |
| + |
| +func r3RestartCount(u *r2Units) int { |
| + n := 0 |
| + for _, effect := range u.effects { |
| + if strings.HasPrefix(effect, "restart ") { |
| + n++ |
| + } |
| + } |
| + return n |
| +} |
| + |
| +func TestR3StickyBindingErrorMasksCrash(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, kind := range []string{"none", "list"} { |
| + for _, state := range []string{"failed", "auto-restart", "healthy"} { |
| + t.Run(kind+"/"+state, func(t *testing.T) { |
| + f := r2Fixture(t, kind) |
| + original := f.c.host |
| + f.c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| + if r3RestartCount(f.u) > 0 && strings.HasSuffix(f.u.exe, "/v2/console") { |
| + return proofBinding{}, os.ErrPermission |
| + } |
| + return original(ctx, host) |
| + } |
| + ready := func(registry.Route) func() error { |
| + return func() error { |
| + if f.u.exe == release.VersionPath(f.u.opt, "", "v2", f.rt.App) && state != "healthy" { |
| + return os.ErrDeadlineExceeded |
| + } |
| + return nil |
| + } |
| + } |
| + f.c.ready, f.c.legacyReady = ready, ready |
| + sleep := f.c.sleep |
| + f.c.sleep = func(d time.Duration) { |
| + sleep(d) |
| + if f.u.exe != release.VersionPath(f.u.opt, "", "v2", f.rt.App) || state == "healthy" { |
| + return |
| + } |
| + f.u.props.MainPID = 0 |
| + if state == "failed" { |
| + f.u.props.ActiveState, f.u.props.SubState = "failed", "failed" |
| + } else { |
| + f.u.props.ActiveState, f.u.props.SubState = "activating", "auto-restart" |
| + } |
| + } |
| + if err := f.productionPass(t); err != nil { |
| + t.Fatal("per-host failure changed main's pass status", err) |
| + } |
| + want, restarts := "v1", 2 |
| + if state == "healthy" { |
| + want, restarts = "v2", 1 |
| + } |
| + f.wantLive(t, want) |
| + f.wantRunning(t, want) |
| + if got := r3RestartCount(f.u); got != restarts { |
| + t.Fatalf("restarts=%d want=%d: %v", got, restarts, f.u.effects) |
| + } |
| + }) |
| + } |
| + } |
| +} |
| + |
| +func TestR3FallbackDoubleRestart(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, rollback := range []bool{false, true} { |
| + for _, journalFailure := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "restart", true: "rollback"}[rollback]+map[bool]string{false: "/binding", true: "/journal"}[journalFailure], func(t *testing.T) { |
| + f := r2Fixture(t, "list") |
| + initial, err := f.c.resolve(t.Context(), f.rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rollback { |
| + f.u.crashNew = true |
| + } |
| + original := f.c.host |
| + f.c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| + if r3RestartCount(f.u) > 0 { |
| + return proofBinding{}, os.ErrPermission |
| + } |
| + return original(ctx, host) |
| + } |
| + if journalFailure { |
| + f.c.host = original |
| + store, err := f.c.attempts() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + memory := store.(*rollbackMemory) |
| + f.u.afterEffect = func() { memory.failWrite = true } |
| + } |
| + err = f.restart()(f.rt.Host) |
| + if !rollback { |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantRunning(t, "v2") |
| + if r3RestartCount(f.u) != 1 { |
| + t.Fatal(f.u.effects) |
| + } |
| + return |
| + } |
| + var notUp *release.UnitNotUpError |
| + if !errors.As(err, ¬Up) { |
| + t.Fatal("missing crash verdict", err) |
| + } |
| + if err := release.PointLive(f.u.opt, f.rt.Host, "", "v1", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + if journalFailure { |
| + store, _ := f.c.attempts() |
| + store.(*rollbackMemory).failWrite = false |
| + } |
| + f.c.host = original |
| + f.c.rollback = func(ctx context.Context, b proofBinding) (proofBinding, error) { |
| + return f.source.rollback(ctx, initial) |
| + } |
| + f.u.afterEffect = nil |
| + if journalFailure { |
| + store, _ := f.c.attempts() |
| + f.u.afterEffect = func() { store.(*rollbackMemory).failWrite = true } |
| + } |
| + // Refuse identity after the rollback effect, inside the callback's copy. |
| + host := f.c.host |
| + f.c.host = func(ctx context.Context, h string) (proofBinding, error) { |
| + if r3RestartCount(f.u) >= 2 { |
| + return proofBinding{}, os.ErrPermission |
| + } |
| + return host(ctx, h) |
| + } |
| + before := r3RestartCount(f.u) |
| + if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if r3RestartCount(f.u)-before != 1 { |
| + t.Fatal(f.u.effects) |
| + } |
| + f.wantLive(t, "v1") |
| + f.wantRunning(t, "v1") |
| + }) |
| + } |
| + } |
| +} |
| + |
| +type r3ExeUnits struct { |
| + *r2Units |
| + denied bool |
| +} |
| + |
| +func (u *r3ExeUnits) Exe(pid int) (string, error) { |
| + if u.denied { |
| + return "", os.ErrPermission |
| + } |
| + return u.r2Units.Exe(pid) |
| +} |
| + |
| +func TestPlan14Fix4ExeDeniedAfterRestart(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + f := r2Fixture(t, "list") |
| + u := &r3ExeUnits{r2Units: f.u} |
| + f.source.sys = u |
| + f.u.afterEffect = func() { u.denied = true } |
| + if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := rootRestartRouteWith(f.e.root.reg, u, f.u.opt, func(string) string { return "5s" }, nil, f.c)(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantLive(t, "v2") |
| + f.wantRunning(t, "v2") |
| + if r3RestartCount(f.u) != 1 { |
| + t.Fatal(f.u.effects) |
| + } |
| +} |
| + |
| +func TestR3CensusProbeErrorBlocksOrdinary(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + if err := os.MkdirAll(rootparticipation.Namespace, 0770); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Chmod(rootparticipation.Namespace, 0770); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := legacyRootRestart(); err == nil { |
| + t.Fatal("unreadable census fixture did not refuse") |
| + } |
| + for _, stamped := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "unstamped", true: "derived"}[stamped], func(t *testing.T) { |
| + f := r2Fixture(t, "none") |
| + f.rt.Unit = "" |
| + if stamped { |
| + f.rt.Unit = activator.UnitName(f.rt.Host) |
| + } |
| + if err := f.e.root.reg.Delete(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.e.root.reg.Upsert(f.rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.c.legacy = func() (bool, error) { t.Fatal("ordinary route probed census"); return false, os.ErrPermission } |
| + if err := f.productionPass(t); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantLive(t, "v2") |
| + f.wantRunning(t, "v2") |
| + if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestR3PreviousPins(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + t.Run("relative live link", func(t *testing.T) { |
| + f := r2Fixture(t, "list") |
| + link := release.LiveLink(f.u.opt, f.rt.Host) |
| + relative, err := filepath.Rel(filepath.Dir(link), f.u.exe) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Remove(link); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Symlink(relative, link); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.productionPass(t); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantLive(t, "v2") |
| + f.wantRunning(t, "v2") |
| + prior, err := filepath.EvalSymlinks(filepath.Join(f.u.opt, "previous", f.rt.Host)) |
| + if err != nil || prior != release.VersionPath(f.u.opt, "", "v1", f.rt.App) { |
| + t.Fatal(prior, err) |
| + } |
| + }) |
| +} |
| + |
| +type r3DiscoveryStore struct { |
| + release.Store |
| + lists, gets int |
| + blocked <-chan struct{} |
| +} |
| + |
| +func (s *r3DiscoveryStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + s.lists++ |
| + if s.blocked != nil { |
| + <-s.blocked |
| + } |
| + return nil, os.ErrPermission |
| +} |
| +func (s *r3DiscoveryStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + s.gets++ |
| + return s.Store.Get(ctx, key) |
| +} |
| + |
| +func TestPlan14Fix4OrdinaryProductionWrapper(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, kind := range []string{"unit", "exec", "derived", "hibernate"} { |
| + for _, stall := range []bool{false, true} { |
| + t.Run(kind+map[bool]string{false: "/error", true: "/hang"}[stall], func(t *testing.T) { |
| + f := r2Fixture(t, "none") |
| + f.rt.Unit = "" |
| + f.rt.Backing = registry.BackingUnit |
| + f.rt.Socket = "/run/ordinary.sock" |
| + f.rt.Addr, f.rt.Health = "", "" |
| + if kind == "exec" { |
| + f.rt.Backing = "exec" |
| + } |
| + if kind == "derived" { |
| + f.rt.Unit = activator.UnitName(f.rt.Host) |
| + } |
| + if kind == "hibernate" { |
| + f.rt.Hibernate = true |
| + } |
| + if err := f.e.root.reg.Delete(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.e.root.reg.Upsert(f.rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + store := &r3DiscoveryStore{Store: f.e.store} |
| + if stall { |
| + unblock := make(chan struct{}) |
| + store.blocked = unblock |
| + timer := time.AfterFunc(time.Second, func() { close(unblock) }) |
| + defer timer.Stop() |
| + } |
| + f.source.store = store |
| + if err := f.productionPass(t); err != nil { |
| + t.Fatal(err) |
| + } |
| + if store.lists != 0 || store.gets != 0 { |
| + t.Fatalf("ordinary proof wrapper read records: lists=%d gets=%d", store.lists, store.gets) |
| + } |
| + f.wantLive(t, "v2") |
| + if kind == "hibernate" { |
| + if r3RestartCount(f.u) != 0 { |
| + t.Fatal("hibernating route restarted", f.u.effects) |
| + } |
| + } else { |
| + f.wantRunning(t, "v2") |
| + } |
| + }) |
| + } |
| + } |
| +} |
| + |
| +func TestPlan14Fix4IndexFailureCachedForPass(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + f := r2Fixture(t, "none") |
| + store := &r3DiscoveryStore{Store: f.e.store} |
| + f.source.store = store |
| + other := f.rt |
| + other.Host = "second.example.com" |
| + if err := f.e.root.reg.Upsert(other); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Observation, adoption, readiness recording and minting share one resolver. |
| + if err := f.productionPass(t); err != nil { |
| + t.Fatal(err) |
| + } |
| + if store.lists != 1 { |
| + t.Fatalf("index attempts=%d want=1", store.lists) |
| + } |
| + next := productionProofBindingSource(store, f.u, f.u.opt) |
| + if _, err := next.resolve(t.Context(), f.rt); err == nil { |
| + t.Fatal("failed discovery accepted") |
| + } |
| + if store.lists != 2 { |
| + t.Fatalf("new pass did not retry: %d", store.lists) |
| + } |
| +} |
| + |
| +func TestPlan14Fix4HibernateStartGuard(t *testing.T) { |
| + e, rt, u, c, _ := plan14AdoptionFixture(t) |
| + rt.Hibernate = true |
| + if err := e.root.reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + c.legacyStart = func(string) error { t.Fatal("woke hibernating route"); return nil } |
| + c.legacy = func() (bool, error) { t.Fatal("hibernating start read census"); return false, os.ErrPermission } |
| + if err := rollbackUnitRestart(e.root.reg, u, u.opt, nil, c)(rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(u.effects) != 0 { |
| + t.Fatal(u.effects) |
| + } |
| +} |
| + |
| +func TestPlan14Fix4NotUpWinsOverBinding(t *testing.T) { |
| + if !restartEffectFailed(errors.Join(errRestartBinding, &release.UnitNotUpError{Unit: "console.service"})) { |
| + t.Fatal("binding masked D1 failure") |
| + } |
| +} |
| + |
| +func TestPlan14Fix4IncidentProductionWiring(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, kind := range []string{"none", "guard", "list"} { |
| + for _, order := range []string{"reset before adoption", "reset after adoption", "never failed"} { |
| + t.Run(kind+"/"+order, func(t *testing.T) { |
| + f := r2Fixture(t, kind) |
| + if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.u.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App) |
| + f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 |
| + if order == "never failed" { |
| + f.u.props.ActiveState, f.u.props.SubState = "inactive", "dead" |
| + } |
| + if order == "reset before adoption" { |
| + if err := f.u.ResetFailed(f.rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + before := len(f.u.effects) |
| + if kind != "none" { |
| + if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { |
| + t.Fatal("PID-zero binding rejected", err) |
| + } |
| + } |
| + f.promote(t, "v1") |
| + if err := f.productionPass(t); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantLive(t, "v1") |
| + if len(f.u.starts) != 0 { |
| + t.Fatal("adoption called start", f.u.starts) |
| + } |
| + if order != "reset after adoption" && (f.u.props.MainPID != 0 || len(f.u.effects) != before) { |
| + t.Fatal("woke inactive unit", f.u.effects) |
| + } |
| + if order == "reset after adoption" { |
| + if err := f.u.ResetFailed(f.rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := f.u.Restart(f.rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantRunning(t, "v1") |
| + }) |
| + } |
| + } |
| +} |
| + |
| +func TestPlan14Fix4RepointFailureStillLogged(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + f := r2Fixture(t, "list") |
| + if err := os.MkdirAll(filepath.Join(f.u.opt, "previous", f.rt.Host), 0755); err != nil { |
| + t.Fatal(err) |
| + } |
| + output, err := os.CreateTemp(t.TempDir(), "pass-output") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + original := os.Stdout |
| + os.Stdout = output |
| + defer func() { os.Stdout = original; output.Close() }() |
| + err = f.productionPass(t) |
| + os.Stdout = original |
| + if err != nil { |
| + t.Fatal("repoint failure changed pass status", err) |
| + } |
| + if err := output.Close(); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, err := os.ReadFile(output.Name()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !strings.Contains(string(raw), "repoint failed:") { |
| + t.Fatal("missing repoint diagnostic", string(raw)) |
| + } |
| + f.wantLive(t, "v1") |
| + f.wantRunning(t, "v1") |
| + if r3RestartCount(f.u) != 0 { |
| + t.Fatal(f.u.effects) |
| + } |
| +} |
| + |
| +type r3ResetErrorUnits struct { |
| + *r2Units |
| + resets int |
| +} |
| + |
| +func (u *r3ResetErrorUnits) ResetFailed(string) error { u.resets++; return os.ErrPermission } |
| + |
| +func TestPlan14Fix4RollbackResetFailureMatchesMain(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + f := r2Fixture(t, "list") |
| + u := &r3ResetErrorUnits{r2Units: f.u} |
| + f.source.sys = u |
| + if err := release.StageArtifact(u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + u.crashNew = true |
| + err := rootRestartRouteWith(f.e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, f.c)(f.rt.Host) |
| + var notUp *release.UnitNotUpError |
| + if !errors.As(err, ¬Up) { |
| + t.Fatal("missing crash verdict", err) |
| + } |
| + if err := release.PointLive(u.opt, f.rt.Host, "", "v1", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := rollbackUnitRestart(f.e.root.reg, u, u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if u.resets != 1 || r3RestartCount(f.u) != 2 { |
| + t.Fatal("reset failure repeated or prevented restart", u.resets, u.effects) |
| + } |
| + f.wantLive(t, "v1") |
| + f.wantRunning(t, "v1") |
| +} |
diff --git a/internal/host/plan14_fix5_live_linux_test.go b/internal/host/plan14_fix5_live_linux_test.go| new file mode 100644 |
| index 000000000..748bf028e |
| --- /dev/null |
| +++ b/internal/host/plan14_fix5_live_linux_test.go |
| @@ -0,0 +1,305 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "reflect" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +// Each unit has separate process state. In particular, restarting the local |
| +// binding must never look like a restart of a different box-local stamp. |
| +type fix5Units struct { |
| + units map[string]*r2Units |
| + events []string |
| + resetError, tryError bool |
| +} |
| + |
| +func (u *fix5Units) Show(name string) (unitProps, error) { return u.units[name].Show(name) } |
| +func (u *fix5Units) Exe(pid int) (string, error) { |
| + for _, unit := range u.units { |
| + if unit.props.MainPID == pid { |
| + return unit.Exe(pid) |
| + } |
| + } |
| + return "", os.ErrNotExist |
| +} |
| +func (u *fix5Units) StartTime(pid int) (string, error) { |
| + for _, unit := range u.units { |
| + if unit.props.MainPID == pid { |
| + return unit.StartTime(pid) |
| + } |
| + } |
| + return "", os.ErrNotExist |
| +} |
| +func (u *fix5Units) JournalTail(string, int) string { return "" } |
| +func (u *fix5Units) TryRestart(name string) error { |
| + u.events = append(u.events, "try-restart "+name) |
| + if u.tryError { |
| + u.tryError = false |
| + return errors.New("try-restart: bus timeout") |
| + } |
| + return u.units[name].TryRestart(name) |
| +} |
| +func (u *fix5Units) Restart(name string) error { |
| + u.events = append(u.events, "restart "+name) |
| + return u.units[name].Restart(name) |
| +} |
| +func (u *fix5Units) ResetFailed(name string) error { |
| + u.events = append(u.events, "reset-failed "+name) |
| + if u.resetError { |
| + return errors.New("reset-failed: bus timeout") |
| + } |
| + return u.units[name].ResetFailed(name) |
| +} |
| +func (u *fix5Units) Start(name string) error { |
| + u.events = append(u.events, "start "+name) |
| + return u.units[name].Start(name) |
| +} |
| + |
| +type fix5Result struct { |
| + Events []string |
| + Live, Previous string |
| + Route registry.Route |
| + Processes map[string]string |
| +} |
| + |
| +func fix5Run(t *testing.T, kind, history string, main bool) []fix5Result { |
| + t.Helper() |
| + sourceKind := kind |
| + if kind == "record" { |
| + sourceKind = "none" |
| + } |
| + if kind == "different stamp" { |
| + sourceKind = "list" |
| + } |
| + f := r2Fixture(t, sourceKind) |
| + |
| + if kind == "record" { |
| + f.rt.Account = "bab" |
| + plan14PutRecord(t, f.e.store, provision.Record{Schema: 1, Host: f.rt.Host, Account: "bab", App: f.rt.App, Kind: f.rt.Kind, Unit: f.rt.Unit, Addr: f.rt.Addr, Channel: f.rt.Channel, Health: &provision.Health{Path: "/ready"}}) |
| + } |
| + u := &fix5Units{units: map[string]*r2Units{f.rt.Unit: f.u}} |
| + target := f.u |
| + if kind == "different stamp" { |
| + copyUnit := *f.u.plan14RestartUnits |
| + copyUnit.props.Name, copyUnit.props.MainPID = "stamped.service", 700 |
| + target = &r2Units{plan14RestartUnits: ©Unit} |
| + f.rt.Unit = copyUnit.props.Name |
| + u.units[f.rt.Unit] = target |
| + } |
| + if err := f.e.root.reg.Upsert(f.rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.source.sys = u |
| + if kind != "none" && !(kind == "record" && os.Getenv("CARLOS_HOST_CLAIM") == "1") { |
| + if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { |
| + t.Fatal("fixture must bind", err) |
| + } |
| + } |
| + incident := strings.HasPrefix(history, "incident") || history == "TestR4ForwardResetFailedError" |
| + if incident { |
| + if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil { |
| + t.Fatal(err) |
| + } |
| + target.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App) |
| + target.props.ActiveState, target.props.SubState, target.props.MainPID = "failed", "failed", 0 |
| + f.promote(t, "v1") |
| + } |
| + switch history { |
| + case "crash", "failed-then-reset-failed before rollback": |
| + target.crashNew = true |
| + case "auto-restart": |
| + target.props.ActiveState, target.props.SubState, target.props.MainPID = "activating", "auto-restart", 0 |
| + case "inactive": |
| + target.props.ActiveState, target.props.SubState, target.props.MainPID = "inactive", "dead", 0 |
| + case "incident reset before": |
| + if err := u.ResetFailed(f.rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + case "TestR4ForwardResetFailedError": |
| + u.resetError = true |
| + case "TestR4ForwardTryRestartError": |
| + u.tryError = true |
| + case "interrupted repoint": |
| + // A root pass crashed after pointing v2 live and before restarting |
| + // v1's process; the next build, v3, then fails to come up. Main |
| + // rolls back to what the link names (v2), not to the running v1. |
| + account := release.RouteAccount(f.rt) |
| + if err := release.StageArtifact(f.u.opt, account, "v2", f.rt.App, []byte("new build")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(f.u.opt, f.rt.Host, account, "v2", f.rt.App); err != nil { |
| + t.Fatal(err) |
| + } |
| + m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| + if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.promote(t, "v3") |
| + v3 := release.VersionPath(f.u.opt, account, "v3", f.rt.App) |
| + target.afterEffect = func() { |
| + if target.exe == v3 { |
| + target.props.ActiveState, target.props.SubState, target.props.MainPID = "failed", "failed", 0 |
| + } |
| + } |
| + } |
| + getenv := func(string) string { return "5s" } |
| + observer := f.e.root.obs |
| + observer.resolve, observer.sys = f.c.resolve, u |
| + observer.procExe = func(pid int) string { exe, _ := u.Exe(pid); return exe } |
| + pr := f.e.root |
| + pr.obs = observer |
| + restart := rootRestartRouteWith(f.e.root.reg, u, f.u.opt, getenv, observer.recordSettled(f.e.bake), f.c) |
| + rollback := rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) |
| + startChecks := f.c |
| + startChecks.legacyStart = u.Start |
| + start := rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, startChecks) |
| + if main { |
| + restart = makeRestartRoute(f.e.root.reg, func(host string) { _ = u.TryRestart(activator.UnitName(host)) }, fix5MainCheckedRestart(u, f.u.opt, getenv, nil, f.c), nil) |
| + rollback = fix5MainRollbackRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) |
| + start = makeStartRoute(f.e.root.reg, u.Start) |
| + } else { |
| + // Binding may observe health and retain a running build, but cannot enter |
| + // the strict effect machinery or touch its durable restart journal. |
| + f.c.attempts = func() (rootstate.Store, error) { |
| + t.Fatal("no-census pass opened restart journal") |
| + return nil, os.ErrPermission |
| + } |
| + f.c.dispatch = func(context.Context, registry.Route) (bool, error) { |
| + t.Fatal("no-census pass dispatched binding") |
| + return false, os.ErrPermission |
| + } |
| + restart = rootRestartRouteWith(f.e.root.reg, u, f.u.opt, getenv, observer.recordSettled(f.e.bake), f.c) |
| + rollback = rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) |
| + startChecks = f.c |
| + startChecks.legacyStart = u.Start |
| + start = rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, startChecks) |
| + } |
| + originalRollback := rollback |
| + rollback = func(host string) error { |
| + if history == "failed-then-reset-failed before rollback" { |
| + if err := u.ResetFailed(f.rt.Unit); err != nil { |
| + return err |
| + } |
| + } |
| + return originalRollback(host) |
| + } |
| + snapshot := func() fix5Result { |
| + t.Helper() |
| + live, err := release.LiveVersion(f.u.opt, f.rt.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + previous, err := filepath.EvalSymlinks(filepath.Join(f.u.opt, "previous", f.rt.Host)) |
| + if err != nil && !os.IsNotExist(err) { |
| + t.Fatal(err) |
| + } |
| + rt, err := f.e.root.reg.Get(f.rt.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + rt.CreatedAt = 0 |
| + processes := map[string]string{} |
| + for name, unit := range u.units { |
| + processes[name] = strings.TrimPrefix(unit.exe, f.u.opt) + "|" + unit.props.ActiveState + "|" + unit.props.SubState + "|" + unit.props.InvocationID |
| + } |
| + return fix5Result{append([]string(nil), u.events...), live, strings.TrimPrefix(previous, f.u.opt), *rt, processes} |
| + } |
| + pass := func() { |
| + t.Helper() |
| + run := func() error { |
| + opts := []release.PassOption{release.WithStartUnit(start), release.WithRollbackRestart(rollback)} |
| + if !main { |
| + recovery := productionNamedUnitRecovery(f.e.store, f.pub, f.u.opt, hostwrite.Config{}) |
| + opts = append(opts, release.WithRootHistory(nil), release.WithRunningVersion(f.c.runningVersion), release.WithRestartAttemptSettled(f.c.finish), |
| + recovery.passOption(f.e.root.reg, f.c, getenv)) |
| + } |
| + return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, restart, opts...) |
| + } |
| + var err error |
| + if main { |
| + err = run() |
| + } else { |
| + err = pr.around(t.Context(), run) |
| + } |
| + if err != nil { |
| + t.Fatal("pass status differs from main", err) |
| + } |
| + } |
| + pass() |
| + results := []fix5Result{snapshot()} |
| + if incident { |
| + u.resetError = false |
| + if history != "incident reset before" { |
| + if err := u.ResetFailed(f.rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := u.Start(f.rt.Unit); err != nil { |
| + t.Fatal(err) |
| + } |
| + results = append(results, snapshot()) |
| + } |
| + // A second pass detects a synthetic refusal that would reject the fix. |
| + pass() |
| + return append(results, snapshot()) |
| +} |
| + |
| +func TestPlan14Fix5MainEquivalence(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, flag := range []string{"", "1"} { |
| + t.Run(map[string]string{"": "claims-off", "1": "claims-on"}[flag], func(t *testing.T) { |
| + t.Setenv("CARLOS_HOST_CLAIM", flag) |
| + for _, kind := range []string{"none", "record", "list", "guard", "different stamp"} { |
| + for _, history := range []string{"healthy", "crash", "failed-then-reset-failed before rollback", "auto-restart", "inactive", "incident reset before", "incident reset after", "TestR4ForwardResetFailedError", "TestR4ForwardTryRestartError", "interrupted repoint"} { |
| + t.Run(kind+"/"+history, func(t *testing.T) { |
| + want := fix5Run(t, kind, history, true) |
| + got := fix5Run(t, kind, history, false) |
| + if !reflect.DeepEqual(got, want) { |
| + t.Fatalf("no-census effects/pointers differ from main:\ngot %+v\nwant %+v", got, want) |
| + } |
| + }) |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestPlan14Fix5NoCensusSkipsJournalRecovery(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, flag := range []string{"", "1"} { |
| + t.Setenv("CARLOS_HOST_CLAIM", flag) |
| + f := r2Fixture(t, "list") |
| + f.c.attempts = func() (rootstate.Store, error) { |
| + t.Fatal("read retained journal without a census") |
| + return nil, os.ErrPermission |
| + } |
| + if err := recoverRootRestarts(t.Context(), f.u, f.u.opt, nil, f.c, hostwrite.Config{}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(f.u.effects) != 0 { |
| + t.Fatal(f.u.effects) |
| + } |
| + } |
| +} |
diff --git a/internal/host/plan14_fix5_main_linux_test.go b/internal/host/plan14_fix5_main_linux_test.go| new file mode 100644 |
| index 000000000..3ffea14fe |
| --- /dev/null |
| +++ b/internal/host/plan14_fix5_main_linux_test.go |
| @@ -0,0 +1,97 @@ |
| +package host |
| + |
| +import ( |
| + "errors" |
| + "fmt" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "log" |
| + "path/filepath" |
| + "time" |
| +) |
| + |
| +// Frozen from origin/main 34a4328f, internal/host/agent.go. Only names, |
| +// clock and readiness injection differ, so the oracle cannot drift with |
| +// the production legacy helpers. |
| +func fix5MainCheckedRestart(sys unitSystem, optDir string, getenv func(string) string, settled func(rt registry.Route), c rootRestartChecks) func(rt registry.Route) error { |
| + return func(rt registry.Route) error { |
| + before, err := sys.Show(rt.Unit) |
| + if err != nil { |
| + log.Printf("adopt: %s: cannot read %s (%v); restart not checked", rt.Host, rt.Unit, err) |
| + return sys.TryRestart(rt.Unit) |
| + } |
| + link := filepath.Join(optDir, "live", rt.Host) |
| + if before.ExecPath != link { |
| + log.Printf("adopt: binding: %s runs %s, but the unit stamp is on %s; restart not checked", rt.Unit, before.ExecPath, rt.Host) |
| + return sys.TryRestart(rt.Unit) |
| + } |
| + if before.ActiveState != "active" { |
| + return sys.TryRestart(rt.Unit) // stopped by an operator: stays a no-op |
| + } |
| + want, err := filepath.EvalSymlinks(link) |
| + if err != nil { |
| + return sys.TryRestart(rt.Unit) |
| + } |
| + if err := sys.TryRestart(rt.Unit); err != nil { |
| + return err |
| + } |
| + timeout := unitSettleTimeout(getenv) |
| + err = settleUnit(sys, rt.Unit, before.InvocationID, settleOpts{ |
| + Timeout: timeout, Poll: 500 * time.Millisecond, Hold: 5 * time.Second, |
| + WantDir: filepath.Dir(want), Ready: c.legacyReady(rt), |
| + Now: c.now, Sleep: c.sleep}) |
| + if errors.Is(err, errUnitStopped) { |
| + // An operator stopped it between the read above and now. Not a |
| + // verdict on the build, and a rollback restart would undo their |
| + // stop: same as a unit that was stopped before the restart. |
| + log.Printf("adopt: %s: %s was stopped during settle; not judged, not restarted", rt.Host, rt.Unit) |
| + return nil |
| + } |
| + if err == nil { |
| + log.Printf("adopt: %s: %s settled on %s", rt.Host, rt.Unit, filepath.Dir(want)) |
| + // D3 (proofobserve.go): a build that passed the settle check is a |
| + // build this box has run well. |
| + if settled != nil { |
| + settled(rt) |
| + } |
| + } |
| + return err |
| + } |
| +} |
| + |
| +// rollbackUnitRestart is D1's rollback restart (release.WithRollbackRestart): |
| +// the live link already names the restored build; replace whatever is |
| +// running (a deadline failure can leave the rejected build up) and check |
| +// the restored build comes up too. |
| +func fix5MainRollbackRestart(reg *registry.Registry, sys unitSystem, optDir string, getenv func(string) string, c rootRestartChecks) func(host string) error { |
| + return func(host string) error { |
| + rt, err := reg.Get(host) |
| + if err != nil || rt == nil || rt.Unit == "" { |
| + return nil |
| + } |
| + before, _ := sys.Show(rt.Unit) |
| + if err := sys.ResetFailed(rt.Unit); err != nil { |
| + log.Printf("adopt: %s: reset-failed %s: %v (continuing)", host, rt.Unit, err) |
| + } |
| + if err := sys.Restart(rt.Unit); err != nil { |
| + return err |
| + } |
| + want, err := filepath.EvalSymlinks(filepath.Join(optDir, "live", host)) |
| + if err != nil { |
| + return err |
| + } |
| + err = settleUnit(sys, rt.Unit, before.InvocationID, settleOpts{ |
| + Timeout: unitSettleTimeout(getenv), Poll: 500 * time.Millisecond, Hold: 5 * time.Second, |
| + WantDir: filepath.Dir(want), Ready: c.legacyReady(*rt), |
| + Now: c.now, Sleep: c.sleep}) |
| + if errors.Is(err, errUnitStopped) { |
| + // The restored build is what the link names; a stop an operator |
| + // made while it settled is theirs to keep, not a failed recovery. |
| + log.Printf("adopt: %s: %s was stopped during the rollback's settle; not judged, not restarted", host, rt.Unit) |
| + return nil |
| + } |
| + if err != nil { |
| + return fmt.Errorf("rollback did not come up either: %w", err) |
| + } |
| + return nil |
| + } |
| +} |
diff --git a/internal/host/plan14_history_retirement_test.go b/internal/host/plan14_history_retirement_test.go| new file mode 100644 |
| index 000000000..ce9422ffc |
| --- /dev/null |
| +++ b/internal/host/plan14_history_retirement_test.go |
| @@ -0,0 +1,227 @@ |
| +package host |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "encoding/json" |
| + "os" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +func TestPlan14RecoveryRetirementProductionCallback(t *testing.T) { |
| + for _, replacement := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "reconstruct", true: "replace"}[replacement], func(t *testing.T) { |
| + e, u, c, source, pub := signedRecoveryFixture(t) |
| + rt, err := e.root.reg.Get(observedHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + h := seedHostRootHistory(t, *rt) |
| + n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} |
| + if _, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg); err != nil { |
| + t.Fatal(err) |
| + } |
| + disk, err := c.attempts() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + good, err := readRollbackAttempt(disk, rt.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + old := good |
| + old.ID = strings.Repeat("1", 32) |
| + old.Verified = false |
| + attempt := *good.History |
| + attempt.ID = old.ID |
| + attempt.Version = "v0" |
| + attempt.SHA256 = "old-digest" |
| + attempt.Started = attempt.Started.Add(-time.Hour) |
| + attempt.PromotedAt = attempt.PromotedAt.Add(-time.Hour) |
| + old.History = &attempt |
| + if err := writeRollbackState(disk, recoveryResultKey(old), old); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := good.HistoryBinding |
| + signingPub, key, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + projection := &rollbackMemory{files: map[string][]byte{}} |
| + now := time.Now().UTC() |
| + maintenance := rootHistoryMaintenance{history: h, store: e.store, source: source, checks: c, pub: func() (ed25519.PublicKey, error) { return signingPub, nil }, projection: func() (rootstate.Store, error) { return projection, nil }, now: func() time.Time { return now }} |
| + deps := boundRootHistory(h, source.resolve, e.root.obs) |
| + deps.Reconstruct = maintenance.run |
| + if err := deps.Reconstruct(t.Context(), b); err == nil { |
| + t.Fatal("ambiguous original did not restrict reconstruction result") |
| + } |
| + if _, err := c.mintRefusals(h, b); err == nil { |
| + t.Fatal("unsigned ambiguity allowed mint") |
| + } |
| + var view release.HistoryMigrationStatus |
| + raw, err := projection.Read(release.HistoryStatusKey(b.Account, b.App)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := json.Unmarshal(raw, &view); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 2 { |
| + t.Fatalf("independent evidence missing from status: %+v", view) |
| + } |
| + grant := release.HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: view.Identity, Revision: view.Revision, Operation: "reconstruct-with-recovery", IssuedAt: now, Account: b.Account, App: b.App, AppRevision: view.AppRevision, BakeRevision: view.BakeRevision, Hosts: []release.HistoryReconstructionHost{view.Hosts[0].HistoryReconstructionHost}} |
| + if replacement { |
| + prior := grant |
| + prior.Operation = "interrupted-history" |
| + prior.Hosts = append([]release.HistoryReconstructionHost(nil), grant.Hosts...) |
| + prior.Hosts[0].Source = "previous-source" |
| + prior.Holds = map[string]release.HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}} |
| + wire, err := release.SignRootEnvelope(release.RootSigner(key), prior) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + base := h.Store |
| + h.Store = &historyIntentFailure{Store: base} |
| + if err := h.Reconstruct(signingPub, wire, b.Account, b.App, now, func(release.HistoryBinding) (string, error) { return "previous-source", nil }); err == nil { |
| + t.Fatal("missing interrupted intent") |
| + } |
| + h.Store = base |
| + if err := deps.Reconstruct(t.Context(), b); err == nil { |
| + t.Fatal("changed source resumed old intent") |
| + } |
| + raw, err := projection.Read(release.HistoryStatusKey(b.Account, b.App)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var pending release.HistoryMigrationStatus |
| + if err := json.Unmarshal(raw, &pending); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !pending.Pending || pending.PendingDigest == "" { |
| + t.Fatal("no pending status") |
| + } |
| + grant.Replaces = pending.PendingDigest |
| + } |
| + if err := release.PublishHistoryReconstruction(t.Context(), e.store, release.RootSigner(key), grant); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := deps.Reconstruct(t.Context(), b); err == nil { |
| + t.Fatal("signed reconstruction erased ambiguity") |
| + } |
| + retirement := release.HistoryRecoveryRetirement{Schema: 1, Kind: "carlos-release-recovery-retirement", Identity: view.Identity, Operation: "retire-exact-ambiguity", IssuedAt: now, Binding: b, Source: view.Hosts[0].Source, AttemptID: old.ID, OutcomeDigest: view.Hosts[0].RecoveryDigests[old.ID], VerifiedID: good.ID, VerifiedDigest: view.Hosts[0].RecoveryDigests[good.ID]} |
| + if err := release.PublishHistoryRecoveryRetirement(t.Context(), e.store, release.RootSigner(key), retirement); err != nil { |
| + t.Fatal(err) |
| + } |
| + before := len(u.effects) |
| + if err := deps.Reconstruct(t.Context(), b); err != nil { |
| + t.Fatal(err) |
| + } |
| + if refs, err := c.mintRefusals(h, b); err != nil || len(refs) != 0 { |
| + t.Fatalf("exact authorized retirement blocked mint: %+v %v", refs, err) |
| + } |
| + if len(u.effects) != before { |
| + t.Fatal("bookkeeping repeated a process effect") |
| + } |
| + raw, err = projection.Read(release.HistoryStatusKey(b.Account, b.App)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + view = release.HistoryMigrationStatus{} |
| + if err := json.Unmarshal(raw, &view); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(view.Hosts[0].RetiredRecoveries) != 1 || view.Hosts[0].RetiredRecoveries[0] != old.ID { |
| + t.Fatalf("retirement not projected: %+v", view) |
| + } |
| + // A stale settled copy cannot hide an outstanding source journal. |
| + old.Settled = false |
| + if err := writeRollbackState(disk, recoveryResultKey(old), old); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := c.mintRefusals(h, b); err == nil { |
| + t.Fatal("retirement consumed a pending original") |
| + } |
| + if err := deps.Reconstruct(t.Context(), b); err == nil { |
| + t.Fatal("maintenance consumed a pending original") |
| + } |
| + raw, err = projection.Read(release.HistoryStatusKey(b.Account, b.App)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + view = release.HistoryMigrationStatus{} |
| + if err := json.Unmarshal(raw, &view); err != nil { |
| + t.Fatal(err) |
| + } |
| + if view.RecoverySources == nil || !view.RecoverySources.Known || len(view.RecoverySources.Unresolved) != 1 || view.RecoverySources.Unresolved[0].Attempt.ID != old.ID || len(view.Hosts[0].RetiredRecoveries) != 0 { |
| + t.Fatalf("pending original hidden by projection: %+v", view) |
| + } |
| + |
| + }) |
| + } |
| +} |
| + |
| +func TestPlan14IndependentRecoveryPublishesStatus(t *testing.T) { |
| + for _, state := range []string{"success", "failed rollback", "history unavailable"} { |
| + t.Run(state, func(t *testing.T) { |
| + e, u, c, source, pub := signedRecoveryFixture(t) |
| + rt, err := e.root.reg.Get(observedHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + h := seedHostRootHistory(t, *rt) |
| + projection := &rollbackMemory{files: map[string][]byte{}} |
| + m := rootHistoryMaintenance{history: h, store: e.store, source: source, checks: c, pub: func() (ed25519.PublicKey, error) { return pub, nil }, projection: func() (rootstate.Store, error) { return projection, nil }, now: time.Now} |
| + deps := boundRootHistory(h, source.resolve, e.root.obs) |
| + deps.Reconstruct = m.run |
| + c.history = func() (*release.RootHistoryPass, error) { |
| + if state == "history unavailable" { |
| + return nil, os.ErrPermission |
| + } |
| + return deps, nil |
| + } |
| + u.crashNew, u.badRollback = state == "failed rollback", state == "failed rollback" |
| + n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} |
| + handled, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg) |
| + if !handled || (err != nil) != (state == "failed rollback") { |
| + t.Fatalf("history changed independent effect: %v %v", handled, err) |
| + } |
| + if state == "history unavailable" { |
| + if u.exe != release.VersionPath(u.opt, rt.Account, "v2", rt.App) { |
| + t.Fatal("unavailable history blocked fix") |
| + } |
| + return |
| + } |
| + raw, err := projection.Read(release.HistoryStatusKey(rt.Account, rt.App)) |
| + if err != nil { |
| + t.Fatal("independent status absent", err) |
| + } |
| + var view release.HistoryMigrationStatus |
| + if err := json.Unmarshal(raw, &view); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 1 { |
| + t.Fatalf("wrong projected outcomes: %+v", view) |
| + } |
| + outcome := view.Hosts[0].Recoveries[0] |
| + if !outcome.Settled || outcome.Verified != (state == "success") || outcome.Failed != (state == "failed rollback") { |
| + t.Fatalf("wrong outcome: %+v", outcome) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +type historyIntentFailure struct{ rootstate.Store } |
| + |
| +func (s *historyIntentFailure) Write(name string, raw []byte) error { |
| + if err := s.Store.Write(name, raw); err != nil { |
| + return err |
| + } |
| + if name == "reconstruction.state" && strings.Contains(string(raw), `"pending":`) { |
| + return os.ErrPermission |
| + } |
| + return nil |
| +} |
diff --git a/internal/host/plan14_recovery_pass_test.go b/internal/host/plan14_recovery_pass_test.go| new file mode 100644 |
| index 000000000..f18e73872 |
| --- /dev/null |
| +++ b/internal/host/plan14_recovery_pass_test.go |
| @@ -0,0 +1,60 @@ |
| +package host |
| + |
| +import ( |
| + "os" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func TestPlan14ERecoveryOrdinaryPass(t *testing.T) { |
| + if !plan14RootFixture(t) { |
| + return |
| + } |
| + for _, shape := range []string{"unstamped", "derived", "exec", "named refusal"} { |
| + t.Run(shape, func(t *testing.T) { |
| + f := r2Fixture(t, "none") |
| + f.rt.Unit = "" |
| + f.rt.Backing = registry.BackingUnit |
| + if shape == "derived" { |
| + f.rt.Unit = activator.UnitName(f.rt.Host) |
| + } |
| + if shape == "exec" { |
| + f.rt.Backing = "exec" |
| + } |
| + if err := f.e.root.reg.Delete(f.rt.Host); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.e.root.reg.Upsert(f.rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + reads := 0 |
| + f.c.legacy = func() (bool, error) { |
| + reads++ |
| + if shape != "named refusal" { |
| + t.Fatal("ordinary pass read census") |
| + } |
| + return false, os.ErrPermission |
| + } |
| + if shape == "named refusal" { |
| + named := f.rt |
| + named.Host = "failed.example.com" |
| + named.Unit = "failed.service" |
| + if err := f.e.root.reg.Upsert(named); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + n := productionNamedUnitRecovery(f.e.store, f.pub, f.u.opt, hostwrite.Config{}) |
| + if err := f.pass(t, release.WithRootHistory(nil), n.passOption(f.e.root.reg, f.c, func(string) string { return "5s" })); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.wantLive(t, "v2") |
| + if shape == "named refusal" && reads != 1 { |
| + t.Fatalf("recovery census reads=%d, want one", reads) |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/host/proofbinding.go b/internal/host/proofbinding.go| index f6505039e..3842344e9 100644 |
| --- a/internal/host/proofbinding.go |
| +++ b/internal/host/proofbinding.go |
| @@ -50,6 +50,7 @@ type proofBindingResolver func(context.Context, registry.Route) (proofBinding, e |
| type proofBindingSource struct { |
| indexMu sync.Mutex |
| index *provision.RecordIndex |
| + indexErr error |
| cacheIndex bool |
| checkPointers func(string) error |
| now func() time.Time |
diff --git a/internal/host/proofnamed.go b/internal/host/proofnamed.go| index 1678bb894..487c43940 100644 |
| --- a/internal/host/proofnamed.go |
| +++ b/internal/host/proofnamed.go |
| @@ -206,26 +206,11 @@ func (s *proofBindingSource) findRecord(ctx context.Context, host string) (*prov |
| } |
| s.indexMu.Lock() |
| defer s.indexMu.Unlock() |
| - if s.index == nil { |
| - var index *provision.RecordIndex |
| - var err error |
| - for attempt := 0; attempt < 3; attempt++ { |
| - index, err = provision.IndexRecords(ctx, s.store) |
| - if !errors.Is(err, provision.ErrRecordIndexRetry) { |
| - break |
| - } |
| - timer := time.NewTimer(time.Duration(attempt+1) * 25 * time.Millisecond) |
| - select { |
| - case <-ctx.Done(): |
| - timer.Stop() |
| - return nil, "", ctx.Err() |
| - case <-timer.C: |
| - } |
| - } |
| - if err != nil { |
| - return nil, "", err |
| - } |
| - s.index = index |
| + if s.index == nil && s.indexErr == nil { |
| + s.index, s.indexErr = provision.IndexRecords(ctx, s.store) |
| + } |
| + if s.indexErr != nil { |
| + return nil, "", s.indexErr |
| } |
| return s.index.Find(ctx, s.store, host) |
| } |
diff --git a/internal/host/proofobserve.go b/internal/host/proofobserve.go| index 011f08a76..0b7db3339 100644 |
| --- a/internal/host/proofobserve.go |
| +++ b/internal/host/proofobserve.go |
| @@ -20,6 +20,7 @@ import ( |
| "os" |
| "time" |
| + "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| ) |
| @@ -149,6 +150,9 @@ func recordSettled(sys unitSystem, procExe func(pid int) string, optDir, bakeDir |
| func (o proofObserver) recordSettled(history string) func(registry.Route) error { |
| return func(rt registry.Route) error { |
| + if rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { |
| + return nil |
| + } |
| b, err := o.observe(rt, 0) |
| if err != nil { |
| return err |
diff --git a/internal/host/proofroot.go b/internal/host/proofroot.go| index c703b38e4..ce6803bcc 100644 |
| --- a/internal/host/proofroot.go |
| +++ b/internal/host/proofroot.go |
| @@ -36,6 +36,7 @@ import ( |
| "sort" |
| "time" |
| + "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/safefile" |
| @@ -106,7 +107,7 @@ func (r proofRoot) recordLive(ctx context.Context) proofObservations { |
| return obs |
| } |
| for _, rt := range routes { |
| - if !release.Adoptable(rt) { |
| + if !release.Adoptable(rt) || rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { |
| continue |
| } |
| obs.byHost[rt.Host] = r.observeAndRecord(rt, pol.ProveAfter()) |
| @@ -180,7 +181,7 @@ func (r proofRoot) mintAndLog(ctx context.Context, obs proofObservations) { |
| type acctApp struct{ acct, app string } |
| seen := map[acctApp]bool{} |
| for _, rt := range routes { |
| - if !release.Adoptable(rt) { |
| + if !release.Adoptable(rt) || rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { |
| continue |
| } |
| acct := release.RouteAccount(rt) |
diff --git a/internal/host/recoveryrefusals.go b/internal/host/recoveryrefusals.go| index 74a147248..929bd8b15 100644 |
| --- a/internal/host/recoveryrefusals.go |
| +++ b/internal/host/recoveryrefusals.go |
| @@ -20,52 +20,75 @@ func recoveryHistoryOutcome(a rollbackAttempt) (release.HistoryRecovery, error) |
| return release.HistoryRecovery{Binding: a.HistoryBinding, Attempt: *a.History, Settled: a.Settled, Failed: a.Failed, Recovering: a.Recovering, Verified: a.Verified}, nil |
| } |
| -func (c rootRestartChecks) retainRecoveryHistory(h *release.RootHistory, binding release.HistoryBinding) error { |
| +func (c rootRestartChecks) copyRecoveryHistory(h *release.RootHistory, binding release.HistoryBinding) ([]release.HistoryRecovery, error) { |
| if c.attempts == nil || h == nil { |
| - return os.ErrPermission |
| + return nil, os.ErrPermission |
| } |
| s, err := c.attempts() |
| if err != nil { |
| - return err |
| + return nil, err |
| } |
| defer s.Close() |
| names, err := s.Names() |
| if err != nil { |
| - return err |
| + return nil, err |
| } |
| - unknown := false |
| + var unresolved []release.HistoryRecovery |
| for _, name := range names { |
| if !strings.HasSuffix(name, ".attempt") && !strings.HasSuffix(name, ".recovery") { |
| continue |
| } |
| raw, err := s.Read(name) |
| if err != nil { |
| - return err |
| + return nil, err |
| } |
| var a rollbackAttempt |
| if len(raw) > rootstate.MaxRecord || json.Unmarshal(raw, &a) != nil || (name != rollbackJournalKey(a.Host) && name != recoveryResultKey(a)) { |
| - return os.ErrPermission |
| + return nil, os.ErrPermission |
| } |
| if !a.Independent { |
| continue |
| } |
| r, err := recoveryHistoryOutcome(a) |
| if err != nil { |
| - return err |
| + return nil, err |
| } |
| if r.Binding.Account != binding.Account || r.Binding.App != binding.App { |
| continue |
| } |
| if err := s.Write(name, raw); err != nil { |
| - return err |
| + return nil, err |
| } |
| if err := h.RetainRecovery(r); err != nil { |
| - return err |
| + return nil, err |
| + } |
| + if !r.Settled || (!r.Failed && !r.Recovering && !r.Verified) { |
| + unresolved = append(unresolved, r) |
| } |
| - unknown = unknown || !r.Settled || (!r.Failed && !r.Recovering && !r.Verified) |
| } |
| - if unknown { |
| - return os.ErrPermission |
| + return unresolved, nil |
| +} |
| + |
| +func (c rootRestartChecks) retainRecoveryHistory(h *release.RootHistory, binding release.HistoryBinding) error { |
| + unresolved, err := c.copyRecoveryHistory(h, binding) |
| + if err != nil { |
| + return err |
| + } |
| + return checkUnresolvedRecoveries(h, unresolved) |
| +} |
| + |
| +func checkUnresolvedRecoveries(h *release.RootHistory, unresolved []release.HistoryRecovery) error { |
| + for _, r := range unresolved { |
| + if !r.Settled { |
| + return os.ErrPermission |
| + } |
| + retired, err := h.RecoveryRetired(r) |
| + if err != nil { |
| + return err |
| + } |
| + if !retired { |
| + return os.ErrPermission |
| + } |
| } |
| return nil |
| } |
diff --git a/internal/host/releaseinit_linux_test.go b/internal/host/releaseinit_linux_test.go| index f9c94da44..0ae5ffdca 100644 |
| --- a/internal/host/releaseinit_linux_test.go |
| +++ b/internal/host/releaseinit_linux_test.go |
| @@ -74,7 +74,10 @@ func TestPlan14ProductionReleaseInitialization(t *testing.T) { |
| if got := release.ReadHealthy(release.RootHealthyDir, "", "console"); len(got) != 1 || got[0].SHA256 != b.SHA256 { |
| t.Fatal(got) |
| } |
| - for path, mode := range map[string]os.FileMode{release.RootHealthyDir: 0755, filepath.Join(release.RootHealthyDir, "healthy"): 0755, filepath.Join(release.RootHealthyDir, "private"): 0700, rootRestartState: 0700} { |
| + if _, err := os.Stat(rootRestartState); !os.IsNotExist(err) { |
| + t.Fatal("no-census initialization created restart state", err) |
| + } |
| + for path, mode := range map[string]os.FileMode{release.RootHealthyDir: 0755, filepath.Join(release.RootHealthyDir, "healthy"): 0755} { |
| info, err := os.Stat(path) |
| if err != nil || info.Mode().Perm() != mode { |
| t.Fatalf("%s mode: %v %v", path, info, err) |
diff --git a/internal/host/roothistory.go b/internal/host/roothistory.go| index bf2480c99..66c243bb5 100644 |
| --- a/internal/host/roothistory.go |
| +++ b/internal/host/roothistory.go |
| @@ -2,11 +2,11 @@ package host |
| import ( |
| "context" |
| + "crypto/ed25519" |
| "crypto/sha256" |
| "debug/buildinfo" |
| "encoding/hex" |
| "io" |
| - "log" |
| "os" |
| "time" |
| @@ -22,36 +22,16 @@ func openRootHistory(store release.Store, optDir string) (*release.RootHistoryPa |
| } |
| source := productionProofBindingSource(store, systemdUnits{}, optDir) |
| h := boundRootHistory(&release.RootHistory{Store: s}, source.resolve, productionProofObserver(optDir)) |
| - h.Reconstruct = func(ctx context.Context, b release.HistoryBinding) error { |
| - pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| - if err != nil { |
| - return err |
| - } |
| - resolve := func(want release.HistoryBinding) (string, error) { |
| - got, err := source.forHost(ctx, want.Host) |
| - if err != nil { |
| - return "", err |
| - } |
| - if got.Route.Account != want.Account || got.Route.App != want.App { |
| - return "", os.ErrPermission |
| - } |
| - return got.Source, nil |
| - } |
| - checks := productionRootRestartChecks(systemdUnits{}, optDir) |
| - err = checks.retainRecoveryHistory(h.History, b) |
| - if err == nil { |
| - err = h.History.ReceiveReconstruction(ctx, store, pub, b, time.Now(), resolve) |
| - } |
| - projection, statusErr := rootstate.Open(release.RootHistoryStatusDir, true, true) |
| - if statusErr == nil { |
| - statusErr = h.History.ProjectMigrationStatus(projection, b, resolve) |
| - projection.Close() |
| - } |
| - if statusErr != nil { |
| - log.Print(statusErr) |
| - } |
| - return err |
| + m := rootHistoryMaintenance{history: h.History, store: store, source: source, |
| + checks: productionRootRestartChecks(systemdUnits{}, optDir), now: time.Now, |
| + pub: func() (ed25519.PublicKey, error) { |
| + return release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| + }, |
| + projection: func() (rootstate.Store, error) { |
| + return rootstate.Open(release.RootHistoryStatusDir, true, true) |
| + }, |
| } |
| + h.Reconstruct = m.run |
| return h, nil |
| } |
diff --git a/internal/host/roothistory_maintenance.go b/internal/host/roothistory_maintenance.go| new file mode 100644 |
| index 000000000..33ae892ec |
| --- /dev/null |
| +++ b/internal/host/roothistory_maintenance.go |
| @@ -0,0 +1,63 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "log" |
| + "os" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +type rootHistoryMaintenance struct { |
| + history *release.RootHistory |
| + store release.Store |
| + source *proofBindingSource |
| + checks rootRestartChecks |
| + pub func() (ed25519.PublicKey, error) |
| + projection func() (rootstate.Store, error) |
| + now func() time.Time |
| +} |
| + |
| +func (m rootHistoryMaintenance) run(ctx context.Context, b release.HistoryBinding) error { |
| + resolve := func(want release.HistoryBinding) (string, error) { |
| + _, got, _, err := m.source.unitBinding(ctx, want.Host) |
| + if err != nil { |
| + return "", err |
| + } |
| + if got.Route.Account != want.Account || got.Route.App != want.App { |
| + return "", os.ErrPermission |
| + } |
| + return got.Source, nil |
| + } |
| + sources := release.HistoryRecoverySources{} |
| + defer func() { |
| + projection, err := m.projection() |
| + if err == nil { |
| + err = m.history.ProjectMigrationStatus(projection, b, resolve, sources) |
| + projection.Close() |
| + } |
| + if err != nil { |
| + log.Print(err) |
| + } |
| + }() |
| + unresolved, err := m.checks.copyRecoveryHistory(m.history, b) |
| + sources.Known, sources.Unresolved = err == nil, unresolved |
| + if err != nil { |
| + return err |
| + } |
| + pub, err := m.pub() |
| + if err != nil { |
| + return err |
| + } |
| + now := m.now() |
| + if err := m.history.ReceiveReconstruction(ctx, m.store, pub, b, now, resolve); err != nil { |
| + return err |
| + } |
| + if err := m.history.ReceiveRecoveryRetirements(ctx, m.store, pub, b, now, resolve); err != nil { |
| + return err |
| + } |
| + return checkUnresolvedRecoveries(m.history, unresolved) |
| +} |
diff --git a/internal/host/roothistory_recovery.go b/internal/host/roothistory_recovery.go| index 5a4c38b77..62f9c4264 100644 |
| --- a/internal/host/roothistory_recovery.go |
| +++ b/internal/host/roothistory_recovery.go |
| @@ -83,6 +83,9 @@ func (c rootRestartChecks) settleHistory(ctx context.Context, a rollbackAttempt, |
| outcome, err = recoveryHistoryOutcome(a) |
| if err == nil { |
| err = h.History.RetainRecovery(outcome) |
| + if err == nil && h.Reconstruct != nil { |
| + err = h.Reconstruct(ctx, outcome.Binding) |
| + } |
| } |
| } |
| if err != nil { |
diff --git a/internal/host/rootrestart.go b/internal/host/rootrestart.go| index 00eec5b6d..41db59b68 100644 |
| --- a/internal/host/rootrestart.go |
| +++ b/internal/host/rootrestart.go |
| @@ -50,13 +50,17 @@ type rootRestartChecks struct { |
| func productionRootRestartChecks(sys unitSystem, optDir string) rootRestartChecks { |
| var sourceMu sync.Mutex |
| var cached *proofBindingSource |
| + var sourceErr error |
| + var opened bool |
| source := func(ctx context.Context) (*proofBindingSource, error) { |
| sourceMu.Lock() |
| defer sourceMu.Unlock() |
| - if cached != nil { |
| - return cached, nil |
| + if opened { |
| + return cached, sourceErr |
| } |
| + opened = true |
| s, err := openDeployStore(ctx) |
| + sourceErr = err |
| if err != nil { |
| return nil, err |
| } |
| @@ -128,15 +132,17 @@ func (c rootRestartChecks) runningVersion(ctx context.Context, rt registry.Route |
| if rt.Unit == "" || rt.Unit == activator.UnitName(rt.Host) { |
| return "", nil |
| } |
| + // Merge ruling 6: without a census the binding never chooses an |
| + // effect, and the rollback target is one. Main's cur stands. |
| + legacy, err := c.allowLegacy() |
| + if err != nil { |
| + return "", err |
| + } |
| + if legacy { |
| + return "", nil |
| + } |
| b, err := c.resolve(ctx, rt) |
| if err != nil { |
| - legacy, checkErr := c.allowLegacy() |
| - if checkErr != nil { |
| - return "", checkErr |
| - } |
| - if legacy { |
| - return "", nil |
| - } |
| return "", err |
| } |
| if b.PID > 0 { |
| @@ -151,6 +157,15 @@ func checkedUnitRestart(sys unitSystem, optDir string, getenv func(string) strin |
| c = checks[0] |
| } |
| return func(candidate registry.Route) error { |
| + c := c |
| + legacy, err := c.allowLegacy() |
| + if err != nil { |
| + return err |
| + } |
| + if legacy { |
| + c.noteLegacy() |
| + return legacyCheckedUnitRestart(sys, optDir, getenv, candidate, settled, c) |
| + } |
| if c.attempts == nil { |
| return os.ErrPermission |
| } |
| @@ -250,7 +265,7 @@ func checkedUnitRestart(sys unitSystem, optDir string, getenv func(string) strin |
| if errors.Is(err, errUnitStopped) { |
| return errors.Join(err, settleRollbackAttempt(s, attempt)) |
| } |
| - if errors.Is(err, errRestartBinding) { |
| + if errors.Is(err, errRestartBinding) && !restartEffectFailed(err) { |
| return errors.Join(err, settleRollbackAttempt(s, attempt)) |
| } |
| attempt.Failed = true |
| @@ -369,7 +384,7 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string, |
| WantDir: filepath.Dir(want), Ready: checkedReady, Now: c.now, Sleep: c.sleep, RequireExe: true, |
| }) |
| if err != nil { |
| - if bindingErr != nil { |
| + if bindingErr != nil && !restartEffectFailed(err) { |
| if c.preserveOutcome { |
| return errors.Join(err, bindingErr) |
| } |
| @@ -406,27 +421,30 @@ func rollbackUnitRestart(reg *registry.Registry, sys unitSystem, optDir string, |
| } |
| return func(host string) (result error) { |
| + c := c |
| + rt, routeErr := reg.Get(host) |
| + if rt != nil && (rt.Unit == "" || (c.legacyStart != nil && rt.Hibernate)) { |
| + return nil |
| + } |
| + if rt != nil && rt.Unit == activator.UnitName(host) { |
| + if c.legacyStart != nil { |
| + return c.legacyStart(rt.Unit) |
| + } |
| + return legacyRollbackRestart(sys, optDir, getenv, *rt, c) |
| + } |
| legacy, err := c.allowLegacy() |
| if err != nil { |
| return err |
| } |
| if legacy { |
| - rt, err := reg.Get(host) |
| - if err != nil || rt == nil || rt.Unit == "" { |
| + c.noteLegacy() |
| + if routeErr != nil || rt == nil { |
| return nil |
| } |
| - defer func() { |
| - if errors.Is(result, errUnitStopped) { |
| - result = nil |
| - } else if result != nil && !restartEffectFailed(result) { |
| - c.noteLegacy() |
| - if c.legacyStart != nil { |
| - result = c.legacyStart(rt.Unit) |
| - } else { |
| - result = legacyRollbackRestart(sys, optDir, getenv, *rt, c) |
| - } |
| - } |
| - }() |
| + if c.legacyStart != nil { |
| + return c.legacyStart(rt.Unit) |
| + } |
| + return legacyRollbackRestart(sys, optDir, getenv, *rt, c) |
| } |
| if c.attempts == nil || c.rollback == nil { |
| return os.ErrPermission |
| @@ -443,7 +461,9 @@ func rollbackUnitRestart(reg *registry.Registry, sys unitSystem, optDir string, |
| defer unlock() |
| attempt, binding, err := loadRollback(s, host, c.now()) |
| if attempt.ID != "" { |
| - defer func() { result = errors.Join(result, settleRollbackAttempt(s, attempt)) }() |
| + defer func() { |
| + result = errors.Join(result, settleRollbackAttempt(s, attempt)) |
| + }() |
| } |
| if err != nil { |
| if !errors.Is(err, os.ErrNotExist) && !attempt.Settled && (attempt.ID == "" || !c.now().After(attempt.Deadline.Add(rollbackMargin))) { |
| @@ -521,34 +541,27 @@ func rootSelectedRestart(reg *registry.Registry, sys unitSystem, optDir string, |
| func rootRestartRouteWith(reg *registry.Registry, sys unitSystem, optDir string, getenv func(string) string, settled func(registry.Route) error, checks rootRestartChecks) func(string) error { |
| restart := checkedUnitRestart(sys, optDir, getenv, settled, checks) |
| return func(host string) error { |
| - legacy, err := checks.allowLegacy() |
| - if err != nil { |
| - return err |
| - } |
| rt, err := reg.Get(host) |
| - if legacy && (err != nil || rt == nil || rt.Unit == "" || rt.Unit == activator.UnitName(host)) { |
| + if err != nil || rt == nil || rt.Unit == "" { |
| return restartOrdinaryUnit(sys, host) |
| } |
| + if rt.Unit == activator.UnitName(host) { |
| + return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) |
| + } |
| + legacy, err := checks.allowLegacy() |
| if err != nil { |
| return err |
| } |
| - if rt == nil { |
| - return os.ErrPermission |
| + if legacy { |
| + checks.noteLegacy() |
| + return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) |
| } |
| if checks.dispatch != nil { |
| need, err := checks.dispatch(context.Background(), *rt) |
| if err != nil { |
| - if legacy { |
| - checks.noteLegacy() |
| - return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) |
| - } |
| return err |
| } |
| if !need { |
| - if legacy { |
| - checks.noteLegacy() |
| - return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) |
| - } |
| return restartVerifiedOrdinaryUnit(sys, optDir, host, checks.targetFile) |
| } |
| } |
| @@ -556,10 +569,6 @@ func rootRestartRouteWith(reg *registry.Registry, sys unitSystem, optDir string, |
| if errors.Is(err, errUnitStopped) { |
| return nil |
| } |
| - if legacy && err != nil && !restartEffectFailed(err) { |
| - checks.noteLegacy() |
| - return legacyCheckedUnitRestart(sys, optDir, getenv, *rt, settled, checks) |
| - } |
| return err |
| } |
| } |
diff --git a/internal/host/rootrollback_resume.go b/internal/host/rootrollback_resume.go| index cfcb96109..988c9b115 100644 |
| --- a/internal/host/rootrollback_resume.go |
| +++ b/internal/host/rootrollback_resume.go |
| @@ -13,6 +13,16 @@ import ( |
| ) |
| func recoverRootRestarts(ctx context.Context, sys unitSystem, optDir string, getenv func(string) string, checks rootRestartChecks, writes hostwrite.Config) error { |
| + legacy, err := checks.allowLegacy() |
| + if err != nil { |
| + return err |
| + } |
| + if legacy { |
| + checks.noteLegacy() |
| + // D3 still records root health without a census. Preparing its |
| + // public parent must not create or recover bound restart state. |
| + return rootstate.EnsurePublicDir(release.RootHealthyDir) |
| + } |
| if checks.attempts == nil || checks.prepareRollback == nil { |
| return os.ErrPermission |
| } |
diff --git a/internal/provision/plan14_e_write_census_test.go b/internal/provision/plan14_e_write_census_test.go| index 1e654fb8c..e4395a974 100644 |
| --- a/internal/provision/plan14_e_write_census_test.go |
| +++ b/internal/provision/plan14_e_write_census_test.go |
| @@ -2,6 +2,7 @@ package provision |
| // Track E overlays the reviewed Part 1 write census without changing its baseline. |
| var plan14EWriteCensusOverlay = []censusEntry{ |
| + {"internal/release/roothistory_delivery.go", "PublishHistoryRecoveryRetirement", "nonrecord", "sel:Put", "ec499a70f267b5b551e918c7ebb7ce36745afa87f77160fe9dc826d8dbe3032b"}, |
| {"internal/host/proofbinding.go", "proofBindingSource.binding", "key-site", "key:RecordKey", "56edaf01411de3813fe327973302b1df35e403d2870625fc341355b5bc5cfa0b"}, |
| {"internal/host/proofbinding.go", "proofBindingSource.unitBinding", "key-site", "key:RecordKey", "fc6bbd54101f36b986ea48656973667139575d20b75780ad3892915904ef5151"}, |
| {"internal/host/proofbinding.go", "type proofBinding", "key-site", "key:RecordKey", "e5de1305f80b32368afbb70cc44dcf46628b9ad79004046010de3edcbbd401f6"}, |
diff --git a/internal/provision/plan14_legacy_identity_test.go b/internal/provision/plan14_legacy_identity_test.go| index 6166ecb63..f96fea929 100644 |
| --- a/internal/provision/plan14_legacy_identity_test.go |
| +++ b/internal/provision/plan14_legacy_identity_test.go |
| @@ -151,3 +151,46 @@ func TestPlan14Merge3OverlayPreservesLegacySeal(t *testing.T) { |
| t.Fatal("unapproved legacy fingerprint accepted") |
| } |
| } |
| + |
| +func TestPlan14Fix4OverlayPreservesLegacySeal(t *testing.T) { |
| + root := censusRoot(t) |
| + inv := rootReadInventory{Entries: map[string]classifiedRootRead{}, Coverage: map[string]string{}} |
| + for i, name := range []string{"plan14-root-reads.tsv", "plan14-root-reads-moves.tsv", "plan14-root-reads-merge.tsv", "plan14-root-reads-fix1.tsv", "plan14-root-reads-fix2.tsv", "plan14-root-reads-merge3.tsv"} { |
| + if err := loadRootReadInventory(root+"/internal/provision/testdata/"+name, &inv, i > 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + seal := rootReadLegacySeal(inv) |
| + if err := loadRootReadInventory(root+"/internal/provision/testdata/plan14-root-reads-fix4.tsv", &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 { |
| + t.Fatal(errs) |
| + } |
| + if seal != "65dba959f80cc47e9fd99be145ba8b24bb3b37ffaf77b47d1b9cc3e01da14b47" { |
| + t.Fatal("unexpected legacy seal", seal) |
| + } |
| +} |
| + |
| +func TestPlan14Fix5OverlayPreservesLegacySeal(t *testing.T) { |
| + root := censusRoot(t) |
| + inv := rootReadInventory{Entries: map[string]classifiedRootRead{}, Coverage: map[string]string{}} |
| + for i, name := range []string{"plan14-root-reads.tsv", "plan14-root-reads-moves.tsv", "plan14-root-reads-merge.tsv", "plan14-root-reads-fix1.tsv", "plan14-root-reads-fix2.tsv", "plan14-root-reads-merge3.tsv", "plan14-root-reads-fix4.tsv"} { |
| + if err := loadRootReadInventory(root+"/internal/provision/testdata/"+name, &inv, i > 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + seal := rootReadLegacySeal(inv) |
| + if err := loadRootReadInventory(root+"/internal/provision/testdata/plan14-root-reads-fix5.tsv", &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := loadRootReadInventory(root+"/internal/provision/testdata/plan14-root-reads-fix6.tsv", &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 { |
| + t.Fatal(errs) |
| + } |
| + if seal != "65dba959f80cc47e9fd99be145ba8b24bb3b37ffaf77b47d1b9cc3e01da14b47" { |
| + t.Fatal("unexpected legacy seal", seal) |
| + } |
| +} |
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go| index 07ecd5c6a..e9fb7effb 100644 |
| --- a/internal/provision/plan14_root_reads_test.go |
| +++ b/internal/provision/plan14_root_reads_test.go |
| @@ -628,6 +628,18 @@ func TestPlan14RootReadCensus(t *testing.T) { |
| t.Fatal(err) |
| } |
| seal = "65dba959f80cc47e9fd99be145ba8b24bb3b37ffaf77b47d1b9cc3e01da14b47" |
| + if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 { |
| + t.Fatal(errs) |
| + } |
| + if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-fix4.tsv"), &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-fix5.tsv"), &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-fix6.tsv"), &inv, true); err != nil { |
| + t.Fatal(err) |
| + } |
| if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-e.tsv"), &inv, true); err != nil { |
| t.Fatal(err) |
| } |
diff --git a/internal/provision/testdata/plan14-root-reads-e.tsv b/internal/provision/testdata/plan14-root-reads-e.tsv| index fc2a7b505..5d6bd2b37 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-e.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-e.tsv |
| @@ -1,5 +1,5 @@ |
| # Track E integration after merge3. Earlier baselines, overlays and legacy fingerprints are immutable. |
| -internal/host/agent.go runAdoptUnderRootRelease authority B3,E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. registry:Host@14260,whole:Route@10431,whole:Route@17902,whole:Route@17938,whole:Route@7852 27e449dcb59c49398197e2c2ce2f49a9a12747bd6a48200728bd53d9a7ed40c3 |
| +internal/host/agent.go runAdoptUnderRootRelease authority B3,E1 Production recovery wrapper preserves main exit status for per-host failures; only remaining named routes can probe the census for history selection. Ordinary routes make no census or binding-store reads. Fixtures: TestPlan14ERecoveryOrdinaryPass, TestPlan14Fix4AdoptPerHostFailureExitsZero, TestPlan14Fix5MainEquivalence. whole:Route@10480,whole:Route@17661,whole:Route@17697,whole:Route@7901 c5c055aad44dd599334e4c37a958b836987c19ae69f2a386986b4839776266d0 |
| internal/host/agent.go runAdoptWithEnsureSystemUnits authority B3,C,D,E1,E2,E3,E4,F,J1 E1 selects the checked private history, canonical named-unit provider and root mint inventory under release exclusion with claims off or on; reconstruction and restart consumption remain incomplete. file:Open@191 9cdd0a958f5924c9c8d8c5613fd665f2c96a194cd11322adb6f0050fb183b541 |
| internal/host/namedrecovery.go namedUnitRecovery.repair authority E1,J1 Signed channel target, canonical deployment record, checked named unit and root-owned live pointer authorize stopped-unit pointer repair. Rechecks source, claim selection, pointer and artifact at the effect; does not start a process or change grants. Fixtures: TestPlan14Incident20261006WithLedger and TestPlan14SignedNamedRecoveryAuthority. Uses account/app from the same canonical read as the unit binding. Fixture: TestPlan14SignedNamedRecoveryUsesOneCanonicalRecord. file:Open@2565,registry:Channel@1814,whole:Route@1060,whole:Route@1822 8a42a6a27537be49d30d32891f407789449f9b77d37674fea5cb288d908adf6a |
| internal/host/namedrecovery_active.go namedUnitRecovery.recover authority B3,E1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. file:Open@5386,registry:Channel@2511,registry:Unit@216,registry:Unit@233,registry:Version@3500,registry:Version@3606,whole:Route@1431,whole:Route@202,whole:Route@2513,whole:Route@3426 22054f9fecd0e7c896087188ba5f9cf84ffdf3f5d8afca7574f29569b9311332 |
| @@ -9,23 +9,23 @@ internal/host/proofbinding.go proofBindingSource.recovered authority B3,E1 Signe |
| internal/host/proofbinding.go proofBindingSource.unitBinding authority B3,J1,E1 Reads canonical deployment record, claim, unit file identities and systemd facts without assuming a running PID. Existing running/rollback binding uses the same verifier. Fixtures: TestPlan14SignedNamedRecoveryAuthority and TestPlan14ProofBindingProvenance. registry:Unit@508,registry:Unit@609,registry:Unit@800,whole:Route@2707,whole:Route@2714 fc6bbd54101f36b986ea48656973667139575d20b75780ad3892915904ef5151 |
| internal/host/proofroot.go proofRoot.mint authority B3,J1,J4,E1 E1 production uses canonical observed programs and complete root refusal inventory; legacy diagnostics do not grant root mint authority. registry:Account@136,registry:App@1338,registry:App@1960,registry:App@2075,registry:App@2279,registry:App@316,registry:App@934,registry:Host@1135,registry:Host@1259,registry:Host@1599,registry:Host@2066,registry:Host@2189,registry:Host@287,registry:Host@469,registry:Host@622,whole:Route@129,whole:Route@144 8c4e2457e74ac917a5be303d74493247190e344585da622e989048fd0e00e61a |
| internal/host/proofroot.go proofRoot.observeAndRecord authority B3,J1,J4,E1 E1 production uses canonical observed programs and complete root refusal inventory; legacy diagnostics do not grant root mint authority. file:Readlink@108,registry:Account@443,registry:App@468,registry:Host@151,registry:Host@657,whole:Route@186,whole:Route@453,whole:Route@478 88065f1cfe7ad31178ba65c2672d8dd1e25fab26820290368fc063907f7b3b98 |
| -internal/host/recoveryrefusals.go rootRestartChecks.mintRefusals authority B3,E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 8087848fae0c92cc94908a7565019249aac203dbb3efbc313fafdf4b1a55b49d |
| -internal/host/roothistory.go boundRootHistory authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. file:Open@1061,file:Read@1379,registry:Account@613,registry:App@635,registry:Channel@456,registry:Channel@470,registry:Host@590,whole:Route@1469,whole:Route@377,whole:Route@472,whole:Route@592,whole:Route@615,whole:Route@637,whole:Route@846 fe46269101116374044658bbb87c1bd44317525dcadf227d4515b47845238be2 |
| -internal/host/roothistory.go openRootHistory authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Open@104,file:Open@1118,registry:Account@740,registry:App@777,whole:Route@744,whole:Route@781 28c836abc244d5bca5c22db2ed565b8d0e60e381f846401ab6969f339a1c67fa |
| -internal/host/roothistory.go recordRootProgram authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. whole:Route@151,whole:Route@297,whole:Route@398 fbbc9451c39309090580486d75da6120b02e1ba054157159846bea057ae0c80f |
| -internal/host/roothistory_recovery.go historyRestartAttempt authority B3,E1 Captured root capability and exact process/source identity authorize restart independently of release history availability. Fixture: TestPlan14HistoryUnavailableAfterCaptureCannotBlockRestart. registry:Host@187,registry:Host@361,whole:Route@189,whole:Route@363 1c2af7b90cb56053c1d58c10ccaa851c3aa25b0be9e595bf8e90a013ad0bae66 |
| -internal/host/roothistory_recovery.go linkRootHistory authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Prepare@121,registry:Account@374,registry:App@412,whole:Route@376,whole:Route@414 a60324796d2b25b3b130bb0439bbd9a415b679e6b51bc92a44f23f77e2d93e1e |
| -internal/host/roothistory_recovery.go resumeHistoryRestart authority B3,E1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. registry:Account@1202,registry:Account@2371,registry:App@1234,registry:App@2403,registry:Unit@1001,whole:Route@1005,whole:Route@1206,whole:Route@1238,whole:Route@2375,whole:Route@2407 8263cf0571efdd448b944fdb8492ce78ddf0223fdef18255eab97c40434eef60 |
| -internal/host/roothistory_recovery.go rootRestartChecks.settleHistory authority B3,E1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. registry:Account@911,registry:App@1682,registry:App@958,whole:Route@1176,whole:Route@1322,whole:Route@1684,whole:Route@913,whole:Route@960 72c220e95eff0ea648de00acf7da10cdb49842e9a949d074dd267622f92eb03a |
| -internal/host/rootrestart.go checkedUnitRestart authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@1798,registry:Account@1367,registry:Account@2166,registry:App@1399,registry:App@2198,registry:Host@1069,registry:Host@1854,registry:Host@2741,registry:Host@2963,registry:Host@3606,registry:Host@510,registry:Unit@3488,registry:Unit@711,whole:Route@1375,whole:Route@1407,whole:Route@2174,whole:Route@2206,whole:Route@3496,whole:Route@638,whole:Route@719 3a29634563392118e927685e478c8265916fbe89d922b9dde4c08aa5178c1c3c |
| -internal/host/rootrestart.go productionRootRestartChecks authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Open@1356,whole:Route@1189,whole:Route@1990,whole:Route@2257 921445f1e11f9bfd202e309a0b0be4864f057a0907164f32ec8603f6949f4bdd |
| -internal/host/rootrestart.go restartBoundUnit authority B3,J1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. registry:Account@2482,registry:Account@3967,registry:App@2518,registry:App@4007,registry:Host@1585,registry:Host@2301,registry:Host@650,registry:Unit@1071,registry:Unit@1139,registry:Unit@1894,registry:Unit@1962,registry:Unit@2004,registry:Unit@3299,registry:Unit@395,whole:Route@198,whole:Route@2488,whole:Route@2524,whole:Route@317,whole:Route@3975,whole:Route@4015,whole:Route@4226 e379b31272674c4983e0d0aea66a1091d6537ece5ec05b64474cc292a8f40ba4 |
| -internal/host/rootrestart.go rollbackUnitRestart authority B3,J1,E1 Captured canonical rollback authority is unchanged. Durable verified outcome permits later history settlement without another restart. Fixtures: TestPlan14HistoryRollbackLink and TestPlan14HistoryUnavailableCannotBlockCapturedRollback. file:Read@1458,registry:Host@1750,registry:Unit@1852,registry:Unit@2153,registry:Unit@435,registry:Unit@692,whole:Route@1758,whole:Route@1860,whole:Route@2161,whole:Route@422,whole:Route@774 4d02504999ba2746f3815cba0e3ca7fe8723cdf2528e57d3fa78b5acc9b90998 |
| +internal/host/recoveryrefusals.go rootRestartChecks.mintRefusals authority B3,E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 8087848fae0c92cc94908a7565019249aac203dbb3efbc313fafdf4b1a55b49d |
| +internal/host/roothistory.go boundRootHistory authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Open@1061,file:Read@1379,registry:Account@613,registry:App@635,registry:Channel@456,registry:Channel@470,registry:Host@590,whole:Route@1469,whole:Route@377,whole:Route@472,whole:Route@592,whole:Route@615,whole:Route@637,whole:Route@846 fe46269101116374044658bbb87c1bd44317525dcadf227d4515b47845238be2 |
| +internal/host/roothistory.go openRootHistory authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Open@104,file:Open@726 6bf6a245da1bf921eccae36aa3644155f688b02703e98951e23e0064bf4b5eff |
| +internal/host/roothistory.go recordRootProgram authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. whole:Route@151,whole:Route@297,whole:Route@398 fbbc9451c39309090580486d75da6120b02e1ba054157159846bea057ae0c80f |
| +internal/host/roothistory_recovery.go historyRestartAttempt authority B3,E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. registry:Host@187,registry:Host@361,whole:Route@189,whole:Route@363 1c2af7b90cb56053c1d58c10ccaa851c3aa25b0be9e595bf8e90a013ad0bae66 |
| +internal/host/roothistory_recovery.go linkRootHistory authority B3,E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Prepare@121,registry:Account@374,registry:App@412,whole:Route@376,whole:Route@414 a60324796d2b25b3b130bb0439bbd9a415b679e6b51bc92a44f23f77e2d93e1e |
| +internal/host/roothistory_recovery.go resumeHistoryRestart authority B3,E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. registry:Account@1202,registry:Account@2371,registry:App@1234,registry:App@2403,registry:Unit@1001,whole:Route@1005,whole:Route@1206,whole:Route@1238,whole:Route@2375,whole:Route@2407 8263cf0571efdd448b944fdb8492ce78ddf0223fdef18255eab97c40434eef60 |
| +internal/host/roothistory_recovery.go rootRestartChecks.settleHistory authority B3,E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. registry:Account@1008,registry:App@1055,registry:App@1779,whole:Route@1010,whole:Route@1057,whole:Route@1273,whole:Route@1419,whole:Route@1781 ff5cb1baa6f139194134de67159d12a5431672c32388314b221ed0da8065feb9 |
| +internal/host/rootrestart.go checkedUnitRestart authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@1991,registry:Account@1560,registry:Account@2359,registry:App@1592,registry:App@2391,registry:Host@1262,registry:Host@2047,registry:Host@2934,registry:Host@3156,registry:Host@3828,registry:Host@703,registry:Unit@3710,registry:Unit@904,whole:Route@1568,whole:Route@1600,whole:Route@2367,whole:Route@2399,whole:Route@3718,whole:Route@479,whole:Route@831,whole:Route@912 610cd677cd158c30816eafef47c572405d9d2f82ead0c1b247de6605c118f3c7 |
| +internal/host/rootrestart.go productionRootRestartChecks authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Open@1427,whole:Route@1260,whole:Route@2061,whole:Route@2328 729fe4f571f3eaf40d9880304c3d2db924625de3bbeba709cac03a67f5baa49e |
| +internal/host/rootrestart.go restartBoundUnit authority B3,J1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. registry:Account@2482,registry:Account@3996,registry:App@2518,registry:App@4036,registry:Host@1585,registry:Host@2301,registry:Host@650,registry:Unit@1071,registry:Unit@1139,registry:Unit@1894,registry:Unit@1962,registry:Unit@2004,registry:Unit@3299,registry:Unit@395,whole:Route@198,whole:Route@2488,whole:Route@2524,whole:Route@317,whole:Route@4004,whole:Route@4044,whole:Route@4255 4103258d590d273a341fe6570ec5cbb3684cf19a221987eb38401528bb47145e |
| +internal/host/rootrestart.go rollbackUnitRestart authority B3,J1,E1 Captured canonical rollback authority is unchanged. Durable verified outcome permits later history settlement without another restart. Fixtures: TestPlan14HistoryRollbackLink and TestPlan14HistoryUnavailableCannotBlockCapturedRollback. file:Read@1568,registry:Hibernate@393,registry:Host@1860,registry:Unit@1962,registry:Unit@2263,registry:Unit@351,registry:Unit@446,registry:Unit@538,registry:Unit@829,whole:Route@1868,whole:Route@1970,whole:Route@2271,whole:Route@337,whole:Route@433,whole:Route@606,whole:Route@743,whole:Route@897 d03694f9dcee767a6f42383b4cf1903286e206da11f805af2ce558bbea63133e |
| internal/host/rootrestart.go rootRestartChecks.live authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@158 28721577488e77d229d3a5cbe6eb5806a4af0696d350a144c5ed2e4d10c36828 |
| internal/host/rootrollback.go captureRollback authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Read@704,registry:Account@1191,registry:App@1227,registry:Host@1154,registry:Host@341,registry:Host@823,whole:Route@1156,whole:Route@1193,whole:Route@1229,whole:Route@343,whole:Route@825 6d3ec51488ef1be9dac2d5fe168cb38e72614242fd18064ba441273b14af6efa |
| internal/host/rootrollback.go loadRollback authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Read@451,registry:Host@1104,whole:Route@1116 6f1034337d1d03ea22a2fdd448cdd2b84dcadd455bf643086f3877c1241d1214 |
| internal/host/rootrollback.go settleRollbackAttempt authority B3,E1 Independent signed recovery authority and retained attempt outcomes. Fixtures: TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14ActiveRecoveryProcessCrash. 0a3e03771e9205e546e1b91371f8b17afba93525a5009afaf15736ca1c7bfa80 |
| -internal/host/rootrollback_resume.go recoverRootRestarts authority B3 Recover every attempt independently, skip settled writes and remove leftover capabilities. file:Read@519 976de0edf31099d112caca68639679a5d4142f1994a0f86d0e8b247d92cf725d |
| +internal/host/rootrollback_resume.go recoverRootRestarts authority B3 Recover every attempt independently, skip settled writes and remove leftover capabilities. file:Read@819 0eb3221e553183869f907660c74bae544d48da57f1e5592b92400377f5769a0e |
| internal/host/rootrollback_resume.go resumeRootRestart authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Account@1419,registry:Account@2125,registry:App@1447,registry:App@2153,registry:Unit@1225,whole:Route@1227,whole:Route@1421,whole:Route@1449,whole:Route@2127,whole:Route@2155 f69e185f75b5758b439be86809c2a83dd300e2e1235879c4860b70cfb5cb39ba |
| internal/release/adopt.go PassFilter authority B3,E3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. E7 applies merge ruling 4: unbound no-census routes use the legacy pass and restart; ordinary candidates make no binding store call. Fixtures: TestR2UnguardedERecoveryUpdate, TestR2UnguardedERecoveryIncidentRecipe, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix. registry:App@26180,registry:App@28637,registry:App@29129,registry:App@29366,registry:App@30773,registry:App@5975,registry:App@9000,registry:App@9141,registry:Hibernate@29012,registry:Host@1055,registry:Host@2746,registry:Host@2813,registry:Host@2909,registry:Host@29182,registry:Host@2944,registry:Host@30098,registry:Host@30826,registry:Host@690,registry:UpdateSettings@8878,whole:Route@1090,whole:Route@1251,whole:Route@2360,whole:Route@2384,whole:Route@2415,whole:Route@2472,whole:Route@2558,whole:Route@2580,whole:Route@26175,whole:Route@2680,whole:Route@28632,whole:Route@28890,whole:Route@28912,whole:Route@28959,whole:Route@28977,whole:Route@28987,whole:Route@29030,whole:Route@29051,whole:Route@29064,whole:Route@29124,whole:Route@29150,whole:Route@29224,whole:Route@29263,whole:Route@29279,whole:Route@29307,whole:Route@29361,whole:Route@29387,whole:Route@30140,whole:Route@30194,whole:Route@30210,whole:Route@30717,whole:Route@30768,whole:Route@30794,whole:Route@30868,whole:Route@30903,whole:Route@30919,whole:Route@3665,whole:Route@4312,whole:Route@579,whole:Route@5897,whole:Route@5919,whole:Route@5970,whole:Route@5996,whole:Route@617,whole:Route@675,whole:Route@793,whole:Route@817,whole:Route@8829,whole:Route@8943,whole:Route@8988,whole:Route@9136 49427f5f398cf7df2b0ac2f63bfacfee3d3c630a253b176493ad87e0f7a49f00 |
| internal/release/adopt.go adoptHost authority B3,E1,E2,E3,E4,J1 Root history intent is linked to captured canonical rollback authority before live effects; verified recovery settles only its exact intent. Fixtures: TestPlan14HistoryRollbackLink, TestPlan14HistoryRecoveryRefusesChanges, TestPlan14HistoryRecoveryProcessCrash, TestPlan14HistorySettlementRetriesVisibleSync. E1 migration and restart consumption remain incomplete. registry:App@10601,registry:App@10761,registry:App@11790,registry:App@12514,registry:App@14076,registry:App@19235,registry:App@24711,registry:App@3114,registry:App@32470,registry:App@6423,registry:App@7724,registry:App@7918,registry:Hibernate@33331,registry:Host@10481,registry:Host@10514,registry:Host@10614,registry:Host@10774,registry:Host@10876,registry:Host@10908,registry:Host@10973,registry:Host@11423,registry:Host@11556,registry:Host@11986,registry:Host@12112,registry:Host@12604,registry:Host@13380,registry:Host@13848,registry:Host@14062,registry:Host@14404,registry:Host@14994,registry:Host@18725,registry:Host@18833,registry:Host@19203,registry:Host@20023,registry:Host@20181,registry:Host@20643,registry:Host@20695,registry:Host@2075,registry:Host@20842,registry:Host@20929,registry:Host@22035,registry:Host@22279,registry:Host@2264,registry:Host@22939,registry:Host@23324,registry:Host@2345,registry:Host@23465,registry:Host@24679,registry:Host@24806,registry:Host@26148,registry:Host@26798,registry:Host@30031,registry:Host@30649,registry:Host@30974,registry:Host@31048,registry:Host@3127,registry:Host@31295,registry:Host@31446,registry:Host@32642,registry:Host@32809,registry:Host@32916,registry:Host@33144,registry:Host@33287,registry:Host@34032,registry:Host@34158,registry:Host@34246,registry:Host@4248,registry:Host@5211,registry:Host@5316,registry:Host@6535,registry:Host@6583,registry:Host@6700,registry:Host@6732,registry:Host@7610,registry:Host@7778,registry:Host@7904,registry:Host@8023,registry:Host@8501,registry:Host@8888,registry:Host@9114,registry:Host@9210,registry:Host@9301,registry:Kind@552,registry:Unit@31194,registry:Unit@577,whole:Route@10596,whole:Route@10756,whole:Route@11780,whole:Route@12504,whole:Route@14044,whole:Route@14609,whole:Route@14970,whole:Route@16193,whole:Route@16496,whole:Route@16652,whole:Route@18982,whole:Route@19126,whole:Route@19225,whole:Route@2176,whole:Route@21764,whole:Route@22163,whole:Route@24701,whole:Route@29602,whole:Route@3109,whole:Route@32618,whole:Route@3554,whole:Route@6418,whole:Route@9738 a2a16fd0445044393881dbc9b33807831cba5097c25c6fa1c0b36202fb63a5f0 |
| @@ -34,7 +34,7 @@ internal/release/rootbake.go RootHistory.LoadBake authority E1 E1 strict root ba |
| internal/release/roothistory.go RootHistory.read authority E1 Root per-binding refusal/program completeness and pending attempt state; strict decode and exact durable recommit before authority reads. E1 production integration is selected but incomplete. 4af99259d080d2b4c4ffee7cf2bc324e665cb07416031a495692421bd21878b4 |
| internal/release/roothistory.go RootHistory.readRecord authority E1 Root reconstruction reads exact durable records and preserves pending restrictions during signed replacement. Fixtures: TestPlan14ReconstructionReplacement, TestPlan14ReconstructionReplacementCrash. file:Read@189 c98eb40269145d3e3d5b41f18e9779f30e618ed0884c556e9bf132bfdc9dfae6 |
| internal/release/roothistory_app.go RootHistory.AppRefusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@297,file:Read@819 4cc038ec851042efe7878b15462c8942783d3b5878873a74910a0b3d16b7f3b5 |
| -internal/release/roothistory_delivery.go RootHistory.ReceiveReconstruction authority E1 Signed replacement binds the exact pending digest and current sources/revisions; old intended refusals and hold bounds remain restrictive. Fixtures: TestPlan14ReconstructionReplacement, TestPlan14ReconstructionReplacementCrash. de43924ddbe526035c8987582d58907cf1d9258fca0cbb56997943487316bb2c |
| +internal/release/roothistory_delivery.go RootHistory.ReceiveReconstruction authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. de43924ddbe526035c8987582d58907cf1d9258fca0cbb56997943487316bb2c |
| internal/release/roothistory_pass.go RootHistoryPass.CompleteAttempt authority B3,E1 Root history intent is linked to captured canonical rollback authority before live effects; verified recovery settles only its exact intent. Fixtures: TestPlan14HistoryRollbackLink, TestPlan14HistoryRecoveryRefusesChanges, TestPlan14HistoryRecoveryProcessCrash, TestPlan14HistorySettlementRetriesVisibleSync. E1 migration and restart consumption remain incomplete. whole:Route@136,whole:Route@255 97d46675209fa1d5ba064afded9b8b3265b741be9a30dfec6efab01a745d1d82 |
| internal/release/roothistory_pass.go RootHistoryPass.begin authority E1 Root history intent is linked to captured canonical rollback authority before live effects; verified recovery settles only its exact intent. Fixtures: TestPlan14HistoryRollbackLink, TestPlan14HistoryRecoveryRefusesChanges, TestPlan14HistoryRecoveryProcessCrash, TestPlan14HistorySettlementRetriesVisibleSync. E1 migration and restart consumption remain incomplete. file:Prepare@607,file:Prepare@638,file:Read@292,whole:Route@218 dde605d1d382309a8546c7fb4a78e276909bc7741a83b93c93d0f2ec9c783004 |
| internal/release/roothistory_pass.go RootHistoryPass.complete authority E1 E1 canonical binding selection and recheck; durable attempt precedes effects; only independent installed observation settles and retires the exact target refusal. whole:Route@205,whole:Route@278,whole:Route@350 10a2d72856ad28919399878857ee8d40e60877b86f76ba12d6f98d456001ccd2 |
| @@ -54,18 +54,46 @@ internal/release/roothistory_replace.go replacementReconstruction authority E1 S |
| internal/release/signedhead.go StageSignedHead authority E1 Canonical route is supplied by the root unit verifier. Verifies signed channel, digest ratchet and manifest/artifact before staging; unavailable pin census never authorizes restaging differing bytes. Fixtures: TestPlan14Incident20261006WithLedger and TestPlan14SignedNamedRecoveryAuthority. registry:Account@202,registry:Account@302,registry:Account@403,registry:App@214,registry:App@415,registry:Channel@222,registry:Host@434 ca02f04e55556ba0233dfa0848b0c533a9d9aa06cad6abd33cfd2968b3bd838a |
| internal/rootstate/link_linux.go ReadTrustedLink authority A,E1 Checks the full root-owned parent chain before descriptor-based live-link reading. Fixture: TestPlan14TrustedLiveLink; privileged acceptance remains required. file:Open@156 90f8e8de4e2c71f07b5d749cbe4bb1c1dfa426576a82138823449f56fc9ec5ab |
| internal/rootstate/link_linux.go directory.readLink authority A,E1 Pins the symlink inode with O_PATH and checks owner/type/link count before readlinkat. Fixture: TestPlan14TrustedLiveLink. file:Fstat@356,file:Openat@183,file:Readlinkat@554 a8559fe63dce48830916004ecfe7fbbe5d419568c7daf7a3073cf1b5bb12c3d7 |
| -@coverage internal/host 0153cc1966470f409f75a095b0599abed5e4df65767d101d333b1a3bb032b237 |
| -@coverage internal/release a468a41aa5baea35364345cdcd4e9e0cf212b0ebb7f413d763dd2a0def2b21e9 |
| +@coverage internal/host 6a059aea86f22c9bdd87e3c4cf9640aece937c608ae84eb481711ed7620e5913 |
| +@coverage internal/release 553767b0c5b8b73ade28191091454b327d64f78b3a4afa7675c8f0899787782b |
| @coverage internal/rootstate 6338f74a6b2059cf77cf478bd6111b37b441eb9d65a309464c378359d4bdac05 |
| -internal/release/roothistory_status.go RootHistory.MigrationStatus authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@1062 14304263a38c2473fad2efc7e775974b0b5e632a7faa896ce307681d34aecabc |
| +internal/release/roothistory_status.go RootHistory.MigrationStatus authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Read@1062 fdd21d4a9823eab7937fb900655f8e7d342cf226f66e5da526ca8b6285845bf2 |
| internal/host/legacyrestart.go restartEffectFailed authority B3 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding. b68bb65884a14fe0cf78a65f56ded4ea41584bba99af9195827d1ea191b340a9 |
| internal/host/legacyrestart.go legacyRootRestart authority B3,G E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. 5fd48683abd82664baa77c2ddc7528f92f557b04586225d64b4a6e77147df931 |
| -internal/host/recoveryrefusals.go recoveryHistoryOutcome authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. ab25e7fa0cca603c50cfee8862c669c674e714b5a8a263748257ce26e1495491 |
| -internal/host/recoveryrefusals.go rootRestartChecks.retainRecoveryHistory authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@473 4784879e151c40b9b1c7fb2719e3d97657d0ad387842f07e5d6a59fe8059462c |
| +internal/host/recoveryrefusals.go recoveryHistoryOutcome authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. ab25e7fa0cca603c50cfee8862c669c674e714b5a8a263748257ce26e1495491 |
| +internal/host/recoveryrefusals.go rootRestartChecks.retainRecoveryHistory authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 1e1b2235aef0c5b009f311b00ea3fbaae52b4f3bdfa4821a0ff8f6cf80f88c48 |
| internal/release/roothistory.go RootHistory.Blocks authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 4ea9aa88a59c8bbfdea5f76fde1f27385b31ec5bcab4a362d9630f252d744b64 |
| -internal/release/roothistory.go RootHistory.Refusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 58467bc2224dc2cdbae451462005f1cd11c198dc41443052425eb629493b2f58 |
| -internal/release/roothistory_outcomes.go RootHistory.RetainRecovery authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@292 2b03f6d1976c9d37748a46ae2323be25163d1622aeb4f456ba4f257f8a89d714 |
| -internal/release/roothistory_outcomes.go RootHistory.recoveryOutcomes authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@337 fb5dd3c1b5fe0d4124bb69cc295127f52026e19349c4e81f2d55e437747748db |
| -internal/release/roothistory_outcomes.go historyRecoveryKey authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 324ba2ea59a112bb176c542787407a0009bea90c3f195e03f0468706e57e7f9e |
| -internal/release/roothistory_outcomes.go recoveryRefusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 9c2bd14d9d384baea312c2c0bbd452c9040cc7b9f343d18886a19afdb17c02be |
| -internal/release/roothistory_outcomes.go validHistoryRecovery authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 8d28b60e48d8c3991876a9213f1f51e30fbf0e31d1a6245c9cfe374a37277a06 |
| +internal/release/roothistory.go RootHistory.Refusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 864a1b6119bb0c2985bb02700884daa1654e744a0549cc84371910795a474de4 |
| +internal/release/roothistory_outcomes.go RootHistory.RetainRecovery authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Read@292 2b03f6d1976c9d37748a46ae2323be25163d1622aeb4f456ba4f257f8a89d714 |
| +internal/release/roothistory_outcomes.go RootHistory.recoveryOutcomes authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Read@337 fb5dd3c1b5fe0d4124bb69cc295127f52026e19349c4e81f2d55e437747748db |
| +internal/release/roothistory_outcomes.go historyRecoveryKey authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 324ba2ea59a112bb176c542787407a0009bea90c3f195e03f0468706e57e7f9e |
| +internal/release/roothistory_outcomes.go validHistoryRecovery authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 8d28b60e48d8c3991876a9213f1f51e30fbf0e31d1a6245c9cfe374a37277a06 |
| +internal/host/recoveryrefusals.go checkUnresolvedRecoveries authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. ba894f3a28dc17b84919bafe2791e3fc375467804703987374229602204020b7 |
| +internal/host/recoveryrefusals.go recoveryResultKey authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 57cbd2daa62878ce2a4e08ea60771644f5cc2fa696a9b22ed7e24cb3a967aa21 |
| +internal/host/recoveryrefusals.go rootRestartChecks.copyRecoveryHistory authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Read@539 679ba65e4cf5205bb33f92b6c477a7b05883dfc97bba8d80b80d42d278ada88e |
| +internal/host/roothistory_maintenance.go rootHistoryMaintenance.run authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. registry:Account@257,registry:App@294,whole:Route@261,whole:Route@298 894c42672df2ee971a523a249ff9d3d3a2e90ad5db7e0bad5f406c6df9311499 |
| +internal/host/roothistory_maintenance.go type rootHistoryMaintenance authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. dab42760a16aea247aecab436074d59ec49115906699b5448e209a9da1f802f3 |
| +internal/host/roothistory_recovery.go type historyRollbackLink authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. f7941675b4596196e30f29f0ebbdbd9c0c83cd10202a351c29a67a184873202b |
| +internal/release/roothistory_delivery.go HistoryReconstructionKey authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 8bcba1729d76ad5776048a4e68904b88b50392381909149e6ad592f66b87b8af |
| +internal/release/roothistory_delivery.go HistoryRecoveryRetirementKey authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. f1623ab87222bd42012497205ba369f9b859d65b7e2e3c0c7ce2e6eb6bf3c6d4 |
| +internal/release/roothistory_delivery.go PublishHistoryReconstruction authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. c2ff8e966cbe2e9f86bc3ddbd61d1df62a5df501927e2791e11518bb62d200c7 |
| +internal/release/roothistory_delivery.go PublishHistoryRecoveryRetirement authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. ec499a70f267b5b551e918c7ebb7ce36745afa87f77160fe9dc826d8dbe3032b |
| +internal/release/roothistory_delivery.go RootHistory.ReceiveRecoveryRetirements authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. ca2588cbe68aa98c8bd7c5a9d85afad318746cc64e77782b2b95e7d7e33bd335 |
| +internal/release/roothistory_outcomes.go RootHistory.recoveryRefusals authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 5d4a50cf425c7ef250c5b72fd1df3efd04f108d30316296831df9b917838c27c |
| +internal/release/roothistory_outcomes.go type HistoryRecovery authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 173280cd4002f9c003b07109ea5b36b05fa648e5ee80de459453b76fa78f8501 |
| +internal/release/roothistory_outcomes.go type historyRecoveryRecord authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 49deb93476b73ca13e9133916babba5ded70b0e01dc793a1a252b9eee07d5a15 |
| +internal/release/roothistory_retirement.go RootHistory.RecoveryRetired authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 70b4fd04aa52687f9c4f140b8ddd345c26e9aca78e325e4f093a39a93d8dec67 |
| +internal/release/roothistory_retirement.go RootHistory.RetireRecovery authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 3bc3d947862f05d7407179d896646aa68ab61c9a3d861086709724869b32e3b7 |
| +internal/release/roothistory_retirement.go RootHistory.recoveryRetired authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. file:Read@114 e862a761df28742d9bb468740b7ab3b239085177d0404aacd5c50eb4920acd2a |
| +internal/release/roothistory_retirement.go historyRecoveryDigest authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 92010ae7a24d9bcdfa237da83c2e832926c8244c6df9d2f697833ac69671f9e4 |
| +internal/release/roothistory_retirement.go historyRetirementKey authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 9e32f1cc354ae4921dd12e2a103208891dabba66594b8fa15d8782e391dd0540 |
| +internal/release/roothistory_retirement.go retirementEvidence authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 98a3c84eede1e1d039d42db42a61f167877b0bbf8a5f45cea384a2941281c81c |
| +internal/release/roothistory_retirement.go type HistoryRecoveryRetirement authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 343ffe9c0b87daf2cd0c2d11452b51ebee92befd313a42b2d7b367360b844167 |
| +internal/release/roothistory_retirement.go validHistoryRetirement authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 9e5d277f9ffa0f303ef680ba37b9c536bc5d8b7d7a5277c423e13984b015c759 |
| +internal/release/roothistory_status.go HistoryStatusKey authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 302a4b007238395e23f3649bac42ad5c68c9b9727bbce54fc775e75da7e0af5e |
| +internal/release/roothistory_status.go RootHistory.ProjectMigrationStatus authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 490ab503f915fc8eb916a21a3de8a8f004cfc457c0fe28fa7d9ef44513678169 |
| +internal/release/roothistory_status.go const RootHistoryStatusDir authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. c7fda64b13809b5d5b9c61e055aa2e5cc47fc018eaf0ee6a62a0b37408990142 |
| +internal/release/roothistory_status.go type HistoryHostStatus authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. 4cfaff43ce1651d564797ccfc41e0b17dfdefda033d03d68416d6cc6361e983d |
| +internal/release/roothistory_status.go type HistoryMigrationStatus authority E1 Exact signed recovery retirement binds namespace, complete outcome digest and a later root-verified success; pending originals and changed evidence remain restrictive. Independent settlement publishes status without controlling effects. Fixtures: TestPlan14HistoryExactRecoveryRetirement, TestPlan14HistoryRecoveryRetirementAuthority, TestPlan14HistoryRetirementVisibleSync, TestPlan14HistoryRetirementProcessCrash, TestPlan14RecoveryRetirementProductionCallback, TestPlan14IndependentRecoveryPublishesStatus. dcf1ce27171203061f7b7877a80e3fdcf8ac964309f0fcc47282a7f719f0a32d |
| +internal/release/roothistory_status.go type HistoryRecoverySources diagnostic E1 Root journal source availability and unresolved attempts remain explicit in advisory status; no projection grants permission. Fixture: TestPlan14RecoveryRetirementProductionCallback. ff307b83963f45fdefe91b61699f090bd8e3f5e7ec539cec4b4bca51956ad07f |
| +internal/host/namedrecovery_pass.go namedUnitRecovery.passOption authority B3,E1 Production recovery wrapper preserves main exit status for per-host failures; only remaining named routes can probe the census for history selection. Ordinary routes make no census or binding-store reads. Fixtures: TestPlan14ERecoveryOrdinaryPass, TestPlan14Fix4AdoptPerHostFailureExitsZero, TestPlan14Fix5MainEquivalence. registry:Host@282,registry:Host@506,registry:Unit@459,registry:Unit@476,whole:Route@419,whole:Route@451 36ef7a4bf960574b35ea896c72612ce9b4a8ad7d2c249eec16f796344b5dd37e |
diff --git a/internal/provision/testdata/plan14-root-reads-fix4.tsv b/internal/provision/testdata/plan14-root-reads-fix4.tsv| new file mode 100644 |
| index 000000000..497ba5f51 |
| --- /dev/null |
| +++ b/internal/provision/testdata/plan14-root-reads-fix4.tsv |
| @@ -0,0 +1,15 @@ |
| +# Plan 14 fix4: merge ruling 5 preserves main settle after one effect; earlier overlays and the legacy freeze remain unchanged. |
| +internal/host/agent.go runAdoptUnderRootRelease authority B3,C rev3=merge-ruling-5;basis=Share one lazy resolver across restart, health observation and minting; preserve main exit status for per-host failures. whole:Route@10481,whole:Route@17015,whole:Route@17051,whole:Route@7901 1ad4ed7bf005fbe9b4aad663c4549782e03576e4e5940559de9b3c5404b5b0f2 |
| +internal/host/legacyrestart.go restartEffectFailed authority B3 rev3=merge-ruling-5;basis=A real UnitNotUpError retains D1 authority even when joined with an optional binding error. b68bb65884a14fe0cf78a65f56ded4ea41584bba99af9195827d1ea191b340a9 |
| +internal/host/proofnamed.go proofBindingSource.findRecord authority B3 rev3=merge-ruling-5;basis=Cache index success or failure once per pass; selected records and ETags remain freshly read. 4312342c8ee5220e99a61d14c3101df79e4fa9b7a958965d1539046cb2fa8def |
| +internal/host/proofobserve.go proofObserver.recordSettled authority B3,J1 rev3=merge-ruling-5;basis=Ordinary unstamped and derived routes do not resolve deployment records for post-restart health observation. registry:Account@308,registry:App@333,registry:Host@174,registry:Unit@127,registry:Unit@144,whole:Route@225,whole:Route@318,whole:Route@343 c295a595019279ffb1a569d480efc6dac87c2a5b09f4f11e5a3376944e5e55bd |
| +internal/host/proofroot.go proofRoot.mintAndLog authority B3,J1,J4 rev3=merge-ruling-5;basis=Exclude ordinary unstamped and derived routes before post-pass binding and proof minting. file:Stat@1633,registry:App@492,registry:App@651,registry:App@774,registry:Host@409,registry:Host@530,registry:Host@706,registry:Unit@362,registry:Unit@379,whole:Route@322,whole:Route@355,whole:Route@467,whole:Route@564,whole:Route@599 31774259fec4b644f3ddf04c0e5cad606e5f22c5be47bfd14c415c16810d71cf |
| +internal/host/proofroot.go proofRoot.recordLive authority B3,J1,J4 rev3=merge-ruling-5;basis=Exclude ordinary unstamped and derived routes before pre-pass binding and health observation. registry:Host@502,registry:Host@542,registry:Unit@455,registry:Unit@472,whole:Route@415,whole:Route@448,whole:Route@572 4bcb1687933ded4a3e316bc504212382b0fb07c35407806190ae2b17462953a9 |
| +internal/host/rootrestart.go checkedUnitRestart authority B3 rev3=merge-ruling-5;basis=No-census restarts settle the original effect using main readiness; journal write failures cannot undo a healthy build. file:EvalSymlinks@1180,file:EvalSymlinks@1957,registry:Account@1526,registry:Account@2325,registry:App@1558,registry:App@2357,registry:Host@1227,registry:Host@2013,registry:Host@2900,registry:Host@3167,registry:Host@3945,registry:Host@636,registry:Unit@3827,registry:Unit@837,whole:Route@1534,whole:Route@1566,whole:Route@2333,whole:Route@2365,whole:Route@3835,whole:Route@410,whole:Route@425,whole:Route@764,whole:Route@845 c5a8ac351af8bd694ea6c65a12560dccc3049792e3018f02aac044074970754b |
| +internal/host/rootrestart.go productionRootRestartChecks authority B3 rev3=merge-ruling-5;basis=Cache a failed store opener as well as a successful source for this pass; root binding authority is unchanged. file:Open@1005,whole:Route@1639,whole:Route@1906,whole:Route@838 1b6f077d4912a86de65a611b981052b0a0ac2a81152bfa209720358f745b511e |
| +internal/host/rootrestart.go restartBoundUnit authority B3 rev3=merge-ruling-5;basis=After an effect, no-census binding uses main settle once; retained-census identity checks stay strict and real NotUp wins. file:EvalSymlinks@635,registry:Account@3053,registry:Account@4552,registry:App@3089,registry:App@4592,registry:Host@1799,registry:Host@2955,registry:Host@3549,registry:Host@4291,registry:Host@682,registry:Unit@1165,registry:Unit@1353,registry:Unit@2172,registry:Unit@2240,registry:Unit@2282,registry:Unit@2581,registry:Unit@3898,registry:Unit@395,whole:Route@1197,whole:Route@198,whole:Route@2535,whole:Route@2650,whole:Route@2765,whole:Route@3059,whole:Route@3095,whole:Route@317,whole:Route@4560,whole:Route@4600,whole:Route@4811 4fb19705e34138b4665405f60e2659b2d1531d8c3ca957086f3bbd7ba15cda50 |
| +internal/host/rootrestart.go rollbackUnitRestart authority B3 rev3=merge-ruling-5;basis=Only pre-effect legacy refusals may retry; ordinary routes skip census reads, Hibernate prevents start, and strict journal recovery survives a missing registry row. file:Read@1990,registry:Hibernate@393,registry:Host@2282,registry:Unit@1078,registry:Unit@2384,registry:Unit@2685,registry:Unit@351,registry:Unit@446,registry:Unit@538,whole:Route@1160,whole:Route@2290,whole:Route@2392,whole:Route@2693,whole:Route@337,whole:Route@433,whole:Route@606,whole:Route@785,whole:Route@822,whole:Route@836 611e1e0cd91a682ee8e1e8cda7ac6479edf167509b2c3a352f28fc71c840b621 |
| +internal/host/rootrestart.go rootRestartRouteWith authority B3 rev3=merge-ruling-5;basis=Ordinary routes bypass census and records; explicit derived stamps retain main checked settle and D1 rollback. registry:Unit@354,registry:Unit@420,whole:Route@1113,whole:Route@1310,whole:Route@341,whole:Route@515,whole:Route@699,whole:Route@822,whole:Route@983 aff504f5ab71c6a5e5f74b96eccb7a96c761daa1d0438ddd233e9d9867578099 |
| +internal/release/versions.go PointLive authority B3,E2 rev3=merge-ruling-5;basis=Compare resolved current and previous paths before retaining the running build; preserve repoint errors and canonical retained pointers. file:EvalSymlinks@1013,file:EvalSymlinks@901,file:Open@1506,file:Readlink@388,file:Stat@147 ab787418ff9aadb632066f25888326984d2167248abd05481221a6e13a8f665d |
| +@coverage internal/host 6d9e8455aed7ddfae23690f7323f8c19855a8df19fdccc096e2628a568dfa186 |
| +@coverage internal/release 5845490a356a59aa52f0ad081e702c98bf39cdfd8804002600596b1219d010d9 |
diff --git a/internal/provision/testdata/plan14-root-reads-fix5.tsv b/internal/provision/testdata/plan14-root-reads-fix5.tsv| new file mode 100644 |
| index 000000000..b1399475e |
| --- /dev/null |
| +++ b/internal/provision/testdata/plan14-root-reads-fix5.tsv |
| @@ -0,0 +1,11 @@ |
| +# Plan 14 fix5: merge ruling 6 makes main effects unconditional without a census; earlier overlays and the legacy freeze are unchanged. |
| +internal/host/legacyrestart.go legacyCheckedUnitRestart authority B3 rev3=merge-ruling-6;basis=Restore main stamped-unit restart and D1 settle, including diagnostics; only clock, readiness and health-hook adapters differ. file:EvalSymlinks@675,registry:Host@1044,registry:Host@1139,registry:Host@241,registry:Host@335,registry:Host@543,registry:Unit@1053,registry:Unit@1148,registry:Unit@198,registry:Unit@344,registry:Unit@382,registry:Unit@517,registry:Unit@576,registry:Unit@649,registry:Unit@744,registry:Unit@782,registry:Unit@844,whole:Route@1218,whole:Route@910 999d8be82a85a24691baaa2d4ffadd0fc6ba073bcc1eee57c40f2167708c370e |
| +internal/host/legacyrestart.go legacyRollbackRestart authority B3 rev3=merge-ruling-6;basis=Restore main reset-failed, restart and D1 settle even after an operator reset; retain main error wrapping and diagnostics. file:EvalSymlinks@378,registry:Host@276,registry:Host@425,registry:Host@706,registry:Unit@157,registry:Unit@193,registry:Unit@285,registry:Unit@325,registry:Unit@491,registry:Unit@715,whole:Route@557 f55b81780b4da5fb5f08fd18079919514edc606bd18e2189e69cf78f1aea8202 |
| +internal/host/legacyrestart.go legacyRootRestart authority B3,G rev3=merge-ruling-6;basis=Census presence alone selects bound effects; the claims flag without a census keeps main effects. 5fd48683abd82664baa77c2ddc7528f92f557b04586225d64b4a6e77147df931 |
| +internal/host/rootrestart.go checkedUnitRestart authority B3 rev3=merge-ruling-6;basis=Return main effects before opening any bound restart state when no census exists; strict journal and identity checks remain census-only. file:EvalSymlinks@1247,file:EvalSymlinks@2024,registry:Account@1593,registry:Account@2392,registry:App@1625,registry:App@2424,registry:Host@1294,registry:Host@2080,registry:Host@2967,registry:Host@3234,registry:Host@3906,registry:Host@703,registry:Unit@3788,registry:Unit@904,whole:Route@1601,whole:Route@1633,whole:Route@2400,whole:Route@2432,whole:Route@3796,whole:Route@479,whole:Route@831,whole:Route@912 1eadec9e355427edc4f8211c2106e855b7d95a0aeae5b71f7961cc19fee0a33e |
| +internal/host/rootrestart.go restartBoundUnit authority B3 rev3=merge-ruling-6;basis=Remove dead no-census settle and effect-tracking exceptions; retain the census-backed strict effect path. file:EvalSymlinks@635,registry:Account@2355,registry:Account@3854,registry:App@2391,registry:App@3894,registry:Host@1618,registry:Host@2257,registry:Host@2851,registry:Host@3593,registry:Host@682,registry:Unit@1104,registry:Unit@1172,registry:Unit@1927,registry:Unit@1995,registry:Unit@2037,registry:Unit@3200,registry:Unit@395,whole:Route@198,whole:Route@2361,whole:Route@2397,whole:Route@317,whole:Route@3862,whole:Route@3902,whole:Route@4113 75130e52351fe6272887aafece15446eeef0760c2e02f9a51cc1dd4e2a6a1455 |
| +internal/host/rootrestart.go rollbackUnitRestart authority B3 rev3=merge-ruling-6;basis=Select main rollback or start before bound journal discovery on no-census boxes; preserve ordinary and Hibernate guards. file:Read@1568,registry:Hibernate@393,registry:Host@1860,registry:Unit@1962,registry:Unit@2263,registry:Unit@351,registry:Unit@446,registry:Unit@538,registry:Unit@829,whole:Route@1868,whole:Route@1970,whole:Route@2271,whole:Route@337,whole:Route@433,whole:Route@606,whole:Route@743,whole:Route@897 569a52f96e0206a496b7718a5103fdb8d6e379a81f9c143e2df5c15ca024fb6b |
| +internal/host/rootrestart.go rootRestartChecks.runningVersion authority B3 rev3=merge-ruling-6;basis=Without a census, ignore another unit local binding for retention so the stamp determines the rollback target; keep resolved-path retention. registry:Host@151,registry:Unit@104,registry:Unit@121,registry:Unit@230,registry:Unit@246,whole:Route@208,whole:Route@232 9618c96657140602f59ade8b967f7fe4729d84d915c7fa33e0a276dd82377a40 |
| +internal/host/rootrestart.go rootRestartRouteWith authority B3 rev3=merge-ruling-6;basis=Select main stamped-unit effects before binding dispatch without a census; ordinary routes still bypass census and record reads. registry:Unit@354,registry:Unit@420,whole:Route@341,whole:Route@515,whole:Route@708,whole:Route@818,whole:Route@978 fad63ffb31320e0154e3a6c76ed0b887812bfefae607d698cd2e6c9386787546 |
| +internal/host/rootrollback_resume.go recoverRootRestarts authority B3 rev3=merge-ruling-6;basis=Prepare the shared D3 health parent but skip bound journal recovery without a census, including with the claims flag set; retained-census recovery stays strict. file:Read@819 5b33733153e36a48058ee3245f0710debe8b9ccffc0bb9c40c0b2d5b8de3599d |
| +@coverage internal/host a2fe6f145509946acb6953012b232f894ece8d79063f254193b13b28a2c97214 |
diff --git a/internal/provision/testdata/plan14-root-reads-fix6.tsv b/internal/provision/testdata/plan14-root-reads-fix6.tsv| new file mode 100644 |
| index 000000000..34d429164 |
| --- /dev/null |
| +++ b/internal/provision/testdata/plan14-root-reads-fix6.tsv |
| @@ -0,0 +1,3 @@ |
| +# Plan 14 fix6: under merge ruling 6 the rollback target is an effect, so runningVersion retains nothing without a census; earlier overlays and the legacy freeze are unchanged. |
| +internal/host/rootrestart.go rootRestartChecks.runningVersion authority B3 rev3=merge-ruling-6;basis=Without a census return no running build, so main's cur stays the rollback target and no record is resolved; census-backed retention unchanged. registry:Host@151,registry:Unit@104,registry:Unit@121,whole:Route@439 173ae670856b3e93089b04f45d6356b88910f7cd8eeb7c3bc7770c9ff4ab13c9 |
| +@coverage internal/host efb1aa8d275862827daeacf4955d653fb48bae69e8fccacb0efdb881334e0419 |
diff --git a/internal/release/plan14_history_retirement_process_test.go b/internal/release/plan14_history_retirement_process_test.go| new file mode 100644 |
| index 000000000..685397e33 |
| --- /dev/null |
| +++ b/internal/release/plan14_history_retirement_process_test.go |
| @@ -0,0 +1,108 @@ |
| +package release |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "os/exec" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +type retirementCrashStore struct { |
| + *healthyRetryStore |
| + boundary string |
| +} |
| + |
| +func (s *retirementCrashStore) Write(name string, raw []byte) error { |
| + if !strings.HasSuffix(name, ".retired") { |
| + return s.healthyRetryStore.Write(name, raw) |
| + } |
| + if s.boundary == "before" { |
| + os.Exit(75) |
| + } |
| + if s.boundary == "visible" { |
| + s.fail = 1 |
| + } |
| + err := s.healthyRetryStore.Write(name, raw) |
| + if err != nil && s.boundary != "visible" { |
| + return err |
| + } |
| + os.Exit(75) |
| + return nil |
| +} |
| +func TestPlan14HistoryRetirementProcessCrash(t *testing.T) { |
| + if boundary := os.Getenv("PLAN14_RETIRE_CRASH"); boundary != "" { |
| + dir := os.Getenv("PLAN14_RETIRE_DIR") |
| + wire, err := os.ReadFile(filepath.Join(dir, "signed")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + pub, err := os.ReadFile(filepath.Join(dir, "public")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var g HistoryRecoveryRetirement |
| + if err := VerifyRootEnvelope(ed25519.PublicKey(pub), wire, &g); err != nil { |
| + t.Fatal(err) |
| + } |
| + h := &RootHistory{Store: &retirementCrashStore{healthyRetryStore: &healthyRetryStore{dir: dir, fault: os.ErrPermission}, boundary: boundary}} |
| + if err := h.RetireRecovery(pub, wire, g.Binding, g.IssuedAt, func(HistoryBinding) (string, error) { return "canonical", nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Fatal("child missed crash boundary") |
| + } |
| + for _, boundary := range []string{"before", "durable", "visible"} { |
| + t.Run(boundary, func(t *testing.T) { |
| + _, s, old, _, g, pub, key := retirementFixture(t) |
| + wire, err := SignRootEnvelope(RootSigner(key), g) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Join(s.dir, "signed"), wire, 0600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(filepath.Join(s.dir, "public"), pub, 0600); err != nil { |
| + t.Fatal(err) |
| + } |
| + cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14HistoryRetirementProcessCrash$", "-test.count=1") |
| + cmd.Dir = t.TempDir() |
| + cmd.Env = append(os.Environ(), "PLAN14_RETIRE_CRASH="+boundary, "PLAN14_RETIRE_DIR="+s.dir) |
| + out, err := cmd.CombinedOutput() |
| + var exit *exec.ExitError |
| + if !errors.As(err, &exit) || exit.ExitCode() != 75 { |
| + t.Fatalf("child: %v\n%s", err, out) |
| + } |
| + fault := &retirementSyncStore{Store: s, fail: boundary == "visible"} |
| + h := &RootHistory{Store: fault} |
| + refs, err := h.Refusals(old.Binding) |
| + if boundary == "visible" { |
| + if err == nil { |
| + t.Fatal("unsynced retirement granted after crash") |
| + } |
| + fault.fail = false |
| + refs, err = h.Refusals(old.Binding) |
| + } |
| + want := 0 |
| + if boundary == "before" { |
| + want = 1 |
| + } |
| + if err != nil || len(refs) != want { |
| + t.Fatalf("reopened retirement: %+v %v", refs, err) |
| + } |
| + raw, err := s.Read(historyRecoveryKey(old)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var retained historyRecoveryRecord |
| + if err := json.Unmarshal(raw, &retained); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !retained.Failed || retained.Attempt.SHA256 != old.Attempt.SHA256 { |
| + t.Fatal("retirement erased evidence") |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/release/plan14_history_retirement_test.go b/internal/release/plan14_history_retirement_test.go| new file mode 100644 |
| index 000000000..561c82e93 |
| --- /dev/null |
| +++ b/internal/release/plan14_history_retirement_test.go |
| @@ -0,0 +1,225 @@ |
| +package release |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "encoding/json" |
| + "os" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +func retirementFixture(t *testing.T) (*RootHistory, *healthyRetryStore, HistoryRecovery, HistoryRecovery, HistoryRecoveryRetirement, ed25519.PublicKey, ed25519.PrivateKey) { |
| + t.Helper() |
| + h, s, b := rootHistoryFixture(t) |
| + now := time.Date(2026, 10, 13, 0, 0, 0, 0, time.UTC) |
| + old := HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: "failed", Source: "canonical", Version: "v2", SHA256: "failed-digest", Platform: AdoptPlatform(), Started: now.Add(-time.Hour), PromotedAt: now.Add(-2 * time.Hour)}, Settled: true, Failed: true} |
| + good := old |
| + good.Attempt.ID, good.Attempt.Version, good.Attempt.SHA256 = "verified", "v3", "good-digest" |
| + good.Attempt.Started, good.Attempt.PromotedAt = now, now |
| + good.Failed, good.Verified = false, true |
| + for _, r := range []HistoryRecovery{old, good} { |
| + if err := h.RetainRecovery(r); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + status, err := h.ReconstructionStatus() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + g := HistoryRecoveryRetirement{Schema: 1, Kind: "carlos-release-recovery-retirement", Identity: status.Identity, Operation: "retire-failed", IssuedAt: now, Binding: b, Source: "canonical", AttemptID: old.Attempt.ID, OutcomeDigest: historyRecoveryDigest(old), VerifiedID: good.Attempt.ID, VerifiedDigest: historyRecoveryDigest(good)} |
| + pub, key, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return h, s, old, good, g, pub, key |
| +} |
| + |
| +func retireForTest(h *RootHistory, g HistoryRecoveryRetirement, pub ed25519.PublicKey, key ed25519.PrivateKey) error { |
| + wire, err := SignRootEnvelope(RootSigner(key), g) |
| + if err != nil { |
| + return err |
| + } |
| + return h.RetireRecovery(pub, wire, g.Binding, g.IssuedAt, func(HistoryBinding) (string, error) { return "canonical", nil }) |
| +} |
| + |
| +func TestPlan14HistoryExactRecoveryRetirement(t *testing.T) { |
| + for _, ambiguous := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "failed", true: "ambiguous"}[ambiguous], func(t *testing.T) { |
| + h, s, old, _, g, pub, key := retirementFixture(t) |
| + if ambiguous { |
| + old.Failed = false |
| + raw, _ := json.Marshal(historyRecoveryRecord{Schema: 1, HistoryRecovery: old}) |
| + if err := s.Write(historyRecoveryKey(old), raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + g.OutcomeDigest = historyRecoveryDigest(old) |
| + } |
| + other := old |
| + other.Attempt.ID = "unrelated" |
| + other.Attempt.SHA256 = "unrelated-digest" |
| + other.Failed = true |
| + if err := h.RetainRecovery(other); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := retireForTest(h, g, pub, key); err != nil { |
| + t.Fatal(err) |
| + } |
| + h = &RootHistory{Store: s} |
| + refs, err := h.Refusals(old.Binding) |
| + if err != nil || len(refs) != 1 || refs[0].SHA256 != "unrelated-digest" { |
| + t.Fatalf("retirement not exact: %+v %v", refs, err) |
| + } |
| + if err := h.RetainRecovery(old); err != nil { |
| + t.Fatal(err) |
| + } |
| + refs, err = h.Refusals(old.Binding) |
| + if err != nil || len(refs) != 1 { |
| + t.Fatalf("source replay erased retirement: %+v %v", refs, err) |
| + } |
| + if _, err := s.Read(historyRecoveryKey(old)); err != nil { |
| + t.Fatal("retirement deleted evidence", err) |
| + } |
| + if err := h.AddRefusal(old.Binding, Refusal{Host: old.Binding.Host, App: old.Binding.App, Version: old.Attempt.Version, SHA256: old.Attempt.SHA256, Kind: RefusalWrongProgram, RefusedAt: g.IssuedAt, PointerPromotedAt: old.Attempt.PromotedAt}); err != nil { |
| + t.Fatal(err) |
| + } |
| + refs, err = h.Refusals(old.Binding) |
| + if err != nil || len(refs) != 2 { |
| + t.Fatalf("retirement cleared another category: %+v %v", refs, err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestPlan14HistoryRecoveryRetirementAuthority(t *testing.T) { |
| + for _, attack := range []string{"signature", "identity", "binding", "source", "outcome", "witness", "pending", "unverified", "failed-witness", "older-witness", "canonical-change", "expired", "reconstruction-pending"} { |
| + t.Run(attack, func(t *testing.T) { |
| + h, s, old, good, g, pub, key := retirementFixture(t) |
| + resolve := func(HistoryBinding) (string, error) { return "canonical", nil } |
| + now := g.IssuedAt |
| + switch attack { |
| + case "signature": |
| + _, key, _ = ed25519.GenerateKey(nil) |
| + case "identity": |
| + g.Identity = strings.Repeat("0", 64) |
| + case "binding": |
| + g.Binding.Host = "other.example.com" |
| + case "source": |
| + g.Source = "forged" |
| + case "outcome": |
| + g.OutcomeDigest = strings.Repeat("0", 64) |
| + case "witness": |
| + g.VerifiedDigest = strings.Repeat("0", 64) |
| + case "pending": |
| + old.Settled = false |
| + case "unverified": |
| + good.Verified = false |
| + case "failed-witness": |
| + good.Failed = true |
| + case "older-witness": |
| + good.Attempt.Started = old.Attempt.Started |
| + case "canonical-change": |
| + calls := 0 |
| + resolve = func(HistoryBinding) (string, error) { |
| + calls++ |
| + if calls > 1 { |
| + return "changed", nil |
| + } |
| + return "canonical", nil |
| + } |
| + case "expired": |
| + now = now.Add(HostRetryGrantTTL + time.Second) |
| + case "reconstruction-pending": |
| + state, err := h.reconstructionState() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + state.Pending = &reconstructionIntent{} |
| + if err := h.writeReconstruction(state); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if attack == "pending" || attack == "unverified" || attack == "failed-witness" || attack == "older-witness" { |
| + for _, r := range []HistoryRecovery{old, good} { |
| + raw, _ := json.Marshal(historyRecoveryRecord{Schema: 1, HistoryRecovery: r}) |
| + if err := s.Write(historyRecoveryKey(r), raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + g.OutcomeDigest, g.VerifiedDigest = historyRecoveryDigest(old), historyRecoveryDigest(good) |
| + } |
| + wire, err := SignRootEnvelope(RootSigner(key), g) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := h.RetireRecovery(pub, wire, old.Binding, now, resolve); err == nil { |
| + t.Fatal("unauthorized retirement accepted") |
| + } |
| + refs, err := h.Refusals(old.Binding) |
| + if err == nil && len(refs) == 0 { |
| + t.Fatal("refusal disappeared") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +type retirementSyncStore struct { |
| + rootstate.Store |
| + fail bool |
| +} |
| + |
| +func (s *retirementSyncStore) Write(name string, raw []byte) error { |
| + if err := s.Store.Write(name, raw); err != nil { |
| + return err |
| + } |
| + if s.fail && strings.HasSuffix(name, ".retired") { |
| + return os.ErrPermission |
| + } |
| + return nil |
| +} |
| +func TestPlan14HistoryRetirementVisibleSync(t *testing.T) { |
| + h, s, old, good, g, pub, key := retirementFixture(t) |
| + fault := &retirementSyncStore{Store: s, fail: true} |
| + h.Store = fault |
| + if err := retireForTest(h, g, pub, key); err == nil { |
| + t.Fatal("unsynced retirement reported success") |
| + } |
| + if refs, err := h.Refusals(old.Binding); err == nil && len(refs) == 0 { |
| + t.Fatal("unsynced retirement granted permission") |
| + } |
| + fault.fail = false |
| + if refs, err := h.Refusals(old.Binding); err != nil || len(refs) != 0 { |
| + t.Fatalf("durable retry: %+v %v", refs, err) |
| + } |
| + good.Failed = true |
| + if err := h.RetainRecovery(good); err != nil { |
| + t.Fatal(err) |
| + } |
| + if refs, err := h.Refusals(old.Binding); err == nil && len(refs) < 2 { |
| + t.Fatalf("changed witness kept retirement: %+v", refs) |
| + } |
| + if err := retireForTest(h, g, pub, key); err == nil { |
| + t.Fatal("old signature retired changed evidence") |
| + } |
| +} |
| + |
| +func TestPlan14HistoryRetirementDelivery(t *testing.T) { |
| + h, _, old, _, g, pub, key := retirementFixture(t) |
| + store := NewDirStore(t.TempDir()) |
| + if err := PublishHistoryRecoveryRetirement(t.Context(), store, RootSigner(key), g); err != nil { |
| + t.Fatal(err) |
| + } |
| + resolve := func(HistoryBinding) (string, error) { return "canonical", nil } |
| + if err := h.ReceiveRecoveryRetirements(t.Context(), store, pub, old.Binding, g.IssuedAt, resolve); err != nil { |
| + t.Fatal(err) |
| + } |
| + view, err := h.MigrationStatus(old.Binding, resolve) |
| + if err != nil || len(view.Hosts) != 1 || len(view.Hosts[0].RetiredRecoveries) != 1 || view.Hosts[0].RecoveryDigests[old.Attempt.ID] != g.OutcomeDigest { |
| + t.Fatalf("retirement status: %+v %v", view, err) |
| + } |
| + if err := h.ReceiveRecoveryRetirements(t.Context(), store, pub, old.Binding, g.IssuedAt.Add(2*HostRetryGrantTTL), resolve); err != nil { |
| + t.Fatal("committed retirement expired", err) |
| + } |
| +} |
diff --git a/internal/release/roothistory.go b/internal/release/roothistory.go| index d97eeea53..e900654f7 100644 |
| --- a/internal/release/roothistory.go |
| +++ b/internal/release/roothistory.go |
| @@ -197,7 +197,7 @@ func (h *RootHistory) Refusals(b HistoryBinding) ([]Refusal, error) { |
| if err != nil { |
| return nil, err |
| } |
| - refs, err := recoveryRefusals(outcomes, b) |
| + refs, err := h.recoveryRefusals(outcomes, b) |
| if err != nil { |
| return nil, err |
| } |
diff --git a/internal/release/roothistory_delivery.go b/internal/release/roothistory_delivery.go| index a120d3b7a..8a6353741 100644 |
| --- a/internal/release/roothistory_delivery.go |
| +++ b/internal/release/roothistory_delivery.go |
| @@ -47,3 +47,72 @@ func (h *RootHistory) ReceiveReconstruction(ctx context.Context, store Store, pu |
| } |
| return h.Reconstruct(pub, wire, b.Account, b.App, now, resolve) |
| } |
| + |
| +func HistoryRecoveryRetirementKey(b HistoryBinding, identity, outcomeDigest string) string { |
| + return appPrefix(b.Account, b.App) + "release-history/" + identity + "/recovery/" + outcomeDigest + ".json" |
| +} |
| + |
| +func PublishHistoryRecoveryRetirement(ctx context.Context, store Store, signer Signer, g HistoryRecoveryRetirement) error { |
| + if !validHistoryRetirement(g) { |
| + return os.ErrPermission |
| + } |
| + for _, digest := range []string{g.Identity, g.OutcomeDigest, g.VerifiedDigest} { |
| + if _, err := hex.DecodeString(digest); err != nil { |
| + return os.ErrPermission |
| + } |
| + } |
| + wire, err := SignRootEnvelope(signer, g) |
| + if err != nil { |
| + return err |
| + } |
| + _, err = store.Put(ctx, HistoryRecoveryRetirementKey(g.Binding, g.Identity, g.OutcomeDigest), wire, "") |
| + return err |
| +} |
| + |
| +func (h *RootHistory) ReceiveRecoveryRetirements(ctx context.Context, store Store, pub ed25519.PublicKey, b HistoryBinding, now time.Time, resolve func(HistoryBinding) (string, error)) error { |
| + if !b.valid() || store == nil { |
| + return os.ErrPermission |
| + } |
| + status, err := h.ReconstructionStatus() |
| + if err != nil { |
| + return err |
| + } |
| + if status.Pending { |
| + return os.ErrPermission |
| + } |
| + outcomes, err := h.recoveryOutcomes() |
| + if err != nil { |
| + return err |
| + } |
| + count := 0 |
| + for _, r := range outcomes { |
| + if r.Binding != b { |
| + continue |
| + } |
| + count++ |
| + if count > historyLimit { |
| + return os.ErrPermission |
| + } |
| + if !r.Settled || (r.Verified && !r.Failed && !r.Recovering) { |
| + continue |
| + } |
| + retired, err := h.recoveryRetired(r, outcomes) |
| + if err != nil { |
| + return err |
| + } |
| + if retired { |
| + continue |
| + } |
| + wire, _, err := store.Get(ctx, HistoryRecoveryRetirementKey(b, status.Identity, historyRecoveryDigest(r))) |
| + if errors.Is(err, os.ErrNotExist) { |
| + continue |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + if err := h.RetireRecovery(pub, wire, b, now, resolve); err != nil { |
| + return err |
| + } |
| + } |
| + return nil |
| +} |
diff --git a/internal/release/roothistory_outcomes.go b/internal/release/roothistory_outcomes.go| index 89afb7c4a..dc10abec6 100644 |
| --- a/internal/release/roothistory_outcomes.go |
| +++ b/internal/release/roothistory_outcomes.go |
| @@ -90,7 +90,7 @@ func (h *RootHistory) RetainRecovery(r HistoryRecovery) error { |
| return h.Store.Write(key, raw) |
| } |
| -func recoveryRefusals(outcomes []HistoryRecovery, b HistoryBinding) ([]Refusal, error) { |
| +func (h *RootHistory) recoveryRefusals(outcomes []HistoryRecovery, b HistoryBinding) ([]Refusal, error) { |
| var refs []Refusal |
| count := 0 |
| for _, r := range outcomes { |
| @@ -98,7 +98,17 @@ func recoveryRefusals(outcomes []HistoryRecovery, b HistoryBinding) ([]Refusal, |
| continue |
| } |
| count++ |
| - if count > historyLimit || !r.Settled { |
| + if count > historyLimit { |
| + return nil, os.ErrPermission |
| + } |
| + retired, err := h.recoveryRetired(r, outcomes) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if retired { |
| + continue |
| + } |
| + if !r.Settled { |
| return nil, os.ErrPermission |
| } |
| if r.Failed || r.Recovering { |
diff --git a/internal/release/roothistory_retirement.go b/internal/release/roothistory_retirement.go| new file mode 100644 |
| index 000000000..e97653770 |
| --- /dev/null |
| +++ b/internal/release/roothistory_retirement.go |
| @@ -0,0 +1,152 @@ |
| +package release |
| + |
| +import ( |
| + "crypto/ed25519" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/rootstate" |
| +) |
| + |
| +type HistoryRecoveryRetirement struct { |
| + Schema int `json:"schema"` |
| + Kind string `json:"kind"` |
| + Identity string `json:"identity"` |
| + Operation string `json:"operation"` |
| + IssuedAt time.Time `json:"issued_at"` |
| + Binding HistoryBinding `json:"binding"` |
| + Source string `json:"source"` |
| + AttemptID string `json:"attempt_id"` |
| + OutcomeDigest string `json:"outcome_digest"` |
| + VerifiedID string `json:"verified_id"` |
| + VerifiedDigest string `json:"verified_digest"` |
| +} |
| + |
| +func historyRecoveryDigest(r HistoryRecovery) string { |
| + r.Attempt.Started, r.Attempt.PromotedAt = r.Attempt.Started.UTC(), r.Attempt.PromotedAt.UTC() |
| + raw, err := json.Marshal(r) |
| + if err != nil { |
| + return "" |
| + } |
| + return reconstructionDigest(raw) |
| +} |
| + |
| +func historyRetirementKey(r HistoryRecovery) string { |
| + return strings.TrimSuffix(historyRecoveryKey(r), ".outcome") + ".retired" |
| +} |
| + |
| +func validHistoryRetirement(g HistoryRecoveryRetirement) bool { |
| + return g.Schema == 1 && g.Kind == "carlos-release-recovery-retirement" && len(g.Identity) == 64 && |
| + g.Operation != "" && len(g.Operation) <= 256 && !g.IssuedAt.IsZero() && g.Binding.valid() && g.Source != "" && |
| + g.AttemptID != "" && g.VerifiedID != "" && g.AttemptID != g.VerifiedID && len(g.OutcomeDigest) == 64 && len(g.VerifiedDigest) == 64 |
| +} |
| + |
| +func retirementEvidence(g HistoryRecoveryRetirement, outcomes []HistoryRecovery) bool { |
| + var old, good *HistoryRecovery |
| + for i := range outcomes { |
| + r := &outcomes[i] |
| + if r.Binding != g.Binding { |
| + continue |
| + } |
| + if r.Attempt.ID == g.AttemptID { |
| + old = r |
| + } |
| + if r.Attempt.ID == g.VerifiedID { |
| + good = r |
| + } |
| + } |
| + return old != nil && good != nil && old.Settled && good.Settled && good.Verified && !good.Failed && !good.Recovering && |
| + good.Attempt.Source == g.Source && good.Attempt.Started.After(old.Attempt.Started) && |
| + historyRecoveryDigest(*old) == g.OutcomeDigest && historyRecoveryDigest(*good) == g.VerifiedDigest |
| +} |
| + |
| +func (h *RootHistory) RetireRecovery(pub ed25519.PublicKey, wire []byte, b HistoryBinding, now time.Time, resolve func(HistoryBinding) (string, error)) error { |
| + if !b.valid() || resolve == nil || now.IsZero() || len(wire) > rootstate.MaxRecord { |
| + return os.ErrPermission |
| + } |
| + var g HistoryRecoveryRetirement |
| + if err := VerifyRootEnvelope(pub, wire, &g); err != nil { |
| + return err |
| + } |
| + if !validHistoryRetirement(g) || g.Binding != b || g.IssuedAt.Before(now.Add(-HostRetryGrantTTL)) || g.IssuedAt.After(now.Add(hostRetrySkew)) { |
| + return os.ErrPermission |
| + } |
| + if err := h.reconstructionReady(); err != nil { |
| + return err |
| + } |
| + state, err := h.reconstructionState() |
| + if err != nil { |
| + return err |
| + } |
| + if state.Identity != g.Identity { |
| + return os.ErrPermission |
| + } |
| + source, err := resolve(b) |
| + if err != nil { |
| + return err |
| + } |
| + if source != g.Source { |
| + return os.ErrPermission |
| + } |
| + outcomes, err := h.recoveryOutcomes() |
| + if err != nil { |
| + return err |
| + } |
| + if !retirementEvidence(g, outcomes) { |
| + return os.ErrPermission |
| + } |
| + source, err = resolve(b) |
| + if err != nil { |
| + return err |
| + } |
| + if source != g.Source { |
| + return os.ErrPermission |
| + } |
| + raw, err := json.Marshal(g) |
| + if err != nil { |
| + return err |
| + } |
| + return h.Store.Write(historyRetirementKey(HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: g.AttemptID}}), raw) |
| +} |
| + |
| +func (h *RootHistory) recoveryRetired(r HistoryRecovery, outcomes []HistoryRecovery) (bool, error) { |
| + raw, err := h.Store.Read(historyRetirementKey(r)) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return false, nil |
| + } |
| + if err != nil { |
| + return false, err |
| + } |
| + var g HistoryRecoveryRetirement |
| + if len(raw) > rootstate.MaxRecord || strictCanonicalJSON(raw, &g) != nil || !validHistoryRetirement(g) || g.Binding != r.Binding || g.AttemptID != r.Attempt.ID { |
| + return false, os.ErrPermission |
| + } |
| + state, err := h.reconstructionState() |
| + if err != nil { |
| + return false, err |
| + } |
| + if state.Pending != nil || state.Identity != g.Identity { |
| + return false, os.ErrPermission |
| + } |
| + if historyRecoveryDigest(r) != g.OutcomeDigest || !retirementEvidence(g, outcomes) { |
| + return false, nil |
| + } |
| + if err := h.Store.Write(historyRetirementKey(r), raw); err != nil { |
| + return false, err |
| + } |
| + return true, nil |
| +} |
| + |
| +func (h *RootHistory) RecoveryRetired(r HistoryRecovery) (bool, error) { |
| + if h == nil || h.Store == nil || !validHistoryRecovery(r) { |
| + return false, os.ErrPermission |
| + } |
| + outcomes, err := h.recoveryOutcomes() |
| + if err != nil { |
| + return false, err |
| + } |
| + return h.recoveryRetired(r, outcomes) |
| +} |
diff --git a/internal/release/roothistory_status.go b/internal/release/roothistory_status.go| index a51cca8ab..f6a91b4ec 100644 |
| --- a/internal/release/roothistory_status.go |
| +++ b/internal/release/roothistory_status.go |
| @@ -14,25 +14,33 @@ const RootHistoryStatusDir = "/var/lib/carlos-release/history-status" |
| type HistoryHostStatus struct { |
| HistoryReconstructionHost |
| - Programs historyPrograms `json:"programs"` |
| - RefusalStatus historyRefusals `json:"refusal_status"` |
| - Recoveries []HistoryRecovery `json:"recoveries,omitempty"` |
| - Attempt *HistoryAttempt `json:"attempt,omitempty"` |
| + Programs historyPrograms `json:"programs"` |
| + RefusalStatus historyRefusals `json:"refusal_status"` |
| + Recoveries []HistoryRecovery `json:"recoveries,omitempty"` |
| + RecoveryDigests map[string]string `json:"recovery_digests,omitempty"` |
| + RetiredRecoveries []string `json:"retired_recoveries,omitempty"` |
| + Attempt *HistoryAttempt `json:"attempt,omitempty"` |
| +} |
| + |
| +type HistoryRecoverySources struct { |
| + Known bool `json:"known"` |
| + Unresolved []HistoryRecovery `json:"unresolved,omitempty"` |
| } |
| type HistoryMigrationStatus struct { |
| HistoryReconstructionStatus |
| - Schema int `json:"schema"` |
| - Account string `json:"account"` |
| - App string `json:"app"` |
| - AppRevision uint64 `json:"app_revision"` |
| - AppComplete bool `json:"app_complete"` |
| - BakeRevision uint64 `json:"bake_revision"` |
| - BakeComplete uint64 `json:"bake_complete"` |
| - Hosts []HistoryHostStatus `json:"hosts"` |
| - Clock FirstSeen `json:"clock"` |
| - PendingDigest string `json:"pending_digest,omitempty"` |
| - PendingGrant *HistoryReconstruction `json:"pending_grant,omitempty"` |
| + Schema int `json:"schema"` |
| + RecoverySources *HistoryRecoverySources `json:"recovery_sources,omitempty"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + AppRevision uint64 `json:"app_revision"` |
| + AppComplete bool `json:"app_complete"` |
| + BakeRevision uint64 `json:"bake_revision"` |
| + BakeComplete uint64 `json:"bake_complete"` |
| + Hosts []HistoryHostStatus `json:"hosts"` |
| + Clock FirstSeen `json:"clock"` |
| + PendingDigest string `json:"pending_digest,omitempty"` |
| + PendingGrant *HistoryReconstruction `json:"pending_grant,omitempty"` |
| } |
| func HistoryStatusKey(acct, app string) string { return historyAppKey(acct, app) + ".status" } |
| @@ -143,12 +151,24 @@ func (h *RootHistory) MigrationStatus(b HistoryBinding, resolve func(HistoryBind |
| return out, err |
| } |
| var recoveries []HistoryRecovery |
| + digests := map[string]string{} |
| + var retired []string |
| for _, r := range outcomes { |
| if r.Binding == binding { |
| recoveries = append(recoveries, r) |
| + digests[r.Attempt.ID] = historyRecoveryDigest(r) |
| + if state.Pending == nil { |
| + ok, err := h.recoveryRetired(r, outcomes) |
| + if err != nil { |
| + return out, err |
| + } |
| + if ok { |
| + retired = append(retired, r.Attempt.ID) |
| + } |
| + } |
| } |
| } |
| - out.Hosts = append(out.Hosts, HistoryHostStatus{Recoveries: recoveries, HistoryReconstructionHost: HistoryReconstructionHost{Host: host, Source: source, Revision: record.Revision, Refusals: record.Refusals.Values}, Programs: record.Programs, RefusalStatus: record.Refusals, Attempt: record.Attempt}) |
| + out.Hosts = append(out.Hosts, HistoryHostStatus{Recoveries: recoveries, RecoveryDigests: digests, RetiredRecoveries: retired, HistoryReconstructionHost: HistoryReconstructionHost{Host: host, Source: source, Revision: record.Revision, Refusals: record.Refusals.Values}, Programs: record.Programs, RefusalStatus: record.Refusals, Attempt: record.Attempt}) |
| } |
| for _, host := range out.Hosts { |
| source, err := resolve(HistoryBinding{Host: host.Host, Account: b.Account, App: b.App}) |
| @@ -162,14 +182,41 @@ func (h *RootHistory) MigrationStatus(b HistoryBinding, resolve func(HistoryBind |
| return out, nil |
| } |
| -func (h *RootHistory) ProjectMigrationStatus(dst rootstate.Store, b HistoryBinding, resolve func(HistoryBinding) (string, error)) error { |
| - if dst == nil { |
| +func (h *RootHistory) ProjectMigrationStatus(dst rootstate.Store, b HistoryBinding, resolve func(HistoryBinding) (string, error), sources ...HistoryRecoverySources) error { |
| + if dst == nil || len(sources) > 1 { |
| return os.ErrPermission |
| } |
| view, err := h.MigrationStatus(b, resolve) |
| if err != nil { |
| return err |
| } |
| + if len(sources) == 1 { |
| + effective := HistoryRecoverySources{Known: sources[0].Known} |
| + if len(sources[0].Unresolved) > historyLimit { |
| + return os.ErrPermission |
| + } |
| + for _, r := range sources[0].Unresolved { |
| + if !validHistoryRecovery(r) || r.Binding.Account != b.Account || r.Binding.App != b.App { |
| + return os.ErrPermission |
| + } |
| + if r.Settled { |
| + retired, err := h.RecoveryRetired(r) |
| + if err != nil { |
| + effective.Known = false |
| + } |
| + if err == nil && retired { |
| + continue |
| + } |
| + } |
| + effective.Unresolved = append(effective.Unresolved, r) |
| + for i := range view.Hosts { |
| + if view.Hosts[i].Host == r.Binding.Host { |
| + view.Hosts[i].RetiredRecoveries = slices.DeleteFunc(view.Hosts[i].RetiredRecoveries, func(id string) bool { return id == r.Attempt.ID }) |
| + } |
| + } |
| + } |
| + view.RecoverySources = &effective |
| + } |
| raw, err := json.Marshal(view) |
| if err != nil { |
| return err |
diff --git a/internal/release/versions.go b/internal/release/versions.go| index 829f1739b..25b7886f1 100644 |
| --- a/internal/release/versions.go |
| +++ b/internal/release/versions.go |
| @@ -301,13 +301,17 @@ func PointLive(optDir, host, acct, ver, app string, runningVersion ...string) er |
| return os.ErrPermission |
| } |
| running := VersionPath(optDir, acct, runningVersion[0], app) |
| - if running != old { |
| - prior, err := os.Readlink(filepath.Join(dir, host)) |
| + current, err := filepath.EvalSymlinks(link) |
| + if err != nil { |
| + return err |
| + } |
| + if running != current { |
| + prior, err := filepath.EvalSymlinks(filepath.Join(dir, host)) |
| if err != nil || prior != running { |
| return os.ErrPermission |
| } |
| - old = running |
| } |
| + old = running |
| } |
| if err := safefile.MkdirAll(dir, 0755); err != nil { |
| return err |