carlos / platform Public

WIP claim Plan 14 track E checkpoint 9 (not for merge): E recovery effects gated on census (merge ruling 6), E1 retained recovery outcomes in root history

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude agent, operated by paul@keymail.dev ebcf62317af14e4975e471b299c7fb7cf8bf329c parent f751559
21 files changed, +733 −185
  • internal/host/agent.go +1 −1
  • internal/host/legacyrestart.go +1 −2
  • internal/host/namedrecovery_active.go +13 −55
  • internal/host/plan14_active_recovery_process_test.go +0 −1
  • internal/host/plan14_active_recovery_test.go +23 −10
  • internal/host/plan14_e7_legacy_linux_test.go +16 −0
  • internal/host/plan14_e8_outcome_test.go +28 −6
  • internal/host/plan14_e9_census_test.go +110 −0
  • internal/host/plan14_recovery_mint_test.go +57 −5
  • internal/host/recoveryrefusals.go +38 −27
  • internal/host/recoverysettle.go +0 −37
  • internal/host/roothistory.go +5 −1
  • internal/host/roothistory_recovery.go +42 −3
  • internal/host/rootrestart.go +0 −19
  • internal/provision/testdata/plan14-root-reads-e.tsv +21 −12
  • internal/release/plan14_history_outcomes_process_test.go +60 −0
  • internal/release/plan14_history_outcomes_test.go +160 −0
  • internal/release/roothistory.go +18 −2
  • internal/release/roothistory_app.go +9 −0
  • internal/release/roothistory_outcomes.go +111 −0
  • internal/release/roothistory_status.go +20 −4
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 7416832f1..df7b8ebc6 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -4506,7 +4506,7 @@ func runAdoptUnderRootRelease(args []string, buildClaims ClaimFactory, ensureUni
pr := proofRoot{
recordProgram: func(b liveBuild) error { return recordRootProgram(history, b) },
refusals: func(b release.HistoryBinding) ([]release.Refusal, error) {
- return restartChecks.mintRefusals(history.History.MintRefusals, b)
+ return restartChecks.mintRefusals(history.History, b)
},
obs: productionProofObserver(release.OptDir()),
reg: reg,
diff --git a/internal/host/legacyrestart.go b/internal/host/legacyrestart.go
index 55092d1b3..888d586fb 100644
--- a/internal/host/legacyrestart.go
+++ b/internal/host/legacyrestart.go
@@ -7,7 +7,6 @@ import (
"path/filepath"
"time"
- "github.com/carlosframework/platform/internal/hostwrite"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/rootparticipation"
@@ -22,7 +21,7 @@ func legacyRootRestart() (bool, error) {
if l.State != nil {
l.State.Close()
}
- return !required && !hostwrite.Enabled(os.Getenv) && err == nil, err
+ return !required && err == nil, err
}
func (c rootRestartChecks) allowLegacy() (bool, error) {
diff --git a/internal/host/namedrecovery_active.go b/internal/host/namedrecovery_active.go
index cb1719d2b..9b8259e61 100644
--- a/internal/host/namedrecovery_active.go
+++ b/internal/host/namedrecovery_active.go
@@ -17,25 +17,18 @@ import (
"github.com/carlosframework/platform/internal/release"
)
-func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootRestartChecks, getenv func(string) string, reg *registry.Registry) (handled bool, result error) {
+func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootRestartChecks, getenv func(string) string, reg *registry.Registry) (bool, error) {
+ if rt, err := reg.Get(host); err == nil && rt != nil && (rt.Unit == "" || rt.Unit == activator.UnitName(host)) {
+ return false, nil
+ }
c.preserveOutcome = true
legacy, err := c.allowLegacy()
if err != nil {
return true, err
}
if legacy {
- rt, err := reg.Get(host)
- if err != nil || rt == nil || rt.Unit == "" || rt.Unit == activator.UnitName(host) {
- return false, nil
- }
- c.recoveryOutcome = &recoveryRestartOutcome{route: *rt}
+ return false, nil
}
- // An optional binding cannot veto the no-census restart path.
- defer func() {
- if legacy && !c.recoveryOutcome.effect && result != nil && !restartEffectFailed(result) {
- handled, result = false, nil
- }
- }()
if n.source == nil || n.readLive == nil {
return true, os.ErrPermission
}
@@ -130,23 +123,8 @@ func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootResta
if err := restartBoundUnit(n.source.sys, n.source.optDir, getenv, old, false, nil, c); err != nil {
return true, err
}
- if c.recoveryOutcome != nil && c.recoveryOutcome.unbound {
- return true, effect(func() error {
- current, err := reg.Get(host)
- if err != nil || current == nil {
- return errors.Join(err, os.ErrPermission)
- }
- if current.Version == pointer.Version {
- return nil
- }
- return reg.SetVersions(host, pointer.Version, current.Version)
- })
- }
return true, effect(func() error {
if err := validate(); err != nil {
- if legacy {
- return nil
- }
return err
}
current, err := reg.Get(host)
@@ -193,6 +171,10 @@ func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootResta
if err := settleRollbackAttempt(s, *superseded); err != nil {
return err
}
+ if superseded.Independent {
+ superseded.Settled = true
+ _ = c.settleHistory(ctx, *superseded, superseded.Recovering)
+ }
}
var err error
attempt, err = captureRollback(s, old, c.now(), unitSettleTimeout(getenv), target, id, historyRollbackLink{
@@ -217,47 +199,23 @@ func (n namedUnitRecovery) recover(ctx context.Context, host string, c rootResta
}
c.identity, c.effect, c.targetDigest = id, effect, digest
if err := restartBoundUnit(n.source.sys, n.source.optDir, getenv, old, false, nil, c); err != nil {
- if legacy && !c.recoveryOutcome.effect && !restartEffectFailed(err) && (errors.Is(err, errRestartBinding) || errors.Is(err, errUnitStopped)) {
- return false, settleRollbackAttempt(s, attempt)
- }
- if legacy && errors.Is(err, errUnitStopped) {
- return true, settleRollbackAttempt(s, attempt)
- }
attempt.Failed = true
if writeErr := writeRollbackState(s, rollbackJournalKey(host), attempt); writeErr != nil {
return true, errors.Join(err, writeErr)
}
- if legacy && restartEffectFailed(err) {
- attempt.Recovering = true
- rollbackErr := effect(func() error {
- if err := writeRollbackState(s, rollbackJournalKey(host), attempt); err != nil {
- return err
- }
- if err := release.PointLive(n.source.optDir, host, rec.Account, old.Version, rec.App); err != nil {
- return err
- }
- if err := legacyRollbackRestart(n.source.sys, n.source.optDir, getenv, c.recoveryOutcome.route, c); err != nil {
- return err
- }
- return reg.SetVersions(host, old.Version, "")
- })
- return true, errors.Join(err, rollbackErr, settleRollbackAttempt(s, attempt))
- }
return true, errors.Join(err, resumeHistoryRestart(ctx, n.source.sys, n.source.optDir, getenv, c, n.writes, s, attempt))
}
- attempt.Verified = c.recoveryOutcome == nil || !c.recoveryOutcome.unbound
+ attempt.Verified = true
if err := settleRollbackAttempt(s, attempt); err != nil {
return true, err
}
- if !attempt.Verified {
- return true, effect(func() error { return reg.SetVersions(host, pointer.Version, old.Version) })
+ attempt.Settled = true
+ if err := c.settleHistory(ctx, attempt, false); err != nil {
+ return true, err
}
return true, effect(func() error {
current, err := c.host(ctx, host)
if err != nil || current.Source != old.Source || current.Version != pointer.Version || current.Digest != digest {
- if legacy {
- return reg.SetVersions(host, pointer.Version, old.Version)
- }
return errors.Join(err, os.ErrPermission)
}
return reg.SetVersions(host, pointer.Version, old.Version)
diff --git a/internal/host/plan14_active_recovery_process_test.go b/internal/host/plan14_active_recovery_process_test.go
index e9de5741e..2d763c0ce 100644
--- a/internal/host/plan14_active_recovery_process_test.go
+++ b/internal/host/plan14_active_recovery_process_test.go
@@ -117,7 +117,6 @@ func TestPlan14ActiveRecoveryProcessCrash(t *testing.T) {
}
now := a.Created.Add(time.Second)
c := rootRestartChecks{attempts: func() (rootstate.Store, error) { return disk, nil }, host: source.forHost, rollback: source.rollback, prepareRollback: source.prepareRollback, recovered: source.recovered, ready: func(registry.Route) func() error { return func() error { return nil } }, now: func() time.Time { return now }, sleep: func(d time.Duration) { now = now.Add(d) }, history: func() (*release.RootHistoryPass, error) {
- t.Fatal("independent recovery opened history")
return nil, os.ErrPermission
}}
before := len(u.effects)
diff --git a/internal/host/plan14_active_recovery_test.go b/internal/host/plan14_active_recovery_test.go
index 18b8173cd..3a811934f 100644
--- a/internal/host/plan14_active_recovery_test.go
+++ b/internal/host/plan14_active_recovery_test.go
@@ -173,11 +173,13 @@ func TestPlan14SignedFixSupersedesOlderAttempt(t *testing.T) {
if err != nil {
t.Fatal(err)
}
+ h := seedHostRootHistory(t, b.Route)
+ c.history = func() (*release.RootHistoryPass, error) { return &release.RootHistoryPass{History: h}, nil }
disk, err := c.attempts()
if err != nil {
t.Fatal(err)
}
- _, err = captureRollback(disk, b, c.now(), unitSettleTimeout(func(string) string { return "5s" }), release.VersionPath(u.opt, b.Route.Account, "obsolete", b.Route.App), c.identity, historyRollbackLink{Binding: release.HistoryBinding{Host: b.Route.Host, Account: b.Route.Account, App: b.Route.App}, Attempt: release.HistoryAttempt{ID: "older-attempt", Version: "obsolete", Source: b.Source, SHA256: "old-digest", PromotedAt: c.now()}, Independent: true})
+ _, err = captureRollback(disk, b, c.now(), unitSettleTimeout(func(string) string { return "5s" }), release.VersionPath(u.opt, b.Route.Account, "obsolete", b.Route.App), c.identity, historyRollbackLink{Binding: release.HistoryBinding{Host: b.Route.Host, Account: b.Route.Account, App: b.Route.App}, Attempt: release.HistoryAttempt{ID: "older-attempt", Version: "obsolete", Source: b.Source, SHA256: "old-digest", Platform: release.AdoptPlatform(), Started: c.now(), PromotedAt: c.now()}, Independent: true})
if err != nil {
t.Fatal(err)
}
@@ -186,6 +188,9 @@ func TestPlan14SignedFixSupersedesOlderAttempt(t *testing.T) {
if !handled || err != nil || len(u.effects) != 1 || u.exe != release.VersionPath(u.opt, b.Route.Account, "v2", b.Route.App) {
t.Fatalf("old attempt blocked new signed fix: %v %v %v", handled, err, u.effects)
}
+ if _, err := h.Refusals(release.HistoryBinding{Host: b.Route.Host, Account: b.Route.Account, App: b.Route.App}); err == nil {
+ t.Fatal("superseded ambiguous outcome erased by success")
+ }
}
func TestPlan14ERecoveryBindingRefusalKeepsPromotedFix(t *testing.T) {
@@ -203,16 +208,24 @@ func TestPlan14ERecoveryBindingRefusalKeepsPromotedFix(t *testing.T) {
n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink}
handled, err := n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg)
live, liveErr := release.LiveVersion(u.opt, observedHost)
- disk, openErr := c.attempts()
- if openErr != nil {
- t.Fatal(openErr)
+ if handled || err != nil || liveErr != nil || live != "v1" || len(u.effects) != 0 {
+ t.Fatalf("no-census recovery selected effects: handled=%v err=%v live=%s/%v effects=%v", handled, err, live, liveErr, u.effects)
}
- defer disk.Close()
- attempt, readErr := readRollbackAttempt(disk, observedHost)
- if readErr != nil || attempt.Failed || !attempt.Settled {
- t.Fatalf("binding refusal poisoned promotion: %+v %v", attempt, readErr)
+ rt, err := e.root.reg.Get(observedHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := release.StageSignedHead(t.Context(), e.store, pub, nil, u.opt, *rt); err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PointLive(u.opt, observedHost, rt.Account, "v2", rt.App); err != nil {
+ t.Fatal(err)
+ }
+ if err := legacyCheckedUnitRestart(u, u.opt, func(string) string { return "5s" }, *rt, nil, c); err != nil {
+ t.Fatal(err)
}
- if !handled || err != nil || liveErr != nil || live != "v2" || len(u.effects) != 1 {
- t.Fatalf("binding refusal undid promotion: handled=%v err=%v live=%s/%v effects=%v", handled, err, live, liveErr, u.effects)
+ live, liveErr = release.LiveVersion(u.opt, observedHost)
+ if liveErr != nil || live != "v2" || len(u.effects) != 1 {
+ t.Fatalf("main outcome changed: %s/%v %v", live, liveErr, u.effects)
}
}
diff --git a/internal/host/plan14_e7_legacy_linux_test.go b/internal/host/plan14_e7_legacy_linux_test.go
index 5f1b48ce5..f5071785f 100644
--- a/internal/host/plan14_e7_legacy_linux_test.go
+++ b/internal/host/plan14_e7_legacy_linux_test.go
@@ -138,3 +138,19 @@ func TestR2ERecoveryKeepsCensusBoundary(t *testing.T) {
})
}
}
+
+func TestR2ERecoveryFlagWithoutCensus(t *testing.T) {
+ if !plan14RootFixture(t) {
+ return
+ }
+ f := r2Fixture(t, "none")
+ t.Setenv("CARLOS_HOST_CLAIM", "1")
+ if legacy, err := legacyRootRestart(); err != nil || !legacy {
+ t.Fatalf("flag selected bound effects without a census: %v %v", legacy, err)
+ }
+ n := namedUnitRecovery{source: f.source, pub: f.pub, readLive: os.Readlink}
+ handled, err := n.recover(t.Context(), f.rt.Host, f.c, func(string) string { return "5s" }, f.e.root.reg)
+ if err != nil || handled {
+ t.Fatalf("E claimed no-census route: %v %v", handled, err)
+ }
+}
diff --git a/internal/host/plan14_e8_outcome_test.go b/internal/host/plan14_e8_outcome_test.go
index c52fafe86..7ed2b6f03 100644
--- a/internal/host/plan14_e8_outcome_test.go
+++ b/internal/host/plan14_e8_outcome_test.go
@@ -23,7 +23,21 @@ func TestPlan14ERecoveryRestartOutcome(t *testing.T) {
for _, fault := range []string{"binding", "final binding", "exe", "failed", "auto-restart", "not ready"} {
t.Run(fault, func(t *testing.T) {
e, u, c, source, pub := signedRecoveryFixture(t)
+ if err := e.root.reg.SetVersions(observedHost, "v1", ""); err != nil {
+ t.Fatal(err)
+ }
u.afterRestart = func() { u.props.NRestarts = 0 }
+ u.afterEffect = func() {
+ if len(u.effects) != 1 {
+ return
+ }
+ if fault == "failed" {
+ u.props.ActiveState, u.props.MainPID = "failed", 0
+ }
+ if fault == "auto-restart" {
+ u.props.ActiveState, u.props.SubState, u.props.NRestarts = "activating", "auto-restart", 1
+ }
+ }
c.legacy = func() (bool, error) { return true, nil }
c.legacyReady = func(registry.Route) func() error {
return func() error {
@@ -61,17 +75,23 @@ func TestPlan14ERecoveryRestartOutcome(t *testing.T) {
return e
}
fallbacks++
- return legacyCheckedUnitRestart(u, u.opt, func(string) string { return "5s" }, *rt, nil, c)
+ return legacyCheckedUnitRestart(source.sys, u.opt, func(string) string { return "5s" }, *rt, nil, c)
}, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRecovery(func(ctx context.Context, rt registry.Route) (bool, error) {
return n.recover(ctx, rt.Host, c, func(string) string { return "5s" }, e.root.reg)
- }, c.allowLegacy))
+ }, c.allowLegacy), release.WithRollbackRestart(func(host string) error {
+ rt, err := e.root.reg.Get(host)
+ if err != nil {
+ return err
+ }
+ return legacyRollbackRestart(source.sys, u.opt, func(string) string { return "5s" }, *rt, c)
+ }))
failed := fault == "failed" || fault == "auto-restart" || fault == "not ready"
want, count := "v2", 1
if failed {
want, count = "v1", 3
}
live, liveErr := release.LiveVersion(u.opt, observedHost)
- if liveErr != nil || live != want || len(u.effects) != count || fallbacks != 0 || (err != nil) != failed {
+ if liveErr != nil || live != want || len(u.effects) != count || fallbacks != 1 || (err != nil) != failed {
t.Fatalf("outcome: live=%s/%v effects=%v fallback=%d err=%v", live, liveErr, u.effects, fallbacks, err)
}
stamp, stampErr := e.root.reg.Get(observedHost)
@@ -79,9 +99,11 @@ func TestPlan14ERecoveryRestartOutcome(t *testing.T) {
t.Fatalf("accepted outcome not stamped: %+v %v", stamp, stampErr)
}
disk, _ := c.attempts()
- a, readErr := readRollbackAttempt(disk, observedHost)
- if readErr != nil || !a.Settled || a.Failed != failed || (!failed && a.Verified) {
- t.Fatalf("mint evidence: %+v %v", a, readErr)
+ if names, err := disk.Names(); err != nil || len(names) != 0 {
+ t.Fatalf("no-census recovery created authority: %v %v", names, err)
+ }
+ if calls != 0 {
+ t.Fatalf("recovery selected binding: %d", calls)
}
})
}
diff --git a/internal/host/plan14_e9_census_test.go b/internal/host/plan14_e9_census_test.go
new file mode 100644
index 000000000..afd417d9a
--- /dev/null
+++ b/internal/host/plan14_e9_census_test.go
@@ -0,0 +1,110 @@
+package host
+
+import (
+ "bytes"
+ "os"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func TestPlan14ERecoveryNoCensus(t *testing.T) {
+ for _, state := range []string{"active", "failed", "inactive"} {
+ for _, flag := range []bool{false, true} {
+ t.Run(state+map[bool]string{false: "/flag off", true: "/flag on"}[flag], func(t *testing.T) {
+ e, u, c, source, pub := signedRecoveryFixture(t)
+ c.legacy = func() (bool, error) { return true, nil }
+ c.legacyReady = c.ready
+ if state != "active" {
+ u.props.ActiveState, u.props.MainPID = state, 0
+ }
+ n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink, writes: hostwrite.Config{Enabled: flag, Selector: namedRecoveryWork{}}}
+ handled, err := n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg)
+ live, liveErr := release.LiveVersion(u.opt, observedHost)
+ if handled || err != nil || liveErr != nil || live != "v1" || len(u.effects) != 0 {
+ t.Fatalf("no-census recovery drove an effect: handled=%v err=%v live=%s/%v effects=%v", handled, err, live, liveErr, u.effects)
+ }
+ disk, _ := c.attempts()
+ if names, err := disk.Names(); err != nil || len(names) != 0 {
+ t.Fatalf("recovery created authority: %v %v", names, err)
+ }
+ })
+ }
+ }
+}
+
+func TestPlan14ERecoveryNoCensusResume(t *testing.T) {
+ for _, independent := range []bool{false, true} {
+ t.Run(map[bool]string{false: "linked", true: "independent"}[independent], func(t *testing.T) {
+ e, u, c, _, _ := signedRecoveryFixture(t)
+ b, err := c.host(t.Context(), observedHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ disk, err := c.attempts()
+ if err != nil {
+ t.Fatal(err)
+ }
+ target := release.VersionPath(u.opt, b.Route.Account, "v2", b.Route.App)
+ a, err := captureRollback(disk, b, c.now(), unitSettleTimeout(func(string) string { return "5s" }), target, c.identity, historyRollbackLink{Binding: release.HistoryBinding{Host: b.Route.Host, Account: b.Route.Account, App: b.Route.App}, Attempt: release.HistoryAttempt{ID: "linked", Source: b.Source, Version: "v2", SHA256: "failed-digest", Platform: release.AdoptPlatform(), Started: c.now(), PromotedAt: c.now()}, Independent: independent})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.StageArtifact(u.opt, b.Route.Account, "v2", b.Route.App, []byte("new build")); err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PointLive(u.opt, observedHost, b.Route.Account, "v2", b.Route.App); err != nil {
+ t.Fatal(err)
+ }
+ u.props.ActiveState, u.props.MainPID = "failed", 0
+ before, err := disk.Read(rollbackJournalKey(a.Host))
+ if err != nil {
+ t.Fatal(err)
+ }
+ rt, _ := e.root.reg.Get(observedHost)
+ h := seedHostRootHistory(t, *rt)
+ c.history = func() (*release.RootHistoryPass, error) { return &release.RootHistoryPass{History: h}, nil }
+ c.legacy = func() (bool, error) { return true, nil }
+ if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil {
+ t.Fatal(err)
+ }
+ after, err := disk.Read(rollbackJournalKey(a.Host))
+ if err != nil {
+ t.Fatal(err)
+ }
+ live, err := release.LiveVersion(u.opt, observedHost)
+ if err != nil || live != "v2" || len(u.effects) != 0 || !bytes.Equal(before, after) {
+ t.Fatalf("resume changed effects/evidence: live=%s/%v effects=%v journalChanged=%v", live, err, u.effects, !bytes.Equal(before, after))
+ }
+ if independent {
+ if _, err := h.Refusals(a.HistoryBinding); err == nil {
+ t.Fatal("no-census observation lost pending evidence")
+ }
+ if _, err := c.mintRefusals(h, a.HistoryBinding); err == nil {
+ t.Fatal("pending recovery granted mint")
+ }
+ }
+ })
+ }
+}
+
+func TestPlan14ERecoveryOrdinaryNoCensusRead(t *testing.T) {
+ e, _, c, source, pub := signedRecoveryFixture(t)
+ rt, err := e.root.reg.Get(observedHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rt.Unit = ""
+ if err := e.root.reg.Delete(rt.Host); err != nil {
+ t.Fatal(err)
+ }
+ if err := e.root.reg.Upsert(*rt); err != nil {
+ t.Fatal(err)
+ }
+ c.legacy = func() (bool, error) { t.Fatal("ordinary E recovery read census"); return true, nil }
+ n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink}
+ if handled, err := n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg); handled || err != nil {
+ t.Fatalf("ordinary recovery: %v %v", handled, err)
+ }
+}
diff --git a/internal/host/plan14_recovery_mint_test.go b/internal/host/plan14_recovery_mint_test.go
index 509d26b1f..4a0fc649d 100644
--- a/internal/host/plan14_recovery_mint_test.go
+++ b/internal/host/plan14_recovery_mint_test.go
@@ -2,6 +2,7 @@ package host
import (
"os"
+ "strings"
"testing"
"github.com/carlosframework/platform/internal/release"
@@ -20,19 +21,36 @@ func TestPlan14IndependentRecoveryRetainsMintBarrier(t *testing.T) {
if _, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg); err == nil {
t.Fatal("failed forward repair reported success")
}
- if refs, err := c.mintRefusals(h.MintRefusals, b); err != nil || len(refs) != 1 || refs[0].Version != "v2" {
+ if refs, err := c.mintRefusals(h, b); err != nil || len(refs) != 1 || refs[0].Version != "v2" {
t.Fatalf("recovery denial lost: %+v %v", refs, err)
}
if err := checkedUnitRestart(u, u.opt, func(string) string { return "5s" }, nil, c)(*rt); err != nil {
t.Fatal(err)
}
- if refs, err := c.mintRefusals(h.MintRefusals, b); err != nil || len(refs) != 1 || refs[0].Version != "v2" {
+ if refs, err := c.mintRefusals(h, b); err != nil || len(refs) != 1 || refs[0].Version != "v2" {
t.Fatalf("ordinary restart erased recovery denial: %+v %v", refs, err)
}
+ disk, err := c.attempts()
+ if err != nil {
+ t.Fatal(err)
+ }
+ names, err := disk.Names()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, name := range names {
+ if strings.HasSuffix(name, ".recovery") {
+ if err := disk.Remove(name); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ if refs, err := c.mintRefusals(h, b); err != nil || len(refs) != 1 || refs[0].SHA256 == "" {
+ t.Fatalf("removing old recovery erased retained digest: %+v %v", refs, err)
+ }
foreign := b
foreign.App = "another"
- called := false
- if _, err := c.mintRefusals(func(release.HistoryBinding) ([]release.Refusal, error) { called = true; return nil, nil }, foreign); err != nil || !called {
+ if err := c.retainRecoveryHistory(h, foreign); err != nil {
t.Fatalf("unrelated app blocked: %v", err)
}
}
@@ -71,7 +89,7 @@ func TestPlan14RecoveryMintEvidence(t *testing.T) {
case "unsynced":
s.(*rollbackMemory).failWrite = true
}
- refs, err := c.mintRefusals(h.MintRefusals, release.HistoryBinding{Host: rt.Host, Account: rt.Account, App: rt.App})
+ refs, err := c.mintRefusals(h, release.HistoryBinding{Host: rt.Host, Account: rt.Account, App: rt.App})
if state == "verified" {
if err != nil || len(refs) != 0 {
t.Fatalf("verified recovery blocked mint: %+v %v", refs, err)
@@ -82,3 +100,37 @@ func TestPlan14RecoveryMintEvidence(t *testing.T) {
})
}
}
+
+func TestPlan14IndependentRecoveryRecordsHistory(t *testing.T) {
+ for _, state := range []string{"success", "rollback", "failed rollback"} {
+ failed := state != "success"
+ t.Run(state, func(t *testing.T) {
+ e, u, c, source, pub := signedRecoveryFixture(t)
+ rt, err := e.root.reg.Get(observedHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h := seedHostRootHistory(t, *rt)
+ c.history = func() (*release.RootHistoryPass, error) { return &release.RootHistoryPass{History: h}, nil }
+ u.crashNew = failed
+ u.badRollback = state == "failed rollback"
+ n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink}
+ handled, err := n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg)
+ if !handled || (err != nil) != failed {
+ t.Fatalf("outcome: %v %v", handled, err)
+ }
+ disk, err := c.attempts()
+ if err != nil {
+ t.Fatal(err)
+ }
+ a, err := readRollbackAttempt(disk, observedHost)
+ if err != nil || !a.Settled || a.Verified != (state != "failed rollback") {
+ t.Fatalf("wrong retained verification: %+v %v", a, err)
+ }
+ refs, err := h.Refusals(release.HistoryBinding{Host: rt.Host, Account: rt.Account, App: rt.App})
+ if err != nil || len(refs) != map[bool]int{false: 0, true: 1}[failed] {
+ t.Fatalf("settlement did not retain outcome: %+v %v", refs, err)
+ }
+ })
+ }
+}
diff --git a/internal/host/recoveryrefusals.go b/internal/host/recoveryrefusals.go
index 18f5511bb..74a147248 100644
--- a/internal/host/recoveryrefusals.go
+++ b/internal/host/recoveryrefusals.go
@@ -6,66 +6,77 @@ import (
"strings"
"github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootstate"
)
func recoveryResultKey(a rollbackAttempt) string {
return rollbackHostKey(a.Host) + "." + a.ID + ".recovery"
}
-func (c rootRestartChecks) mintRefusals(base func(release.HistoryBinding) ([]release.Refusal, error), binding release.HistoryBinding) ([]release.Refusal, error) {
- if c.attempts == nil || base == nil {
- return nil, os.ErrPermission
+func recoveryHistoryOutcome(a rollbackAttempt) (release.HistoryRecovery, error) {
+ if !a.Independent || a.History == nil || a.HistoryBinding.Host != a.Host || a.History.ID != a.ID || len(a.ID) != 32 {
+ return release.HistoryRecovery{}, os.ErrPermission
}
- refs, err := base(binding)
- if err != nil {
- return nil, err
+ return release.HistoryRecovery{Binding: a.HistoryBinding, Attempt: *a.History, Settled: a.Settled, Failed: a.Failed, Recovering: a.Recovering, Verified: a.Verified}, nil
+}
+
+func (c rootRestartChecks) retainRecoveryHistory(h *release.RootHistory, binding release.HistoryBinding) error {
+ if c.attempts == nil || h == nil {
+ return os.ErrPermission
}
s, err := c.attempts()
if err != nil {
- return nil, err
+ return err
}
defer s.Close()
names, err := s.Names()
if err != nil {
- return nil, err
+ return err
}
- seen := map[string]bool{}
+ unknown := false
for _, name := range names {
if !strings.HasSuffix(name, ".attempt") && !strings.HasSuffix(name, ".recovery") {
continue
}
raw, err := s.Read(name)
if err != nil {
- return nil, err
+ return err
}
var a rollbackAttempt
- if json.Unmarshal(raw, &a) != nil || (name != rollbackJournalKey(a.Host) && name != recoveryResultKey(a)) {
- return nil, os.ErrPermission
+ if len(raw) > rootstate.MaxRecord || json.Unmarshal(raw, &a) != nil || (name != rollbackJournalKey(a.Host) && name != recoveryResultKey(a)) {
+ return os.ErrPermission
}
if !a.Independent {
continue
}
- if a.History == nil || a.HistoryBinding.Host != a.Host || a.History.ID != a.ID || a.History.SHA256 == "" {
- return nil, os.ErrPermission
+ r, err := recoveryHistoryOutcome(a)
+ if err != nil {
+ return err
}
- if a.HistoryBinding.Account != binding.Account || a.HistoryBinding.App != binding.App {
+ if r.Binding.Account != binding.Account || r.Binding.App != binding.App {
continue
}
- if !a.Settled {
- return nil, os.ErrPermission
- }
if err := s.Write(name, raw); err != nil {
- return nil, err
+ return err
}
- if seen[a.ID] {
- continue
- }
- seen[a.ID] = true
- if a.Failed || a.Recovering {
- refs = append(refs, release.Refusal{Schema: 1, Host: a.Host, App: a.HistoryBinding.App, Version: a.History.Version, SHA256: a.History.SHA256, Platform: a.History.Platform, PointerPromotedAt: a.History.PromotedAt, RefusedAt: a.Created, Kind: release.RefusalNotUp})
- } else if !a.Verified {
- return nil, os.ErrPermission
+ if err := h.RetainRecovery(r); err != nil {
+ return err
}
+ unknown = unknown || !r.Settled || (!r.Failed && !r.Recovering && !r.Verified)
+ }
+ if unknown {
+ return os.ErrPermission
+ }
+ return nil
+}
+
+func (c rootRestartChecks) mintRefusals(h *release.RootHistory, binding release.HistoryBinding) ([]release.Refusal, error) {
+ if err := c.retainRecoveryHistory(h, binding); err != nil {
+ return nil, err
+ }
+ refs, err := h.MintRefusals(binding)
+ if err != nil {
+ return nil, err
}
if len(refs) > 256 {
return nil, os.ErrPermission
diff --git a/internal/host/recoverysettle.go b/internal/host/recoverysettle.go
deleted file mode 100644
index 508ef4ab0..000000000
--- a/internal/host/recoverysettle.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package host
-
-import (
- "context"
-
- "github.com/carlosframework/platform/internal/release"
-)
-
-func settleLegacyRecovery(sys unitSystem, optDir string, getenv func(string) string, binding proofBinding, rollback bool, before, want string, ready func() error, c rootRestartChecks) error {
- opts := legacySettleOptions(optDir, getenv, c.recoveryOutcome.route, want, c)
- settle := func() error { return settleUnit(sys, binding.Route.Unit, before, opts) }
- if err := settle(); err != nil {
- return err
- }
- c.recoveryOutcome.unbound = true
- observe := func() (proofBinding, error) {
- if rollback && c.recovered != nil {
- return c.recovered(context.Background(), binding)
- }
- return c.host(context.Background(), binding.Route.Host)
- }
- first, err := observe()
- if err != nil || first.Source != binding.Source || want != release.VersionPath(optDir, first.Route.Account, first.Version, first.Route.App) ||
- (c.targetDigest != "" && first.Digest != c.targetDigest) ||
- (rollback && (first.Version != binding.Version || first.Digest != binding.Digest)) {
- return settle()
- }
- if err := ready(); err != nil {
- return settle()
- }
- last, err := observe()
- if err != nil || last.Source != first.Source || last.Version != first.Version || last.Digest != first.Digest || last.Invocation != first.Invocation || last.PID != first.PID || last.Start != first.Start {
- return settle()
- }
- c.recoveryOutcome.unbound = false
- return nil
-}
diff --git a/internal/host/roothistory.go b/internal/host/roothistory.go
index b73cb0b3e..bf2480c99 100644
--- a/internal/host/roothistory.go
+++ b/internal/host/roothistory.go
@@ -37,7 +37,11 @@ func openRootHistory(store release.Store, optDir string) (*release.RootHistoryPa
}
return got.Source, nil
}
- err = h.History.ReceiveReconstruction(ctx, store, pub, b, time.Now(), resolve)
+ checks := productionRootRestartChecks(systemdUnits{}, optDir)
+ err = checks.retainRecoveryHistory(h.History, b)
+ if err == nil {
+ err = h.History.ReceiveReconstruction(ctx, store, pub, b, time.Now(), resolve)
+ }
projection, statusErr := rootstate.Open(release.RootHistoryStatusDir, true, true)
if statusErr == nil {
statusErr = h.History.ProjectMigrationStatus(projection, b, resolve)
diff --git a/internal/host/roothistory_recovery.go b/internal/host/roothistory_recovery.go
index 25eab4b6a..5a4c38b77 100644
--- a/internal/host/roothistory_recovery.go
+++ b/internal/host/roothistory_recovery.go
@@ -3,6 +3,7 @@ package host
import (
"context"
"errors"
+ "log"
"os"
"github.com/carlosframework/platform/internal/hostwrite"
@@ -68,7 +69,25 @@ func historyRestartAttempt(s rootstate.Store, b proofBinding, c rootRestartCheck
}
func (c rootRestartChecks) settleHistory(ctx context.Context, a rollbackAttempt, rollback bool) error {
- if a.History == nil || a.Independent {
+ if a.History == nil {
+ return nil
+ }
+ if a.Independent {
+ if c.history == nil {
+ return nil
+ }
+ h, err := c.history()
+ if err == nil {
+ defer h.History.Store.Close()
+ var outcome release.HistoryRecovery
+ outcome, err = recoveryHistoryOutcome(a)
+ if err == nil {
+ err = h.History.RetainRecovery(outcome)
+ }
+ }
+ if err != nil {
+ log.Print(err)
+ }
return nil
}
if c.history == nil {
@@ -124,6 +143,16 @@ func resumeHistoryRestart(ctx context.Context, sys unitSystem, optDir string, ge
}
return checks.settleHistory(ctx, a, a.Recovering)
}
+ legacy, err := checks.allowLegacy()
+ if err != nil {
+ return err
+ }
+ if legacy {
+ if a.Independent {
+ return checks.settleHistory(ctx, a, a.Recovering)
+ }
+ return nil
+ }
_, old, err := loadRollback(s, a.Host, checks.now())
if err != nil {
return err
@@ -188,6 +217,10 @@ func resumeHistoryRestart(ctx context.Context, sys unitSystem, optDir string, ge
if err != nil {
return err
}
+ checks.targetDigest = a.PreviousDigest
+ if forward {
+ checks.targetDigest = a.History.SHA256
+ }
if err := restartBoundUnit(sys, optDir, getenv, old, !forward, nil, checks); err != nil {
if forward {
a.Failed = true
@@ -196,11 +229,17 @@ func resumeHistoryRestart(ctx context.Context, sys unitSystem, optDir string, ge
}
return resumeHistoryRestart(ctx, sys, optDir, getenv, checks, writes, s, a)
}
- return errors.Join(err, settleRollbackAttempt(s, a))
+ settledErr := settleRollbackAttempt(s, a)
+ if settledErr == nil && a.Independent {
+ a.Settled = true
+ _ = checks.settleHistory(ctx, a, !forward)
+ }
+ return errors.Join(err, settledErr)
}
- a.Verified = checks.recoveryOutcome == nil || !checks.recoveryOutcome.unbound
+ a.Verified = true
if err := settleRollbackAttempt(s, a); err != nil {
return err
}
+ a.Settled = true
return checks.settleHistory(ctx, a, !forward)
}
diff --git a/internal/host/rootrestart.go b/internal/host/rootrestart.go
index 8bc64065e..00eec5b6d 100644
--- a/internal/host/rootrestart.go
+++ b/internal/host/rootrestart.go
@@ -21,14 +21,7 @@ import (
"github.com/carlosframework/platform/internal/rootstate"
)
-type recoveryRestartOutcome struct {
- route registry.Route
- effect bool
- unbound bool
-}
-
type rootRestartChecks struct {
- recoveryOutcome *recoveryRestartOutcome
preserveOutcome bool
history func() (*release.RootHistoryPass, error)
readLive func(string) (string, error)
@@ -307,9 +300,6 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string,
if err := sys.ResetFailed(rt.Unit); err != nil {
return err
}
- if c.recoveryOutcome != nil {
- c.recoveryOutcome.effect = true
- }
if err := sys.Restart(rt.Unit); err != nil {
return err
}
@@ -331,14 +321,8 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string,
if err := sys.ResetFailed(rt.Unit); err != nil {
return err
}
- if c.recoveryOutcome != nil {
- c.recoveryOutcome.effect = true
- }
return sys.Restart(rt.Unit)
}
- if c.recoveryOutcome != nil {
- c.recoveryOutcome.effect = true
- }
if err := sys.TryRestart(rt.Unit); err != nil {
return err
}
@@ -347,9 +331,6 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string,
}); err != nil {
return err
}
- if c.recoveryOutcome != nil {
- return settleLegacyRecovery(sys, optDir, getenv, binding, rollback, before.InvocationID, want, ready, c)
- }
var observed proofBinding
observe := func() (proofBinding, error) {
if rollback && c.recovered != nil {
diff --git a/internal/provision/testdata/plan14-root-reads-e.tsv b/internal/provision/testdata/plan14-root-reads-e.tsv
index c59897302..fc2a7b505 100644
--- a/internal/provision/testdata/plan14-root-reads-e.tsv
+++ b/internal/provision/testdata/plan14-root-reads-e.tsv
@@ -1,25 +1,25 @@
# Track E integration after merge3. Earlier baselines, overlays and legacy fingerprints are immutable.
-internal/host/agent.go runAdoptUnderRootRelease authority B3,E1 Root retains independent recovery results and includes exact failed target digests in proof mint refusals. Pending, malformed or unsynced evidence refuses minting; recovery remains independent. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14RecoveryMintEvidence, TestPlan14SignedActiveFixIgnoresHistory. E7 applies merge ruling 4: unbound no-census routes use the legacy pass and restart; ordinary candidates make no binding store call. Fixtures: TestR2UnguardedERecoveryUpdate, TestR2UnguardedERecoveryIncidentRecipe, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix. registry:Host@14260,whole:Route@10431,whole:Route@17915,whole:Route@17951,whole:Route@7852 f24943a35599f7a46117aff91d3d3c11507757c7f0688f6f831fecee29a88f84
+internal/host/agent.go runAdoptUnderRootRelease authority B3,E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. registry:Host@14260,whole:Route@10431,whole:Route@17902,whole:Route@17938,whole:Route@7852 27e449dcb59c49398197e2c2ce2f49a9a12747bd6a48200728bd53d9a7ed40c3
internal/host/agent.go runAdoptWithEnsureSystemUnits authority B3,C,D,E1,E2,E3,E4,F,J1 E1 selects the checked private history, canonical named-unit provider and root mint inventory under release exclusion with claims off or on; reconstruction and restart consumption remain incomplete. file:Open@191 9cdd0a958f5924c9c8d8c5613fd665f2c96a194cd11322adb6f0050fb183b541
internal/host/namedrecovery.go namedUnitRecovery.repair authority E1,J1 Signed channel target, canonical deployment record, checked named unit and root-owned live pointer authorize stopped-unit pointer repair. Rechecks source, claim selection, pointer and artifact at the effect; does not start a process or change grants. Fixtures: TestPlan14Incident20261006WithLedger and TestPlan14SignedNamedRecoveryAuthority. Uses account/app from the same canonical read as the unit binding. Fixture: TestPlan14SignedNamedRecoveryUsesOneCanonicalRecord. file:Open@2565,registry:Channel@1814,whole:Route@1060,whole:Route@1822 8a42a6a27537be49d30d32891f407789449f9b77d37674fea5cb288d908adf6a
-internal/host/namedrecovery_active.go namedUnitRecovery.recover authority B3,E1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding, TestPlan14ERecoveryStrictNotUpPrecedesBinding. file:Open@5928,registry:Channel@2779,registry:Unit@342,registry:Unit@359,registry:Version@3777,registry:Version@3886,registry:Version@4173,registry:Version@4279,whole:Route@1699,whole:Route@2781,whole:Route@329,whole:Route@3700,whole:Route@4099,whole:Route@468,whole:Route@476,whole:Route@7138 1b2296098659ba83def943d8cbb75cf537fad2ef97b78c093262ff3586c3728a
+internal/host/namedrecovery_active.go namedUnitRecovery.recover authority B3,E1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. file:Open@5386,registry:Channel@2511,registry:Unit@216,registry:Unit@233,registry:Version@3500,registry:Version@3606,whole:Route@1431,whole:Route@202,whole:Route@2513,whole:Route@3426 22054f9fecd0e7c896087188ba5f9cf84ffdf3f5d8afca7574f29569b9311332
internal/host/proofbinding.go proofBinding.authorizedRecovery authority B3,E1 Independent signed recovery authority and retained attempt outcomes. Fixtures: TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14ActiveRecoveryProcessCrash. a7a7cc9edf8437ed2a1fd4d4ed3d54e9ffab78b21ed3d2867a2254c7836e4f13
internal/host/proofbinding.go proofBindingSource.binding authority B3,J1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@1176,file:EvalSymlinks@1485,file:EvalSymlinks@518,file:EvalSymlinks@757,registry:Unit@3161,whole:Route@265,whole:Route@3658,whole:Route@3665 56edaf01411de3813fe327973302b1df35e403d2870625fc341355b5bc5cfa0b
internal/host/proofbinding.go proofBindingSource.recovered authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Host@123,whole:Route@125 9ddd5ae1e5b981654e47a0f97cba43b93e32617feddf57918136c57c9943f8ae
internal/host/proofbinding.go proofBindingSource.unitBinding authority B3,J1,E1 Reads canonical deployment record, claim, unit file identities and systemd facts without assuming a running PID. Existing running/rollback binding uses the same verifier. Fixtures: TestPlan14SignedNamedRecoveryAuthority and TestPlan14ProofBindingProvenance. registry:Unit@508,registry:Unit@609,registry:Unit@800,whole:Route@2707,whole:Route@2714 fc6bbd54101f36b986ea48656973667139575d20b75780ad3892915904ef5151
internal/host/proofroot.go proofRoot.mint authority B3,J1,J4,E1 E1 production uses canonical observed programs and complete root refusal inventory; legacy diagnostics do not grant root mint authority. registry:Account@136,registry:App@1338,registry:App@1960,registry:App@2075,registry:App@2279,registry:App@316,registry:App@934,registry:Host@1135,registry:Host@1259,registry:Host@1599,registry:Host@2066,registry:Host@2189,registry:Host@287,registry:Host@469,registry:Host@622,whole:Route@129,whole:Route@144 8c4e2457e74ac917a5be303d74493247190e344585da622e989048fd0e00e61a
internal/host/proofroot.go proofRoot.observeAndRecord authority B3,J1,J4,E1 E1 production uses canonical observed programs and complete root refusal inventory; legacy diagnostics do not grant root mint authority. file:Readlink@108,registry:Account@443,registry:App@468,registry:Host@151,registry:Host@657,whole:Route@186,whole:Route@453,whole:Route@478 88065f1cfe7ad31178ba65c2672d8dd1e25fab26820290368fc063907f7b3b98
-internal/host/recoveryrefusals.go rootRestartChecks.mintRefusals authority B3,E1 Root retains independent recovery results and includes exact failed target digests in proof mint refusals. Pending, malformed or unsynced evidence refuses minting; recovery remains independent. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14RecoveryMintEvidence, TestPlan14SignedActiveFixIgnoresHistory. file:Read@616 1bb27522e17f4c16f7c09390f2a132ce6a1050e73fc807179d1fa9f4ff71278e
+internal/host/recoveryrefusals.go rootRestartChecks.mintRefusals authority B3,E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 8087848fae0c92cc94908a7565019249aac203dbb3efbc313fafdf4b1a55b49d
internal/host/roothistory.go boundRootHistory authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. file:Open@1061,file:Read@1379,registry:Account@613,registry:App@635,registry:Channel@456,registry:Channel@470,registry:Host@590,whole:Route@1469,whole:Route@377,whole:Route@472,whole:Route@592,whole:Route@615,whole:Route@637,whole:Route@846 fe46269101116374044658bbb87c1bd44317525dcadf227d4515b47845238be2
-internal/host/roothistory.go openRootHistory authority E1 E1 reconstruction status reads exact private category revisions and canonical sources twice; public projection grants nothing. Fixtures: TestPlan14HistoryMigrationStatus, TestPlan14HistoryPendingStatus, TestPlan14HistoryStatusReplacementDelivery. Recovery outcome integration remains incomplete. file:Open@104,file:Open@980,registry:Account@740,registry:App@777,whole:Route@744,whole:Route@781 6615b63fd53196099dc7ced185ad25bf941c201dba4078481ac8e8e6b3aa854d
+internal/host/roothistory.go openRootHistory authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Open@104,file:Open@1118,registry:Account@740,registry:App@777,whole:Route@744,whole:Route@781 28c836abc244d5bca5c22db2ed565b8d0e60e381f846401ab6969f339a1c67fa
internal/host/roothistory.go recordRootProgram authority E1 E1 checked private store and canonical record/unit/process observation; program read is hashed and followed by another binding, readiness and process check. whole:Route@151,whole:Route@297,whole:Route@398 fbbc9451c39309090580486d75da6120b02e1ba054157159846bea057ae0c80f
internal/host/roothistory_recovery.go historyRestartAttempt authority B3,E1 Captured root capability and exact process/source identity authorize restart independently of release history availability. Fixture: TestPlan14HistoryUnavailableAfterCaptureCannotBlockRestart. registry:Host@187,registry:Host@361,whole:Route@189,whole:Route@363 1c2af7b90cb56053c1d58c10ccaa851c3aa25b0be9e595bf8e90a013ad0bae66
internal/host/roothistory_recovery.go linkRootHistory authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Prepare@121,registry:Account@374,registry:App@412,whole:Route@376,whole:Route@414 a60324796d2b25b3b130bb0439bbd9a415b679e6b51bc92a44f23f77e2d93e1e
-internal/host/roothistory_recovery.go resumeHistoryRestart authority B3,E1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding. registry:Account@1025,registry:Account@2194,registry:App@1057,registry:App@2226,registry:Unit@824,whole:Route@1029,whole:Route@1061,whole:Route@2198,whole:Route@2230,whole:Route@828 79d86ba8876ab6cdeced4f27b0bf1f16097f2d56da1c08ae4839b5fc4143ec47
-internal/host/roothistory_recovery.go rootRestartChecks.settleHistory authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. registry:Account@580,registry:App@1351,registry:App@627,whole:Route@1353,whole:Route@582,whole:Route@629,whole:Route@845,whole:Route@991 ea977169cb738ba5e3e9bb83f86fbe0fca6e66ec6fcd9416dae94e8f39bc47fc
+internal/host/roothistory_recovery.go resumeHistoryRestart authority B3,E1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. registry:Account@1202,registry:Account@2371,registry:App@1234,registry:App@2403,registry:Unit@1001,whole:Route@1005,whole:Route@1206,whole:Route@1238,whole:Route@2375,whole:Route@2407 8263cf0571efdd448b944fdb8492ce78ddf0223fdef18255eab97c40434eef60
+internal/host/roothistory_recovery.go rootRestartChecks.settleHistory authority B3,E1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. registry:Account@911,registry:App@1682,registry:App@958,whole:Route@1176,whole:Route@1322,whole:Route@1684,whole:Route@913,whole:Route@960 72c220e95eff0ea648de00acf7da10cdb49842e9a949d074dd267622f92eb03a
internal/host/rootrestart.go checkedUnitRestart authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@1798,registry:Account@1367,registry:Account@2166,registry:App@1399,registry:App@2198,registry:Host@1069,registry:Host@1854,registry:Host@2741,registry:Host@2963,registry:Host@3606,registry:Host@510,registry:Unit@3488,registry:Unit@711,whole:Route@1375,whole:Route@1407,whole:Route@2174,whole:Route@2206,whole:Route@3496,whole:Route@638,whole:Route@719 3a29634563392118e927685e478c8265916fbe89d922b9dde4c08aa5178c1c3c
internal/host/rootrestart.go productionRootRestartChecks authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Open@1356,whole:Route@1189,whole:Route@1990,whole:Route@2257 921445f1e11f9bfd202e309a0b0be4864f057a0907164f32ec8603f6949f4bdd
-internal/host/rootrestart.go restartBoundUnit authority B3,J1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding, TestPlan14ERecoveryStrictNotUpPrecedesBinding. registry:Account@2848,registry:Account@4333,registry:App@2884,registry:App@4373,registry:Host@1659,registry:Host@2667,registry:Host@650,registry:Unit@1071,registry:Unit@1213,registry:Unit@1968,registry:Unit@2113,registry:Unit@2229,registry:Unit@3665,registry:Unit@395,whole:Route@198,whole:Route@2854,whole:Route@2890,whole:Route@317,whole:Route@4341,whole:Route@4381,whole:Route@4592 123efb827a00d4350aa69ab03560cc69b460e3b5cf7f7daed181346efe784fae
+internal/host/rootrestart.go restartBoundUnit authority B3,J1 E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. registry:Account@2482,registry:Account@3967,registry:App@2518,registry:App@4007,registry:Host@1585,registry:Host@2301,registry:Host@650,registry:Unit@1071,registry:Unit@1139,registry:Unit@1894,registry:Unit@1962,registry:Unit@2004,registry:Unit@3299,registry:Unit@395,whole:Route@198,whole:Route@2488,whole:Route@2524,whole:Route@317,whole:Route@3975,whole:Route@4015,whole:Route@4226 e379b31272674c4983e0d0aea66a1091d6537ece5ec05b64474cc292a8f40ba4
internal/host/rootrestart.go rollbackUnitRestart authority B3,J1,E1 Captured canonical rollback authority is unchanged. Durable verified outcome permits later history settlement without another restart. Fixtures: TestPlan14HistoryRollbackLink and TestPlan14HistoryUnavailableCannotBlockCapturedRollback. file:Read@1458,registry:Host@1750,registry:Unit@1852,registry:Unit@2153,registry:Unit@435,registry:Unit@692,whole:Route@1758,whole:Route@1860,whole:Route@2161,whole:Route@422,whole:Route@774 4d02504999ba2746f3815cba0e3ca7fe8723cdf2528e57d3fa78b5acc9b90998
internal/host/rootrestart.go rootRestartChecks.live authority B3,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:EvalSymlinks@158 28721577488e77d229d3a5cbe6eb5806a4af0696d350a144c5ed2e4d10c36828
internal/host/rootrollback.go captureRollback authority B3,J1,E1 Signed named-unit recovery is independent of release and health history. Root rechecks canonical source, unit, process, signed target and trusted live pointer; captures attempt authority before publication; verifies readiness and resumes without repeated effects. Fixtures: TestPlan14SignedActiveFixIgnoresHistory, TestPlan14SignedActiveFixAuthority, TestPlan14SignedFixSupersedesOlderAttempt, TestPlan14ActiveRecoveryProcessCrash, TestPlan14SignedRestartWithoutHealthHistory, TestPlan14CapturedRollbackWithoutHealthHistory, TestPlan14SignedRestartRequiresTrustedLiveLink. file:Read@704,registry:Account@1191,registry:App@1227,registry:Host@1154,registry:Host@341,registry:Host@823,whole:Route@1156,whole:Route@1193,whole:Route@1229,whole:Route@343,whole:Route@825 6d3ec51488ef1be9dac2d5fe168cb38e72614242fd18064ba441273b14af6efa
@@ -33,7 +33,7 @@ internal/release/rollback.go writeGroupRefusals authority E1 E1 group refusal wr
internal/release/rootbake.go RootHistory.LoadBake authority E1 E1 strict root bake and app completeness revisions; backward clocks refuse and visible records are durably recommitted. file:Read@261 bd188ea6f2e76d277e1274043bf0f2a447391b1dc540aa400b36ad2573ccd84e
internal/release/roothistory.go RootHistory.read authority E1 Root per-binding refusal/program completeness and pending attempt state; strict decode and exact durable recommit before authority reads. E1 production integration is selected but incomplete. 4af99259d080d2b4c4ffee7cf2bc324e665cb07416031a495692421bd21878b4
internal/release/roothistory.go RootHistory.readRecord authority E1 Root reconstruction reads exact durable records and preserves pending restrictions during signed replacement. Fixtures: TestPlan14ReconstructionReplacement, TestPlan14ReconstructionReplacementCrash. file:Read@189 c98eb40269145d3e3d5b41f18e9779f30e618ed0884c556e9bf132bfdc9dfae6
-internal/release/roothistory_app.go RootHistory.AppRefusals authority E1 E1 explicit app inventory plus retained host records and per-host completeness; missing, extra, malformed or pending histories refuse minting. file:Read@297,file:Read@819 9df40b5eb5b878ceba87b596204cddce04183cf78df4d1e5cd0dcbfe13e5d25e
+internal/release/roothistory_app.go RootHistory.AppRefusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@297,file:Read@819 4cc038ec851042efe7878b15462c8942783d3b5878873a74910a0b3d16b7f3b5
internal/release/roothistory_delivery.go RootHistory.ReceiveReconstruction authority E1 Signed replacement binds the exact pending digest and current sources/revisions; old intended refusals and hold bounds remain restrictive. Fixtures: TestPlan14ReconstructionReplacement, TestPlan14ReconstructionReplacementCrash. de43924ddbe526035c8987582d58907cf1d9258fca0cbb56997943487316bb2c
internal/release/roothistory_pass.go RootHistoryPass.CompleteAttempt authority B3,E1 Root history intent is linked to captured canonical rollback authority before live effects; verified recovery settles only its exact intent. Fixtures: TestPlan14HistoryRollbackLink, TestPlan14HistoryRecoveryRefusesChanges, TestPlan14HistoryRecoveryProcessCrash, TestPlan14HistorySettlementRetriesVisibleSync. E1 migration and restart consumption remain incomplete. whole:Route@136,whole:Route@255 97d46675209fa1d5ba064afded9b8b3265b741be9a30dfec6efab01a745d1d82
internal/release/roothistory_pass.go RootHistoryPass.begin authority E1 Root history intent is linked to captured canonical rollback authority before live effects; verified recovery settles only its exact intent. Fixtures: TestPlan14HistoryRollbackLink, TestPlan14HistoryRecoveryRefusesChanges, TestPlan14HistoryRecoveryProcessCrash, TestPlan14HistorySettlementRetriesVisibleSync. E1 migration and restart consumption remain incomplete. file:Prepare@607,file:Prepare@638,file:Read@292,whole:Route@218 dde605d1d382309a8546c7fb4a78e276909bc7741a83b93c93d0f2ec9c783004
@@ -54,9 +54,18 @@ internal/release/roothistory_replace.go replacementReconstruction authority E1 S
internal/release/signedhead.go StageSignedHead authority E1 Canonical route is supplied by the root unit verifier. Verifies signed channel, digest ratchet and manifest/artifact before staging; unavailable pin census never authorizes restaging differing bytes. Fixtures: TestPlan14Incident20261006WithLedger and TestPlan14SignedNamedRecoveryAuthority. registry:Account@202,registry:Account@302,registry:Account@403,registry:App@214,registry:App@415,registry:Channel@222,registry:Host@434 ca02f04e55556ba0233dfa0848b0c533a9d9aa06cad6abd33cfd2968b3bd838a
internal/rootstate/link_linux.go ReadTrustedLink authority A,E1 Checks the full root-owned parent chain before descriptor-based live-link reading. Fixture: TestPlan14TrustedLiveLink; privileged acceptance remains required. file:Open@156 90f8e8de4e2c71f07b5d749cbe4bb1c1dfa426576a82138823449f56fc9ec5ab
internal/rootstate/link_linux.go directory.readLink authority A,E1 Pins the symlink inode with O_PATH and checks owner/type/link count before readlinkat. Fixture: TestPlan14TrustedLiveLink. file:Fstat@356,file:Openat@183,file:Readlinkat@554 a8559fe63dce48830916004ecfe7fbbe5d419568c7daf7a3073cf1b5bb12c3d7
-@coverage internal/host f0cd6de14ac64636c123adde18a0b69ea46332ac6a075da5958a65e428e4432e
-@coverage internal/release 8d2ad3c9529ccf8a2da259399b7bb1de77b17a4914fda46c2c54c94779c3a617
+@coverage internal/host 0153cc1966470f409f75a095b0599abed5e4df65767d101d333b1a3bb032b237
+@coverage internal/release a468a41aa5baea35364345cdcd4e9e0cf212b0ebb7f413d763dd2a0def2b21e9
@coverage internal/rootstate 6338f74a6b2059cf77cf478bd6111b37b441eb9d65a309464c378359d4bdac05
-internal/release/roothistory_status.go RootHistory.MigrationStatus authority E1 E1 reconstruction status reads exact private category revisions and canonical sources twice; public projection grants nothing. Fixtures: TestPlan14HistoryMigrationStatus, TestPlan14HistoryPendingStatus, TestPlan14HistoryStatusReplacementDelivery. Recovery outcome integration remains incomplete. file:Read@1062 c4dfc98bf7722bf88302eca9806027ad08eb446f0a0c8561bf926592665683a6
+internal/release/roothistory_status.go RootHistory.MigrationStatus authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@1062 14304263a38c2473fad2efc7e775974b0b5e632a7faa896ce307681d34aecabc
internal/host/legacyrestart.go restartEffectFailed authority B3 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding. b68bb65884a14fe0cf78a65f56ded4ea41584bba99af9195827d1ea191b340a9
-internal/host/recoverysettle.go settleLegacyRecovery authority E1 E8 merge ruling 5: optional identity failures preserve the same restart outcome; no second restart; real not-up takes main rollback. Fixtures: TestPlan14ERecoveryRestartOutcome, TestPlan14ERecoveryBindingRefusalKeepsPromotedFix, TestPlan14ERecoveryNotUpPrecedesBinding, TestPlan14ERecoveryStrictNotUpPrecedesBinding. registry:Account@752,registry:App@788,registry:Host@616,registry:Unit@320,whole:Route@255,whole:Route@328,whole:Route@624,whole:Route@758,whole:Route@794 65785c42a5e21854873d8d9ea952b6754cecbb766495768ef64edc3da4a446a6
+internal/host/legacyrestart.go legacyRootRestart authority B3,G E9 merge ruling 6: E effects require a census; no-census recovery delegates to main or records evidence only. Fixtures: TestPlan14ERecoveryNoCensus, TestPlan14ERecoveryNoCensusResume, TestR2ERecoveryFlagWithoutCensus, TestPlan14ERecoveryRestartOutcome. Census-backed recovery retains captured authority and outcome precedence. 5fd48683abd82664baa77c2ddc7528f92f557b04586225d64b4a6e77147df931
+internal/host/recoveryrefusals.go recoveryHistoryOutcome authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. ab25e7fa0cca603c50cfee8862c669c674e714b5a8a263748257ce26e1495491
+internal/host/recoveryrefusals.go rootRestartChecks.retainRecoveryHistory authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@473 4784879e151c40b9b1c7fb2719e3d97657d0ad387842f07e5d6a59fe8059462c
+internal/release/roothistory.go RootHistory.Blocks authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 4ea9aa88a59c8bbfdea5f76fde1f27385b31ec5bcab4a362d9630f252d744b64
+internal/release/roothistory.go RootHistory.Refusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 58467bc2224dc2cdbae451462005f1cd11c198dc41443052425eb629493b2f58
+internal/release/roothistory_outcomes.go RootHistory.RetainRecovery authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@292 2b03f6d1976c9d37748a46ae2323be25163d1622aeb4f456ba4f257f8a89d714
+internal/release/roothistory_outcomes.go RootHistory.recoveryOutcomes authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. file:Read@337 fb5dd3c1b5fe0d4124bb69cc295127f52026e19349c4e81f2d55e437747748db
+internal/release/roothistory_outcomes.go historyRecoveryKey authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 324ba2ea59a112bb176c542787407a0009bea90c3f195e03f0468706e57e7f9e
+internal/release/roothistory_outcomes.go recoveryRefusals authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 9c2bd14d9d384baea312c2c0bbd452c9040cc7b9f343d18886a19afdb17c02be
+internal/release/roothistory_outcomes.go validHistoryRecovery authority E1 E1 retains independently recorded outcomes in private history before minting/reconstruction. Failure bits never clear by replay, ambiguous attempts remain restrictive, and signed reconstruction cannot erase them. Fixtures: TestPlan14IndependentRecoveryRetainsMintBarrier, TestPlan14IndependentRecoveryRecordsHistory, TestPlan14HistoryRetainsRecoveryOutcomes, TestPlan14HistoryRecoveryCannotWeaken, TestPlan14HistoryReconstructionKeepsRecovery, TestPlan14HistoryRecoveryProcessCrash. 8d28b60e48d8c3991876a9213f1f51e30fbf0e31d1a6245c9cfe374a37277a06
diff --git a/internal/release/plan14_history_outcomes_process_test.go b/internal/release/plan14_history_outcomes_process_test.go
new file mode 100644
index 000000000..cc70c51aa
--- /dev/null
+++ b/internal/release/plan14_history_outcomes_process_test.go
@@ -0,0 +1,60 @@
+package release
+
+import (
+ "errors"
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+)
+
+type recoveryCrashStore struct {
+ *healthyRetryStore
+ boundary string
+}
+
+func (s *recoveryCrashStore) Write(name string, raw []byte) error {
+ if !strings.HasSuffix(name, ".outcome") {
+ return s.healthyRetryStore.Write(name, raw)
+ }
+ if s.boundary == "visible" {
+ s.fail = 1
+ }
+ err := s.healthyRetryStore.Write(name, raw)
+ if err != nil && s.boundary != "visible" {
+ return err
+ }
+ os.Exit(74)
+ return nil
+}
+
+func TestPlan14HistoryRecoveryProcessCrash(t *testing.T) {
+ b := HistoryBinding{Host: proofHost, Account: "bab", App: "console"}
+ now := time.Date(2026, 10, 13, 0, 0, 0, 0, time.UTC)
+ r := HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: "crashed-recovery", Source: "canonical", Version: "v2", SHA256: "failed-digest", Platform: AdoptPlatform(), Started: now, PromotedAt: now}, Settled: true, Failed: true}
+ if boundary := os.Getenv("PLAN14_OUTCOME_CRASH"); boundary != "" {
+ h := &RootHistory{Store: &recoveryCrashStore{healthyRetryStore: &healthyRetryStore{dir: os.Getenv("PLAN14_OUTCOME_DIR"), fault: errors.New("directory sync failed")}, boundary: boundary}}
+ if err := h.RetainRecovery(r); err != nil {
+ t.Fatal(err)
+ }
+ t.Fatal("crash boundary not reached")
+ }
+ for _, boundary := range []string{"durable", "visible"} {
+ t.Run(boundary, func(t *testing.T) {
+ _, s, b := rootHistoryFixture(t)
+ cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14HistoryRecoveryProcessCrash$", "-test.count=1")
+ cmd.Dir = t.TempDir()
+ cmd.Env = append(os.Environ(), "PLAN14_OUTCOME_CRASH="+boundary, "PLAN14_OUTCOME_DIR="+s.dir)
+ out, err := cmd.CombinedOutput()
+ var exit *exec.ExitError
+ if !errors.As(err, &exit) || exit.ExitCode() != 74 {
+ t.Fatalf("child: %v\n%s", err, out)
+ }
+ h := &RootHistory{Store: s}
+ if refs, err := h.Refusals(b); err != nil || len(refs) != 1 || refs[0].SHA256 != "failed-digest" {
+ t.Fatalf("crash erased refusal: %+v %v", refs, err)
+ }
+ })
+ }
+}
diff --git a/internal/release/plan14_history_outcomes_test.go b/internal/release/plan14_history_outcomes_test.go
new file mode 100644
index 000000000..478d2a095
--- /dev/null
+++ b/internal/release/plan14_history_outcomes_test.go
@@ -0,0 +1,160 @@
+package release
+
+import (
+ "bytes"
+ "github.com/carlosframework/platform/internal/rootstate"
+ "os"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestPlan14HistoryRetainsRecoveryOutcomes(t *testing.T) {
+ for _, state := range []string{"failed", "pending", "ambiguous", "verified"} {
+ t.Run(state, func(t *testing.T) {
+ h, s, b := rootHistoryFixture(t)
+ outcome := HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: "attempt-one", Version: "v2", Source: "canonical", SHA256: "digest", Platform: AdoptPlatform(), Started: time.Now().UTC(), PromotedAt: time.Now().UTC()}, Settled: state != "pending", Failed: state == "failed", Verified: state == "verified"}
+ if err := h.RetainRecovery(outcome); err != nil {
+ t.Fatal(err)
+ }
+ h = &RootHistory{Store: s}
+ refs, err := h.Refusals(b)
+ switch state {
+ case "pending", "ambiguous":
+ if err == nil {
+ t.Fatal("ambiguous outcome granted permission")
+ }
+ case "failed":
+ if err != nil || len(refs) != 1 || refs[0].SHA256 != "digest" {
+ t.Fatalf("failed digest lost: %+v %v", refs, err)
+ }
+ case "verified":
+ if err != nil || len(refs) != 0 {
+ t.Fatalf("verified outcome: %+v %v", refs, err)
+ }
+ }
+ view, err := h.MigrationStatus(b, func(HistoryBinding) (string, error) { return "canonical", nil })
+ if err != nil || len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 1 {
+ t.Fatalf("outcome not projected: %+v %v", view, err)
+ }
+ later := outcome
+ later.Attempt.ID = "attempt-two"
+ later.Settled, later.Verified, later.Failed = true, true, false
+ if err := h.RetainRecovery(later); err != nil {
+ t.Fatal(err)
+ }
+ refs, err = h.Refusals(b)
+ if (state == "pending" || state == "ambiguous") && err == nil {
+ t.Fatal("later success erased ambiguity")
+ }
+ if state == "failed" && (err != nil || len(refs) != 1) {
+ t.Fatal("later success erased failed digest")
+ }
+ })
+ }
+}
+
+func TestPlan14HistoryRecoveryCannotWeaken(t *testing.T) {
+ h, s, b := rootHistoryFixture(t)
+ outcome := HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: "attempt", Version: "v2", Source: "canonical", SHA256: "digest", Platform: AdoptPlatform(), Started: time.Now().UTC(), PromotedAt: time.Now().UTC()}, Settled: true, Failed: true}
+ if err := h.RetainRecovery(outcome); err != nil {
+ t.Fatal(err)
+ }
+ outcome.Failed, outcome.Verified = false, true
+ if err := h.RetainRecovery(outcome); err != nil {
+ t.Fatal(err)
+ }
+ if refs, err := h.Refusals(b); err != nil || len(refs) != 1 {
+ t.Fatalf("replay erased denial: %+v %v", refs, err)
+ }
+ outcome.Attempt.SHA256 = "another"
+ if err := h.RetainRecovery(outcome); err == nil {
+ t.Fatal("identity reused for different bytes")
+ }
+ s.fail = 1
+ if _, err := h.Refusals(b); err == nil {
+ t.Fatal("failed sync granted history")
+ }
+ s.fail = 0
+ if refs, err := h.Refusals(b); err != nil || len(refs) != 1 {
+ t.Fatalf("retry lost denial: %+v %v", refs, err)
+ }
+ if err := s.Write(historyRecoveryKey(outcome), []byte("{")); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := h.Refusals(b); err == nil {
+ t.Fatal("malformed outcome granted history")
+ }
+}
+
+func TestPlan14HistoryReconstructionKeepsRecovery(t *testing.T) {
+ for _, state := range []string{"failed", "ambiguous"} {
+ t.Run(state, func(t *testing.T) {
+ h, _, b, pub, wire, now := reconstructionCrashFixture(t)
+ outcome := HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: "recovery", Version: "failed-version", Source: "canonical", SHA256: "failed-digest", Platform: AdoptPlatform(), Started: now, PromotedAt: now}, Settled: true, Failed: state == "failed"}
+ if err := h.RetainRecovery(outcome); err != nil {
+ t.Fatal(err)
+ }
+ if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err != nil {
+ t.Fatal(err)
+ }
+ refs, err := h.Refusals(b)
+ if state == "ambiguous" {
+ if err == nil {
+ t.Fatal("reconstruction cleared ambiguous recovery")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ found := false
+ for _, ref := range refs {
+ found = found || ref.SHA256 == "failed-digest"
+ }
+ if !found {
+ t.Fatal("reconstruction erased failed digest")
+ }
+ })
+ }
+}
+
+type outcomeSyncStore struct {
+ rootstate.Store
+ fail bool
+}
+
+func (s *outcomeSyncStore) Write(name string, raw []byte) error {
+ if err := s.Store.Write(name, raw); err != nil {
+ return err
+ }
+ if s.fail && strings.HasSuffix(name, ".outcome") {
+ return os.ErrPermission
+ }
+ return nil
+}
+func TestPlan14HistoryRecoveryVisibleSync(t *testing.T) {
+ h, s, b := rootHistoryFixture(t)
+ fault := &outcomeSyncStore{Store: s, fail: true}
+ h.Store = fault
+ now := time.Now().UTC()
+ r := HistoryRecovery{Binding: b, Attempt: HistoryAttempt{ID: "attempt", Source: "canonical", Version: "v2", SHA256: "digest", Platform: AdoptPlatform(), Started: now, PromotedAt: now}, Settled: true, Verified: true}
+ if err := h.RetainRecovery(r); err == nil {
+ t.Fatal("visible unsynced outcome reported durable")
+ }
+ raw, err := s.Read(historyRecoveryKey(r))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := h.Refusals(b); err == nil {
+ t.Fatal("visible unsynced outcome granted permission")
+ }
+ fault.fail = false
+ if refs, err := h.Refusals(b); err != nil || len(refs) != 0 {
+ t.Fatalf("durable retry: %+v %v", refs, err)
+ }
+ after, err := s.Read(historyRecoveryKey(r))
+ if err != nil || !bytes.Equal(raw, after) {
+ t.Fatal("retry changed evidence", err)
+ }
+}
diff --git a/internal/release/roothistory.go b/internal/release/roothistory.go
index 08db29acf..d97eeea53 100644
--- a/internal/release/roothistory.go
+++ b/internal/release/roothistory.go
@@ -193,7 +193,19 @@ func (h *RootHistory) Refusals(b HistoryBinding) ([]Refusal, error) {
if !r.Refusals.Complete || r.Attempt != nil {
return nil, os.ErrPermission
}
- return slices.Clone(r.Refusals.Values), nil
+ outcomes, err := h.recoveryOutcomes()
+ if err != nil {
+ return nil, err
+ }
+ refs, err := recoveryRefusals(outcomes, b)
+ if err != nil {
+ return nil, err
+ }
+ refs = append(slices.Clone(r.Refusals.Values), refs...)
+ if len(refs) > historyLimit {
+ return nil, os.ErrPermission
+ }
+ return refs, nil
}
func (h *RootHistory) AddRefusal(b HistoryBinding, v Refusal) error {
@@ -244,7 +256,11 @@ func (h *RootHistory) Blocks(b HistoryBinding, p *ChannelPointer) (bool, error)
if !r.Refusals.Complete || r.Attempt != nil {
return true, os.ErrPermission
}
- for _, v := range r.Refusals.Values {
+ refs, err := h.Refusals(b)
+ if err != nil {
+ return true, err
+ }
+ for _, v := range refs {
if p != nil && v.Version == p.Version && (v.Kind != RefusalNotUp || v.Blocks(p)) {
return true, nil
}
diff --git a/internal/release/roothistory_app.go b/internal/release/roothistory_app.go
index 095aa6ddd..955a1e5ec 100644
--- a/internal/release/roothistory_app.go
+++ b/internal/release/roothistory_app.go
@@ -90,6 +90,15 @@ func (h *RootHistory) AppRefusals(acct, app string) ([]Refusal, error) {
return nil, os.ErrPermission
}
}
+ outcomes, err := h.recoveryOutcomes()
+ if err != nil {
+ return nil, err
+ }
+ for _, r := range outcomes {
+ if r.Binding.Account == acct && r.Binding.App == app && !slices.Contains(a.Hosts, r.Binding.Host) {
+ return nil, os.ErrPermission
+ }
+ }
var out []Refusal
for _, host := range a.Hosts {
refs, err := h.Refusals(HistoryBinding{Host: host, Account: acct, App: app})
diff --git a/internal/release/roothistory_outcomes.go b/internal/release/roothistory_outcomes.go
new file mode 100644
index 000000000..89afb7c4a
--- /dev/null
+++ b/internal/release/roothistory_outcomes.go
@@ -0,0 +1,111 @@
+package release
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "os"
+ "reflect"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/rootstate"
+)
+
+type HistoryRecovery struct {
+ Binding HistoryBinding `json:"binding"`
+ Attempt HistoryAttempt `json:"attempt"`
+ Settled bool `json:"settled"`
+ Failed bool `json:"failed"`
+ Recovering bool `json:"recovering"`
+ Verified bool `json:"verified"`
+}
+
+type historyRecoveryRecord struct {
+ Schema int `json:"schema"`
+ HistoryRecovery
+}
+
+func historyRecoveryKey(r HistoryRecovery) string {
+ sum := sha256.Sum256([]byte(r.Binding.Account + "\x00" + r.Binding.App + "\x00" + r.Binding.Host + "\x00" + r.Attempt.ID))
+ return hex.EncodeToString(sum[:]) + ".outcome"
+}
+
+func validHistoryRecovery(r HistoryRecovery) bool {
+ return r.Binding.valid() && r.Attempt.ID != "" && r.Attempt.Source != "" && r.Attempt.Version != "" && r.Attempt.SHA256 != "" && r.Attempt.Platform != "" && !r.Attempt.Started.IsZero() && !r.Attempt.PromotedAt.IsZero()
+}
+
+func (h *RootHistory) recoveryOutcomes() ([]HistoryRecovery, error) {
+ if h == nil || h.Store == nil {
+ return nil, os.ErrPermission
+ }
+ names, err := h.Store.Names()
+ if err != nil {
+ return nil, err
+ }
+ var out []HistoryRecovery
+ for _, name := range names {
+ if !strings.HasSuffix(name, ".outcome") {
+ continue
+ }
+ raw, err := h.Store.Read(name)
+ if err != nil {
+ return nil, err
+ }
+ var r historyRecoveryRecord
+ if len(raw) > rootstate.MaxRecord || strictCanonicalJSON(raw, &r) != nil || r.Schema != 1 || !validHistoryRecovery(r.HistoryRecovery) || historyRecoveryKey(r.HistoryRecovery) != name {
+ return nil, os.ErrPermission
+ }
+ if err := h.Store.Write(name, raw); err != nil {
+ return nil, err
+ }
+ out = append(out, r.HistoryRecovery)
+ }
+ return out, nil
+}
+
+func (h *RootHistory) RetainRecovery(r HistoryRecovery) error {
+ if h == nil || h.Store == nil || !validHistoryRecovery(r) {
+ return os.ErrPermission
+ }
+ r.Attempt.Started, r.Attempt.PromotedAt = r.Attempt.Started.UTC(), r.Attempt.PromotedAt.UTC()
+ key := historyRecoveryKey(r)
+ raw, err := h.Store.Read(key)
+ if err == nil {
+ var old historyRecoveryRecord
+ if len(raw) > rootstate.MaxRecord || strictCanonicalJSON(raw, &old) != nil || old.Schema != 1 || !validHistoryRecovery(old.HistoryRecovery) || old.Binding != r.Binding || !reflect.DeepEqual(old.Attempt, r.Attempt) {
+ return os.ErrPermission
+ }
+ r.Settled = r.Settled || old.Settled
+ r.Failed = r.Failed || old.Failed
+ r.Recovering = r.Recovering || old.Recovering
+ r.Verified = r.Verified || old.Verified
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ raw, err = json.Marshal(historyRecoveryRecord{Schema: 1, HistoryRecovery: r})
+ if err != nil {
+ return err
+ }
+ return h.Store.Write(key, raw)
+}
+
+func recoveryRefusals(outcomes []HistoryRecovery, b HistoryBinding) ([]Refusal, error) {
+ var refs []Refusal
+ count := 0
+ for _, r := range outcomes {
+ if r.Binding != b {
+ continue
+ }
+ count++
+ if count > historyLimit || !r.Settled {
+ return nil, os.ErrPermission
+ }
+ if r.Failed || r.Recovering {
+ refs = append(refs, Refusal{Schema: 1, Host: b.Host, App: b.App, Version: r.Attempt.Version, SHA256: r.Attempt.SHA256, Platform: r.Attempt.Platform, PointerPromotedAt: r.Attempt.PromotedAt, RefusedAt: r.Attempt.Started, Kind: RefusalNotUp})
+ } else if !r.Verified {
+ return nil, os.ErrPermission
+ }
+ }
+ return refs, nil
+}
diff --git a/internal/release/roothistory_status.go b/internal/release/roothistory_status.go
index da1c8f7ad..a51cca8ab 100644
--- a/internal/release/roothistory_status.go
+++ b/internal/release/roothistory_status.go
@@ -14,9 +14,10 @@ const RootHistoryStatusDir = "/var/lib/carlos-release/history-status"
type HistoryHostStatus struct {
HistoryReconstructionHost
- Programs historyPrograms `json:"programs"`
- RefusalStatus historyRefusals `json:"refusal_status"`
- Attempt *HistoryAttempt `json:"attempt,omitempty"`
+ Programs historyPrograms `json:"programs"`
+ RefusalStatus historyRefusals `json:"refusal_status"`
+ Recoveries []HistoryRecovery `json:"recoveries,omitempty"`
+ Attempt *HistoryAttempt `json:"attempt,omitempty"`
}
type HistoryMigrationStatus struct {
@@ -114,6 +115,15 @@ func (h *RootHistory) MigrationStatus(b HistoryBinding, resolve func(HistoryBind
hosts = append(hosts, record.Binding.Host)
}
}
+ outcomes, err := h.recoveryOutcomes()
+ if err != nil {
+ return out, err
+ }
+ for _, r := range outcomes {
+ if r.Binding.Account == b.Account && r.Binding.App == b.App {
+ hosts = append(hosts, r.Binding.Host)
+ }
+ }
slices.Sort(hosts)
hosts = slices.Compact(hosts)
if len(hosts) > historyLimit {
@@ -132,7 +142,13 @@ func (h *RootHistory) MigrationStatus(b HistoryBinding, resolve func(HistoryBind
if err != nil && !errors.Is(err, os.ErrNotExist) {
return out, err
}
- out.Hosts = append(out.Hosts, HistoryHostStatus{HistoryReconstructionHost: HistoryReconstructionHost{Host: host, Source: source, Revision: record.Revision, Refusals: record.Refusals.Values}, Programs: record.Programs, RefusalStatus: record.Refusals, Attempt: record.Attempt})
+ var recoveries []HistoryRecovery
+ for _, r := range outcomes {
+ if r.Binding == binding {
+ recoveries = append(recoveries, r)
+ }
+ }
+ out.Hosts = append(out.Hosts, HistoryHostStatus{Recoveries: recoveries, HistoryReconstructionHost: HistoryReconstructionHost{Host: host, Source: source, Revision: record.Revision, Refusals: record.Refusals.Values}, Programs: record.Programs, RefusalStatus: record.Refusals, Attempt: record.Attempt})
}
for _, host := range out.Hosts {
source, err := resolve(HistoryBinding{Host: host.Host, Account: b.Account, App: b.App})