amadan

carlos / platform Public

claim Plan 14 Part 1 fix round 2: named-unit authority is the signed release, root current/previous pointers and unit identity (health history never blocks restart, start or rollback); stale-intent abort and quarantine; prune-safe commit; bounded legacy lock; barrier abandon; renew lost-reply settle; activation hard-refused until Part 3

Co-Authored-By: Codex <noreply@openai.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
codex pushed by claude 6291ebaf368b72b57dc133e5364b20af97f2e92c parent 127a159
48 files changed, +1471 −146
  • internal/host/agent.go +2 −2
  • internal/host/claimretryrecovery_test.go +4 −2
  • internal/host/claimstop_run.go +8 −1
  • internal/host/plan14_backend_source_test.go +8 −1
  • internal/host/plan14_canonical_restart_test.go +3 −3
  • internal/host/plan14_fix2_index_test.go +147 −0
  • internal/host/plan14_fix2_inventory_test.go +96 −0
  • internal/host/plan14_fix2_live_linux_test.go +180 −0
  • internal/host/plan14_fix2_recovery_test.go +43 −0
  • internal/host/proofbinding.go +53 −37
  • internal/host/proofnamed.go +31 −1
  • internal/host/proofnamed_linux_test.go +2 −2
  • internal/host/proofobserve.go +2 −8
  • internal/host/proofroot.go +3 −5
  • internal/host/rootexecution_runtime.go +7 −1
  • internal/host/rootfenceview_test.go +1 −1
  • internal/host/rootinventory.go +44 −6
  • internal/host/rootrecovery_rebind.go +4 −0
  • internal/host/rootrestart.go +104 −10
  • internal/host/rootrollback.go +6 −4
  • internal/host/rootrollback_process_test.go +9 −5
  • internal/host/rootrollback_resume.go +21 −2
  • internal/host/rootrollback_test.go +2 −2
  • internal/host/rootroute.go +4 −0
  • internal/hostbox/coldhandover.go +3 −0
  • internal/hostbox/inventory_expected.go +10 −0
  • internal/hostexec/plan14_fix2_linux_test.go +105 −0
  • internal/hostexec/rootbroker.go +121 −13
  • internal/hostguard/coldhandover.go +31 −0
  • internal/hostguard/guard.go +44 −3
  • internal/hostguard/plan14_fix2_unix_test.go +107 −0
  • internal/hostguard/plan14_root_fences_test.go +12 −5
  • internal/hostguard/record.go +1 −0
  • internal/hostguard/recoveryfence.go +1 −1
  • internal/hostguard/retirement.go +1 −1
  • internal/hostguard/root.go +33 −5
  • internal/hostguard/rootrebind.go +1 −0
  • internal/hostguard/rootview.go +14 −2
  • internal/provision/plan14_root_reads_test.go +6 −0
  • internal/provision/plan9_b8_census_test.go +7 −6
  • internal/provision/rootrecord.go +41 −11
  • internal/provision/testdata/plan14-root-reads-fix2.tsv +60 −0
  • internal/release/adopt.go +4 −1
  • internal/release/sweep.go +13 −0
  • internal/release/versions.go +27 −0
  • internal/rootparticipation/ledger.go +2 −4
  • internal/rootparticipation/plan14_fix2_test.go +21 −0
  • internal/rootparticipation/production.go +22 −1
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 45c0200ad..50343c9e7 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -4289,7 +4289,7 @@ func runAdoptUnderRootRelease(args []string, buildClaims ClaimFactory, ensureUni
// dies later still judged. Never fatal; one stat on a box with no floor.
restartChecks := productionRootRestartChecks(systemdUnits{}, release.OptDir())
if err := recoverRootRestarts(claimCtx, systemdUnits{}, release.OptDir(), os.Getenv, restartChecks, claims.Writes); err != nil {
- log.Fatalf("adopt: %v", err)
+ log.Printf("adopt: %v", err)
}
if err := rootJudgePrev(reg); err != nil {
log.Printf("adopt: moves protocol floor: %v", err)
@@ -4498,7 +4498,7 @@ func runAdoptUnderRootRelease(args []string, buildClaims ClaimFactory, ensureUni
if err := pr.around(ctx, func() error {
return release.Pass(ctx, reg, store, pub, release.OptDir(), restart, passOpts...)
}); err != nil {
- log.Fatalf("adopt: %v", err)
+ log.Printf("adopt: %v", err)
}
// A move's held stage (moves design §7.3, T7): the sandboxed agent
diff --git a/internal/host/claimretryrecovery_test.go b/internal/host/claimretryrecovery_test.go
index a7d1f96ff..cbd0f51e3 100644
--- a/internal/host/claimretryrecovery_test.go
+++ b/internal/host/claimretryrecovery_test.go
@@ -111,6 +111,7 @@ func (k *recoveryKernel) RecoverStopped(ctx context.Context, t hostguard.FenceTa
}
type recoveryFixture struct {
+ journalPath string
data genselect.Reader
bucket string
t *testing.T
@@ -159,12 +160,13 @@ func newRecoveryFixture(t *testing.T) *recoveryFixture {
if err = os.Mkdir(filepath.Join(data, "hostguard"), 0770); err != nil {
t.Fatal(err)
}
- journal, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launches.db"))
+ journalPath := filepath.Join(t.TempDir(), "launches.db")
+ journal, err := hostbox.OpenLaunchJournal(journalPath)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { journal.Close() })
- return &recoveryFixture{t: t, claims: st, evidence: release.NewDirStore(t.TempDir()), signer: ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)), guard: hostguard.Guard{DataDir: data, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: st}}, journal: journal, kernel: &recoveryKernel{journal: journal}, now: now, recordPrelaunch: true}
+ return &recoveryFixture{journalPath: journalPath, t: t, claims: st, evidence: release.NewDirStore(t.TempDir()), signer: ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)), guard: hostguard.Guard{DataDir: data, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: st}}, journal: journal, kernel: &recoveryKernel{journal: journal}, now: now, recordPrelaunch: true}
}
func (f *recoveryFixture) newID() string { f.seq++; return fmt.Sprintf("id-%d", f.seq) }
diff --git a/internal/host/claimstop_run.go b/internal/host/claimstop_run.go
index 3dd16f9ca..51ea563d9 100644
--- a/internal/host/claimstop_run.go
+++ b/internal/host/claimstop_run.go
@@ -13,11 +13,18 @@ import (
"github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootparticipation"
)
func RunInstanceControl(args []string) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
+ if hostwrite.Enabled(os.Getenv) {
+ if err := rootparticipation.ActivationReady(ctx); err != nil {
+ log.Print(&ClaimDependencyError{Contracts: []string{"census", "registered verifier", "flag must stay off until Part 3 (G)"}})
+ return
+ }
+ }
ctx, err := claimEntryContext(ctx, os.Getenv, openDeployStore)
if err != nil {
log.Fatal(err)
@@ -72,7 +79,7 @@ func RunInstanceControl(args []string) {
log.Fatal(ErrClaimStopUnavailable)
}
if err := recovery.RecoverPending(startup); err != nil {
- log.Fatal(err)
+ log.Print(err)
}
finishStartup()
execution.Start(ctx)
diff --git a/internal/host/plan14_backend_source_test.go b/internal/host/plan14_backend_source_test.go
index b3a147594..473015710 100644
--- a/internal/host/plan14_backend_source_test.go
+++ b/internal/host/plan14_backend_source_test.go
@@ -77,7 +77,7 @@ func TestPlan14OrdinaryDispatchSourceGap(t *testing.T) {
if err := reg.Upsert(candidate); err != nil {
t.Fatal(err)
}
- u := &plan14RestartUnits{}
+ u := &plan14OrdinaryUnits{}
checks := rootRestartChecks{targetFile: os.Open, dispatch: source.needsRestart}
if err := rootRestartRouteWith(reg, u, "", nil, nil, checks)(record.Host); err != nil {
t.Fatal(err)
@@ -103,3 +103,10 @@ func TestPlan14NamedRecordDispatch(t *testing.T) {
t.Fatalf("named record excluded: need=%v err=%v", need, err)
}
}
+
+type plan14OrdinaryUnits struct{ plan14RestartUnits }
+
+func (u *plan14OrdinaryUnits) TryRestart(unit string) error {
+ u.effects = append(u.effects, unit)
+ return nil
+}
diff --git a/internal/host/plan14_canonical_restart_test.go b/internal/host/plan14_canonical_restart_test.go
index 964f754d9..aa38a7ba7 100644
--- a/internal/host/plan14_canonical_restart_test.go
+++ b/internal/host/plan14_canonical_restart_test.go
@@ -167,13 +167,13 @@ func TestPlan14AdoptionCanonicalReadiness(t *testing.T) {
}
})
}
- t.Run("settled refusal propagates", func(t *testing.T) {
+ t.Run("health write cannot block restart", func(t *testing.T) {
e, _, units, c, pub := plan14AdoptionFixture(t)
sentinel := errors.New("settled witness")
restart := rootRestartRouteWith(e.root.reg, units, units.opt, func(string) string { return "5s" }, func(registry.Route) error { return sentinel }, c)
err := release.Pass(context.Background(), e.root.reg, e.store, pub, units.opt, restart, release.WithRestartFailurePropagation())
- if !errors.Is(err, sentinel) {
- t.Fatalf("lost settled error: %v", err)
+ if err != nil {
+ t.Fatalf("health write blocked restart: %v", err)
}
})
t.Run("positive adoption", func(t *testing.T) {
diff --git a/internal/host/plan14_fix2_index_test.go b/internal/host/plan14_fix2_index_test.go
new file mode 100644
index 000000000..2bb2f39a0
--- /dev/null
+++ b/internal/host/plan14_fix2_index_test.go
@@ -0,0 +1,147 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+type fix2IndexStore struct {
+ release.Store
+ lists int
+ accountFailure bool
+}
+
+func (s *fix2IndexStore) List(ctx context.Context, prefix string) ([]string, error) {
+ s.lists++
+ if prefix == "accounts/" && s.accountFailure {
+ return []string{"accounts/bap/account.json"}, nil
+ }
+ return s.Store.List(ctx, prefix)
+}
+func (s *fix2IndexStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ if key == "accounts/bap/account.json" && s.accountFailure {
+ s.accountFailure = false
+ return nil, "", os.ErrDeadlineExceeded
+ }
+ return s.Store.Get(ctx, key)
+}
+func TestPlan14Fix2RecordIndexRetriesAndRefreshesBodies(t *testing.T) {
+ e, rt, rec, source := plan14ProofFixture(t)
+ store := &fix2IndexStore{Store: e.store, accountFailure: true}
+ source.store = store
+ source.cacheIndex = true
+ if _, err := source.resolve(t.Context(), rt); err != nil {
+ t.Fatal("partial listing was refused", err)
+ }
+ lists := store.lists
+ rec.RecordSeq++
+ plan14PutRecord(t, e.store, rec)
+ first, err := source.resolve(t.Context(), rt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec.RecordSeq++
+ plan14PutRecord(t, e.store, rec)
+ second, err := source.resolve(t.Context(), rt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if first.RecordETag == second.RecordETag || store.lists != lists {
+ t.Fatal("cached bodies or repeated listing", first.RecordETag, second.RecordETag, lists, store.lists)
+ }
+ store.accountFailure = true
+ if _, _, err := provision.FindRecordByHost(t.Context(), store, rt.Host); !errors.Is(err, provision.ErrRecordIndexRetry) || errors.Is(err, os.ErrPermission) {
+ t.Fatal("partial listing not retryable", err)
+ }
+}
+
+func TestPlan14Fix2OrdinaryInstanceUnchanged(t *testing.T) {
+ e, rt, rec, source := plan14ProofFixture(t)
+ rec.Unit = ""
+ rec.Addr = ""
+ rec.Health = nil
+ plan14PutRecord(t, e.store, rec)
+ rt.Unit = "forged.service"
+ if err := e.root.reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ unit := activator.UnitName(rt.Host)
+ file := filepath.Join(t.TempDir(), "carlos-app@.service")
+ if err := os.WriteFile(file, []byte("root unit"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ u := &plan14OrdinaryUnits{}
+ u.props = unitProps{Name: unit, ExecPath: release.LiveLink(source.optDir, rt.Host), FragmentPath: file, ActiveState: "active"}
+ checks := rootRestartChecks{dispatch: source.needsRestart, targetFile: os.Open}
+ restart := rootRestartRouteWith(e.root.reg, u, source.optDir, nil, nil, checks)
+ if err := restart(rt.Host); err != nil {
+ t.Fatal(err)
+ }
+ if len(u.effects) != 1 || u.effects[0] != unit {
+ t.Fatal("ordinary unit not restarted", u.effects)
+ }
+ u.props.ActiveState = "inactive"
+ if err := restart(rt.Host); err != nil {
+ t.Fatal(err)
+ }
+ if len(u.effects) != 1 {
+ t.Fatal("operator-stopped ordinary unit restarted")
+ }
+}
+
+func TestPlan14Fix2OrdinaryAdoptionConfigAndRestartRequests(t *testing.T) {
+ e, rt, u, c, pub := plan14AdoptionFixture(t)
+ if err := e.store.Delete(t.Context(), provision.RecordKey(rt.Account, rt.App, rt.Host)); err != nil {
+ t.Fatal(err)
+ }
+ rt.Unit, rt.Addr, rt.Health = "", "", ""
+ rt.Backing, rt.Socket = registry.BackingUnit, provision.DefaultSocket(rt.Host)
+ if err := e.root.reg.Delete(rt.Host); err != nil {
+ t.Fatal(err)
+ }
+ if err := e.root.reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ u.props.Name = activator.UnitName(rt.Host)
+ source := &proofBindingSource{store: e.store, sys: u, optDir: u.opt, openFile: os.Open, unitDirs: []string{filepath.Dir(u.props.FragmentPath)}, namedPath: filepath.Join(t.TempDir(), "absent.json")}
+ c.dispatch = source.needsRestart
+ restart := rootRestartRouteWith(e.root.reg, u, u.opt, nil, nil, c)
+ if err := release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, restart, release.WithRestartFailurePropagation()); err != nil {
+ t.Fatal("ordinary adoption", err)
+ }
+ if len(u.effects) != 1 || u.exe != release.VersionPath(u.opt, rt.Account, "v2", rt.App) {
+ t.Fatal("ordinary adoption rolled back or missed restart", u.effects, u.exe)
+ }
+ if err := appconfig.SetEnv(t.Context(), e.store, rt.Account, rt.App, map[string]string{"LOG_LEVEL": "debug"}); err != nil {
+ t.Fatal(err)
+ }
+ envDir := t.TempDir()
+ for i := 0; i < 2; i++ {
+ if err := appconfig.Sync(t.Context(), e.root.reg, e.store, nil, envDir, restart, nil); err != nil {
+ t.Fatal("config sync", err)
+ }
+ }
+ if len(u.effects) != 2 {
+ t.Fatal("config restart missing or repeated", u.effects)
+ }
+ routes, err := e.root.reg.List()
+ if err != nil {
+ t.Fatal(err)
+ }
+ marks := filepath.Join(t.TempDir(), "restart-marks.json")
+ restartPass(t.Context(), routes, e.store, marks, restart)
+ askRestart(t, t.Context(), e.store, rt.Account, rt.App)
+ restartPass(t.Context(), routes, e.store, marks, restart)
+ restartPass(t.Context(), routes, e.store, marks, restart)
+ if len(u.effects) != 3 {
+ t.Fatal("restart request missing or repeated", u.effects)
+ }
+}
diff --git a/internal/host/plan14_fix2_inventory_test.go b/internal/host/plan14_fix2_inventory_test.go
new file mode 100644
index 000000000..95630b9c6
--- /dev/null
+++ b/internal/host/plan14_fix2_inventory_test.go
@@ -0,0 +1,96 @@
+package host
+
+import (
+ "database/sql"
+ "errors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/provision"
+ "testing"
+ "time"
+)
+
+func TestPlan14Fix2InventoryReceiptPruned(t *testing.T) {
+ f, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
+ state := &plan14RegistrationCrash{rollbackDisk: &rollbackDisk{dir: t.TempDir()}}
+ root.Stop.Coordinator.Guard.Root = hostguard.NewRootAuthority(state)
+ if _, err := root.commitRootInventory(t.Context(), q); err == nil {
+ t.Fatal("missing crash")
+ }
+ db, err := sql.Open("sqlite", f.journalPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ if _, err = db.Exec("DELETE FROM journal_requests"); err != nil {
+ t.Fatal(err)
+ }
+ if err := root.recoverRootInventory(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ view, err := root.Stop.Coordinator.Guard.Root.View(t.Context(), q.Target.Host, hostguard.Captured{Incarnation: q.Target.Incarnation, Stamp: q.Target.Stamp})
+ if err != nil || view.Installation.InventoryRevision != 1 {
+ t.Fatalf("root commit not recovered: %+v %v", view, err)
+ }
+}
+func TestPlan14Fix2OldInventoryRequestLeavesNoIntent(t *testing.T) {
+ _, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
+ q.IssuedAt = time.Now().Add(-time.Hour).UnixNano()
+ if _, err := root.commitRootInventory(t.Context(), q); err == nil {
+ t.Fatal("old request accepted")
+ }
+ pending, err := root.Stop.Coordinator.Guard.Root.PendingInstallations()
+ if err != nil || len(pending) != 0 {
+ t.Fatal("old request fenced host", pending, err)
+ }
+}
+func TestPlan14Fix2CommittedInventoryAfterETagChange(t *testing.T) {
+ f, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
+ state := &plan14RegistrationCrash{rollbackDisk: &rollbackDisk{dir: t.TempDir()}}
+ root.Stop.Coordinator.Guard.Root = hostguard.NewRootAuthority(state)
+ if _, err := root.commitRootInventory(t.Context(), q); err == nil {
+ t.Fatal("missing crash")
+ }
+ // Change the claim without changing its holder tuple.
+ obs := f.read()
+ obs.Claim.Op.Journal["fix2"] = "changed"
+ if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Journal["fix2"] = "changed"; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ // The committed request still completes after an ETag-only change.
+ if err := root.recoverRootInventory(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ _, err := root.Stop.Coordinator.Guard.Root.PendingInstallations()
+ if err != nil && !errors.Is(err, hostguard.ErrFenced) {
+ t.Fatal(err)
+ }
+}
+
+func TestPlan14Fix2SupersededInventoryIntentAbandoned(t *testing.T) {
+ f, root, q := plan14InstallRegistration(t, hostclaim.KindLanding, provision.LandChecking)
+ state := &rollbackDisk{dir: t.TempDir()}
+ root.Stop.Coordinator.Guard.Root = hostguard.NewRootAuthority(state)
+ state.afterWrite = func(string, []byte) { state.afterWrite = nil; _ = root.Reconcile.Registry.Close() }
+ if _, err := root.commitRootInventory(t.Context(), q); err == nil {
+ t.Fatal("closed registry accepted commit")
+ }
+ pending, err := root.Stop.Coordinator.Guard.Root.PendingInstallations()
+ if err != nil || len(pending) != 1 {
+ t.Fatal("missing pending intent", pending, err)
+ }
+ if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Journal["fix2"] = "changed"; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ if err := root.recoverRootInventory(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ pending, err = root.Stop.Coordinator.Guard.Root.PendingInstallations()
+ if err != nil || len(pending) != 0 {
+ t.Fatal("superseded intent survived", pending, err)
+ }
+ _, err = root.Stop.Coordinator.Guard.Root.View(t.Context(), q.Target.Host, hostguard.Captured{Incarnation: q.Target.Incarnation, Stamp: q.Target.Stamp})
+ if !errors.Is(err, hostguard.ErrFenced) {
+ t.Fatal("abandon removed quarantine", err)
+ }
+}
diff --git a/internal/host/plan14_fix2_live_linux_test.go b/internal/host/plan14_fix2_live_linux_test.go
new file mode 100644
index 000000000..f58a79c8c
--- /dev/null
+++ b/internal/host/plan14_fix2_live_linux_test.go
@@ -0,0 +1,180 @@
+package host
+
+import (
+ "crypto/ed25519"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rootparticipation"
+ "github.com/carlosframework/platform/internal/rootstate"
+)
+
+func TestPlan14Fix2LiveSafety(t *testing.T) {
+ if !plan14RootFixture(t) {
+ return
+ }
+ for _, scenario := range []string{"console update with empty history", "two consecutive updates", "incident recipe", "failed forward fix", "crash rollback", "successful restart crash before settle"} {
+ t.Run(scenario, func(t *testing.T) {
+ e, rt, u, c, _ := plan14AdoptionFixture(t)
+ if err := e.store.Delete(t.Context(), provision.RecordKey(rt.Account, rt.App, rt.Host)); err != nil {
+ t.Fatal(err)
+ }
+ rt.Account = ""
+ if err := e.root.reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ source := productionProofBindingSource(e.store, u, u.opt)
+ source.namedPath = filepath.Join(t.TempDir(), "named-units.json")
+ source.unitDirs = []string{filepath.Dir(u.props.FragmentPath)}
+ source.healthy = nil
+ makeProofAncestorsPublic(t, source.namedPath)
+ put := func(path string, raw []byte) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, raw, 0644); err != nil {
+ t.Fatal(err)
+ }
+ makeProofAncestorsPublic(t, path)
+ }
+ put(filepath.Join(u.opt, "system/guard/console-ready.json"), []byte(release.LiveLink(u.opt, rt.Host)))
+ drop := filepath.Join(source.unitDirs[0], "console.service.d/90-host-floor.conf")
+ put(drop, []byte(hostfloor.ConsoleGuardDropIn(release.LiveLink(u.opt, rt.Host))))
+ u.props.DropInPaths = []string{drop}
+ u.props.ExecPath = "/opt/carlos/system/guard/carlos"
+ put(u.props.ExecPath, []byte("guard"))
+ makeProofAncestorsPublic(t, u.exe)
+ makeProofAncestorsPublic(t, u.props.FragmentPath)
+ c.resolve, c.host, c.rollback, c.prepareRollback, c.dispatch, c.targetFile = source.resolve, source.forHost, source.rollback, source.prepareRollback, source.needsRestart, rootstate.OpenTrustedFile
+ pub, key, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := e.store.Delete(t.Context(), release.ChannelKey("bab", "console", rt.Channel)); err != nil {
+ t.Fatal(err)
+ }
+ promote := func(v string) {
+ t.Helper()
+ if err := release.Promote(t.Context(), e.store, key, "bab", "console", rt.Channel, v); err != nil {
+ t.Fatal(err)
+ }
+ }
+ promote("v1")
+ promote("v2")
+ if scenario == "crash rollback" {
+ u.crashNew = true
+ }
+ if scenario == "failed forward fix" {
+ u.props.ActiveState, u.props.SubState, u.props.MainPID = "failed", "failed", 0
+ }
+ if scenario == "incident recipe" {
+ if err := release.StageArtifact(u.opt, "", "v2", "console", []byte("new build")); err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PointLive(u.opt, rt.Host, "", "v2", "console"); err != nil {
+ t.Fatal(err)
+ }
+ if err := e.root.reg.SetVersions(rt.Host, "v2", "v1"); err != nil {
+ t.Fatal(err)
+ }
+ u.exe = release.VersionPath(u.opt, "", "v2", "console")
+ u.props.ActiveState, u.props.SubState, u.props.MainPID = "failed", "failed", 0
+ promote("v1")
+ }
+ restart := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, c)
+ rollback := rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c)
+ if scenario == "successful restart crash before settle" {
+ u.afterEffect = func() { panic("crash") }
+ func() {
+ defer func() {
+ if recover() != "crash" {
+ t.Fatal("missing crash")
+ }
+ }()
+ _ = release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, restart, release.WithRestartFailurePropagation())
+ }()
+ u.afterEffect = nil
+ before := len(u.effects)
+ if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil {
+ t.Fatal(err)
+ }
+ if len(u.effects) != before || u.exe != release.VersionPath(u.opt, "", "v2", "console") {
+ t.Fatal("recovery repeated restart", u.effects)
+ }
+ return
+ }
+ run := func() error {
+ return release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, restart, release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollback))
+ }
+ err = run()
+ if scenario == "crash rollback" {
+ if err == nil {
+ t.Fatal("failed update succeeded")
+ }
+ } else if err != nil {
+ t.Fatal(err)
+ }
+ want := "v2"
+ if scenario == "crash rollback" || scenario == "incident recipe" {
+ want = "v1"
+ }
+ if u.props.MainPID <= 0 || u.exe != release.VersionPath(u.opt, "", want, "console") {
+ t.Fatalf("not serving %s: %+v", want, u)
+ }
+ if scenario == "two consecutive updates" {
+ m := release.Manifest{App: "console", Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}}
+ if err := release.Ship(t.Context(), e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil {
+ t.Fatal(err)
+ }
+ promote("v3")
+ if err := run(); err != nil {
+ t.Fatal(err)
+ }
+ if u.exe != release.VersionPath(u.opt, "", "v3", "console") || len(u.effects) != 2 {
+ t.Fatal("second update", u.effects, u.exe)
+ }
+ }
+ })
+ }
+}
+
+func TestPlan14Fix2PlatformUpdateClaimsOffNoCensus(t *testing.T) {
+ if !plan14RootFixture(t) {
+ return
+ }
+ for _, flag := range []string{"", "1"} {
+ t.Run("flag="+flag, func(t *testing.T) {
+ t.Setenv("CARLOS_HOST_CLAIM", flag)
+ l, required, err := rootparticipation.OpenForUpdate()
+ if l.State != nil {
+ l.State.Close()
+ }
+ if err != nil || required {
+ t.Fatalf("unactivated census blocked update: %v %v", required, err)
+ }
+ if rootparticipation.ActivationReady(t.Context()) == nil {
+ t.Fatal("claims activated without census and verifier")
+ }
+ guard := hostfloor.Disk{Dir: t.TempDir()}
+ data := t.TempDir()
+ if err := os.Mkdir(filepath.Join(data, ".hostexec"), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := guard.Begin("update", data); err != nil {
+ t.Fatal("real update path", err)
+ }
+ if _, err := os.Stat(filepath.Join(guard.Dir, "update-begun.json")); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(rootparticipation.Namespace); !os.IsNotExist(err) {
+ t.Fatal("update created census", err)
+ }
+ })
+ }
+}
diff --git a/internal/host/plan14_fix2_recovery_test.go b/internal/host/plan14_fix2_recovery_test.go
new file mode 100644
index 000000000..daf49c929
--- /dev/null
+++ b/internal/host/plan14_fix2_recovery_test.go
@@ -0,0 +1,43 @@
+package host
+
+import (
+ "errors"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/release"
+ "testing"
+)
+
+func TestPlan14Fix2RecoveryContinuesPastBadAttempt(t *testing.T) {
+ e, _, u, c, pub := plan14AdoptionFixture(t)
+ u.crashNew = true
+ restart := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, c)
+ crashed := errors.New("crash")
+ func() {
+ defer func() {
+ if recover() != crashed {
+ t.Fatal("missing crash")
+ }
+ }()
+ _ = release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, func(host string) error { _ = restart(host); panic(crashed) }, release.WithRestartFailurePropagation())
+ }()
+ state, err := c.attempts()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := state.Write("0.attempt", []byte("malformed")); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err == nil {
+ t.Fatal("corrupt attempt not reported")
+ }
+ if u.props.MainPID <= 0 || u.exe != release.VersionPath(u.opt, "bab", "v1", "console") {
+ t.Fatal("bad attempt skipped other host recovery")
+ }
+ if err := state.Remove("0.attempt"); err != nil {
+ t.Fatal(err)
+ }
+ state.(*rollbackMemory).failWrite = true
+ if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil {
+ t.Fatal("settled attempt rewritten", err)
+ }
+}
diff --git a/internal/host/proofbinding.go b/internal/host/proofbinding.go
index d9adcdcfe..a30c805a7 100644
--- a/internal/host/proofbinding.go
+++ b/internal/host/proofbinding.go
@@ -5,10 +5,12 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
+ "github.com/carlosframework/platform/internal/provision"
"io"
"os"
"path/filepath"
"strings"
+ "sync"
"time"
"github.com/carlosframework/platform/internal/hostclaim"
@@ -45,17 +47,21 @@ type proofUnitFile struct {
type proofBindingResolver func(context.Context, registry.Route) (proofBinding, error)
type proofBindingSource struct {
- now func() time.Time
- namedPath string
- attempts func() (rootstate.Store, error)
- store release.Store
- claim func(context.Context, string) (hostclaim.Observed, error)
- box func(context.Context) (string, error)
- sys unitSystem
- optDir string
- openFile func(string) (*os.File, error)
- unitDirs []string
- healthy func(string, string, string, string) *release.HealthyBuild
+ indexMu sync.Mutex
+ index *provision.RecordIndex
+ cacheIndex bool
+ checkPointers func(string) error
+ now func() time.Time
+ namedPath string
+ attempts func() (rootstate.Store, error)
+ store release.Store
+ claim func(context.Context, string) (hostclaim.Observed, error)
+ box func(context.Context) (string, error)
+ sys unitSystem
+ optDir string
+ openFile func(string) (*os.File, error)
+ unitDirs []string
+ healthy func(string, string, string, string) *release.HealthyBuild
}
func proofRouteReady(rt registry.Route, timeout time.Duration) func() error {
@@ -83,7 +89,13 @@ func proofRouteReady(rt registry.Route, timeout time.Duration) func() error {
}
func productionProofBindingSource(store release.Store, sys unitSystem, optDir string) *proofBindingSource {
- source := &proofBindingSource{namedPath: rootNamedUnits, attempts: func() (rootstate.Store, error) { return rootstate.Open(rootRestartState, false, false) }, store: store, sys: sys, optDir: optDir, openFile: rootstate.OpenTrustedFile,
+ source := &proofBindingSource{cacheIndex: true, checkPointers: func(dir string) error {
+ f, err := rootstate.Open(dir, false, true)
+ if err != nil {
+ return err
+ }
+ return f.Close()
+ }, namedPath: rootNamedUnits, attempts: func() (rootstate.Store, error) { return rootstate.Open(rootRestartState, false, false) }, store: store, sys: sys, optDir: optDir, openFile: rootstate.OpenTrustedFile,
unitDirs: []string{"/etc/systemd/system", "/run/systemd/system", "/usr/lib/systemd/system"},
healthy: func(acct, app, version, digest string) *release.HealthyBuild {
for _, b := range release.ReadHealthy(release.RootHealthyDir, acct, app) {
@@ -215,14 +227,38 @@ func (s *proofBindingSource) binding(ctx context.Context, host string, rollback
exe, err = release.VersionPath(s.optDir, rec.Account, rollback.Version, rec.App), nil
}
} else {
- if p.MainPID <= 0 || p.InvocationID == "" {
- return proofBinding{}, os.ErrPermission
+ if p.ActiveState == "failed" && p.MainPID == 0 {
+ exe, err = filepath.EvalSymlinks(release.LiveLink(s.optDir, host))
+ } else {
+ if p.MainPID <= 0 || p.InvocationID == "" {
+ return proofBinding{}, os.ErrPermission
+ }
+ exe, err = s.sys.Exe(p.MainPID)
}
- exe, err = s.sys.Exe(p.MainPID)
}
if err != nil {
return proofBinding{}, err
}
+ if s.checkPointers != nil {
+ if err := s.checkPointers(filepath.Join(s.optDir, "live")); err != nil {
+ return proofBinding{}, err
+ }
+ }
+ current, currentErr := filepath.EvalSymlinks(release.LiveLink(s.optDir, host))
+ if currentErr != nil {
+ return proofBinding{}, currentErr
+ }
+ if exe != current {
+ if s.checkPointers != nil {
+ if err := s.checkPointers(filepath.Join(s.optDir, "previous")); err != nil {
+ return proofBinding{}, err
+ }
+ }
+ previous, err := filepath.EvalSymlinks(filepath.Join(s.optDir, "previous", host))
+ if err != nil || previous != exe {
+ return proofBinding{}, os.ErrPermission
+ }
+ }
rel, err := filepath.Rel(filepath.Join(s.optDir, "versions"), exe)
if err != nil {
return proofBinding{}, err
@@ -246,31 +282,11 @@ func (s *proofBindingSource) binding(ctx context.Context, host string, rollback
history = s.healthy(rec.Account, rec.App, version, digest)
}
historical := history != nil
- if rollback != nil && (!historical || !rollback.Historical || rollback.Version != version || rollback.Digest != digest) {
+ if rollback != nil && (rollback.Version != version || rollback.Digest != digest) {
return proofBinding{}, os.ErrPermission
}
- installed := evidence.local && rt.Unit == "console.service" && execPath == "/opt/carlos/system/guard/carlos" && s.installedConsoleVersion(version, digest)
- if evidence.local && !historical && !installed && !s.capturedVersion(host, source, exe, digest) {
- return proofBinding{}, os.ErrPermission
- }
- if !evidence.local && rec.Pin != version && !historical {
- if rec.Pin != "" {
- return proofBinding{}, os.ErrPermission
- }
- pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
- if err != nil {
- return proofBinding{}, err
- }
- pointer, err := release.Channel(ctx, s.store, pub, rec.Account, rec.App, rec.Channel)
- if err != nil {
- return proofBinding{}, err
- }
- if pointer.Version != version {
- return proofBinding{}, os.ErrPermission
- }
- }
start := ""
- if rollback == nil {
+ if rollback == nil && p.MainPID > 0 {
start, err = s.sys.StartTime(p.MainPID)
if err != nil || start == "" {
return proofBinding{}, os.ErrPermission
diff --git a/internal/host/proofnamed.go b/internal/host/proofnamed.go
index 448458a88..1678bb894 100644
--- a/internal/host/proofnamed.go
+++ b/internal/host/proofnamed.go
@@ -64,7 +64,7 @@ func (s *proofBindingSource) trustedBytes(path string) ([]byte, proofUnitFile, e
}
func (s *proofBindingSource) record(ctx context.Context, host string) (proofRecord, error) {
- rec, etag, err := provision.FindRecordByHost(ctx, s.store, host)
+ rec, etag, err := s.findRecord(ctx, host)
if err == nil {
return proofRecord{record: rec, key: provision.RecordKey(rec.Account, rec.App, host), revision: etag}, nil
}
@@ -199,3 +199,33 @@ func (s *proofBindingSource) installedConsoleVersion(version, digest string) boo
}
return false
}
+
+func (s *proofBindingSource) findRecord(ctx context.Context, host string) (*provision.Record, string, error) {
+ if !s.cacheIndex {
+ return provision.FindRecordByHost(ctx, s.store, host)
+ }
+ s.indexMu.Lock()
+ defer s.indexMu.Unlock()
+ if s.index == nil {
+ var index *provision.RecordIndex
+ var err error
+ for attempt := 0; attempt < 3; attempt++ {
+ index, err = provision.IndexRecords(ctx, s.store)
+ if !errors.Is(err, provision.ErrRecordIndexRetry) {
+ break
+ }
+ timer := time.NewTimer(time.Duration(attempt+1) * 25 * time.Millisecond)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return nil, "", ctx.Err()
+ case <-timer.C:
+ }
+ }
+ if err != nil {
+ return nil, "", err
+ }
+ s.index = index
+ }
+ return s.index.Find(ctx, s.store, host)
+}
diff --git a/internal/host/proofnamed_linux_test.go b/internal/host/proofnamed_linux_test.go
index 7b3bd033f..153115735 100644
--- a/internal/host/proofnamed_linux_test.go
+++ b/internal/host/proofnamed_linux_test.go
@@ -116,8 +116,8 @@ func TestPlan14NamedUnitsWithoutDeploymentRecord(t *testing.T) {
}
if scenario == "console bootstrap invalid capture" {
put(filepath.Join(u.opt, "system", "guard", "identities", "v1.0.0.json"), []byte(`{"signature":"forged"}`))
- if _, err := s.resolve(t.Context(), rt); err == nil {
- t.Fatal("invalid root installation capture authorized bootstrap")
+ if _, err := s.resolve(t.Context(), rt); err != nil {
+ t.Fatal("root live pointer must remain authoritative without installation history", err)
}
if len(u.effects) != 0 {
t.Fatal("invalid capture restarted console")
diff --git a/internal/host/proofobserve.go b/internal/host/proofobserve.go
index cd125491b..011f08a76 100644
--- a/internal/host/proofobserve.go
+++ b/internal/host/proofobserve.go
@@ -46,14 +46,8 @@ type proofObserver struct {
func productionProofObserver(optDir string) proofObserver {
return proofObserver{sys: systemdUnits{}, procExe: procExePath, optDir: optDir, now: time.Now,
- resolve: func(ctx context.Context, rt registry.Route) (proofBinding, error) {
- store, err := openDeployStore(ctx)
- if err != nil {
- return proofBinding{}, err
- }
- return productionProofBindingSource(store, systemdUnits{}, optDir).resolve(ctx, rt)
- },
- ready: func(rt registry.Route) func() error { return proofRouteReady(rt, 2*time.Second) }}
+ resolve: productionRootRestartChecks(systemdUnits{}, optDir).resolve,
+ ready: func(rt registry.Route) func() error { return proofRouteReady(rt, 2*time.Second) }}
}
func procExePath(pid int) string { return fmt.Sprintf("/proc/%d/exe", pid) }
diff --git a/internal/host/proofroot.go b/internal/host/proofroot.go
index c29dc25d6..50cb2a6a0 100644
--- a/internal/host/proofroot.go
+++ b/internal/host/proofroot.go
@@ -82,14 +82,12 @@ func (r proofRoot) run(ctx context.Context) {
}
// around is RunAdopt's order: record the live builds, run the pass, then
-// mint on what the pass left live. A pass error is returned without minting.
+// mint on what the pass left live, including routes unaffected by a pass error.
func (r proofRoot) around(ctx context.Context, pass func() error) error {
obs := r.recordLive(ctx)
- if err := pass(); err != nil {
- return err
- }
+ err := pass()
r.mintAndLog(ctx, obs)
- return nil
+ return err
}
// recordLive reads the policy, observes every named-unit route's live build,
diff --git a/internal/host/rootexecution_runtime.go b/internal/host/rootexecution_runtime.go
index e48d79399..bc5958ebf 100644
--- a/internal/host/rootexecution_runtime.go
+++ b/internal/host/rootexecution_runtime.go
@@ -7,7 +7,9 @@ import (
"errors"
"sync"
+ "github.com/carlosframework/platform/internal/hostbox"
execlease "github.com/carlosframework/platform/internal/hostexec"
+ "github.com/carlosframework/platform/internal/hostguard"
"github.com/carlosframework/platform/internal/rootparticipation"
"github.com/carlosframework/platform/internal/rootstate"
)
@@ -40,7 +42,11 @@ func newClaimExecutionRuntime(h ClaimRootHTTP, state rootstate.Store) (*claimExe
broker, err := execlease.NewRootBroker(execlease.RootBrokerConfig{
State: state, Remote: h.Stop.Coordinator.Evidence, Clock: execlease.SystemClock{}, BootID: boot, Identity: hex.EncodeToString(id[:]),
Authority: func(ctx context.Context, q execlease.ExecutionRequest, peer execlease.ControllerProcess) (execlease.ExecutionAuthority, error) {
- return runtime.root.executionAuthorityWithMigration(ctx, state, q, peer)
+ a, err := runtime.root.executionAuthorityWithMigration(ctx, state, q, peer)
+ if errors.Is(err, hostbox.ErrChanged) || errors.Is(err, hostbox.ErrTypedInventory) || errors.Is(err, hostguard.ErrFenced) || errors.Is(err, hostguard.ErrStampMismatch) || errors.Is(err, hostguard.ErrRetiredStamp) {
+ err = errors.Join(execlease.ErrInactive, err)
+ }
+ return a, err
},
Alive: execlease.ControllerAlive,
Cleanup: func(ctx context.Context, r execlease.ExecutionRecord) error {
diff --git a/internal/host/rootfenceview_test.go b/internal/host/rootfenceview_test.go
index 62f884ffd..b2d132810 100644
--- a/internal/host/rootfenceview_test.go
+++ b/internal/host/rootfenceview_test.go
@@ -100,7 +100,7 @@ func TestPlan14FenceViewHandlerAcquisitionSet(t *testing.T) {
if response.Code != http.StatusOK {
t.Fatalf("view handler: %d %s", response.Code, response.Body)
}
- if !reflect.DeepEqual(store.calls, []string{"read shop.example.json"}) {
+ if !reflect.DeepEqual(store.calls, []string{"lock shop.example.lock", "read shop.example.json"}) {
t.Fatalf("handler acquired resources beyond its root snapshot: %v", store.calls)
}
var out hostbox.JournalResponse
diff --git a/internal/host/rootinventory.go b/internal/host/rootinventory.go
index f793fce71..dc06ae586 100644
--- a/internal/host/rootinventory.go
+++ b/internal/host/rootinventory.go
@@ -7,10 +7,12 @@ import (
"encoding/json"
"errors"
"fmt"
+ "log"
"os"
"path/filepath"
"reflect"
"strconv"
+ "time"
"github.com/carlosframework/platform/internal/hostbox"
"github.com/carlosframework/platform/internal/hostclaim"
@@ -25,6 +27,16 @@ func (h ClaimRootHTTP) commitRootInventory(ctx context.Context, q hostbox.Journa
}
func (h ClaimRootHTTP) commitRootInventoryPending(ctx context.Context, q hostbox.JournalRequest, pending bool) (hostbox.JournalResponse, error) {
var out hostbox.JournalResponse
+ if !pending && (q.V == 2 || q.V == 3) {
+ now := time.Now()
+ if h.Journals != nil && h.Journals.Now != nil {
+ now = h.Journals.Now()
+ }
+ age := now.Sub(time.Unix(0, q.IssuedAt))
+ if age > hostbox.JournalRequestWindow || age < -hostbox.JournalRequestWindow {
+ return out, hostbox.ErrJournalUncertain
+ }
+ }
guard := h.Stop.Coordinator.Guard
if guard.Root == nil || h.Journals == nil || h.InventoryActor == nil || h.Reconcile.Registry == nil || h.Reconcile.Census == nil {
return out, hostbox.ErrUnsupported
@@ -49,8 +61,16 @@ func (h ClaimRootHTTP) commitRootInventoryPending(ctx context.Context, q hostbox
if observed.Absent || observed.Claim == nil || observed.Claim.Holder == nil {
return out, hostbox.ErrChanged
}
+ applied := false
+ if pending {
+ saved, e := h.Journals.Inventory.SnapshotRequest(ctx, t)
+ if e != nil {
+ return out, e
+ }
+ applied = saved.Revision == q.Revision+1 && saved.LastRequest == q.ID
+ }
holder := observed.Claim.Holder
- if observed.ETag != t.ClaimETag || holder.Incarnation != t.Incarnation || holder.Stamp != t.Stamp || holder.Session != t.Session || holder.Key != q.RecordKey || observed.Claim.Op.ID != t.Op || observed.Claim.Op.Phase != t.Phase {
+ if (!applied && observed.ETag != t.ClaimETag) || holder.Incarnation != t.Incarnation || holder.Stamp != t.Stamp || holder.Session != t.Session || holder.Key != q.RecordKey || observed.Claim.Op.ID != t.Op || observed.Claim.Op.Phase != t.Phase {
return out, hostbox.ErrChanged
}
firstLanding := q.Revision == 0 && observed.Claim.Op.Kind == hostclaim.KindLanding
@@ -86,7 +106,7 @@ func (h ClaimRootHTTP) commitRootInventoryPending(ctx context.Context, q hostbox
if err != nil {
return err
}
- if fresh.ETag != t.ClaimETag {
+ if !applied && fresh.ETag != t.ClaimETag {
return hostbox.ErrChanged
}
var etag string
@@ -169,7 +189,8 @@ func (h ClaimRootHTTP) commitRootInventoryPending(ctx context.Context, q hostbox
return hostbox.ErrTypedInventory
}
- prior, revision, err := h.Journals.Inventory.Snapshot(ctx, in.Host, hostguard.Captured{Incarnation: in.Incarnation, Stamp: in.InstallationStamp})
+ snapshot, err := h.Journals.Inventory.SnapshotRequest(ctx, hostguard.FenceTarget{Host: in.Host, Incarnation: in.Incarnation, Stamp: in.InstallationStamp})
+ prior, revision := snapshot.Inventory, snapshot.Revision
if err != nil {
return err
}
@@ -185,7 +206,7 @@ func (h ClaimRootHTTP) commitRootInventoryPending(ctx context.Context, q hostbox
replayed = true
return nil
}
- if revision == q.Revision+1 && reflect.DeepEqual(prior, in) && priorRoot.InventoryRevision == revision {
+ if revision == q.Revision+1 && reflect.DeepEqual(prior, in) && (priorRoot.InventoryRevision == revision || pending && snapshot.LastRequest == q.ID) {
out = hostbox.JournalResponse{V: 1, ID: q.ID, Revision: revision}
replayed = true
return nil
@@ -310,7 +331,14 @@ func (h ClaimRootHTTP) recoverRootInventory(ctx context.Context) error {
}
if recovery.Recovery != nil {
if err = h.rebindRecoveryInventory(ctx, *recovery.Recovery); err != nil {
- return err
+ n := recovery.Recovery.Next
+ if errors.Is(err, hostbox.ErrChanged) {
+ err = errors.Join(err, h.Stop.Coordinator.Guard.AbandonRootInventory(ctx, n.Host, raw, func(c *hostclaim.Claim) bool {
+ fresh, e := hostclaim.Read(ctx, h.Stop.Coordinator.Evidence, n.Host)
+ return e == nil && fresh.Claim != nil && reflect.DeepEqual(c, fresh.Claim) && fresh.ETag != n.ETag
+ }))
+ }
+ log.Printf("instance-control: %s: %v", n.Host, err)
}
continue
}
@@ -319,7 +347,17 @@ func (h ClaimRootHTTP) recoverRootInventory(ctx context.Context) error {
return err
}
if _, err = h.commitRootInventoryPending(ctx, q, true); err != nil {
- return err
+ if errors.Is(err, hostbox.ErrChanged) || errors.Is(err, hostbox.ErrJournalUncertain) {
+ err = errors.Join(err, h.Stop.Coordinator.Guard.AbandonRootInventory(ctx, q.Target.Host, raw, func(c *hostclaim.Claim) bool {
+ fresh, readErr := hostclaim.Read(ctx, h.Stop.Coordinator.Evidence, q.Target.Host)
+ if readErr != nil || fresh.Claim == nil || !reflect.DeepEqual(c, fresh.Claim) {
+ return false
+ }
+ _, recordETag, recordErr := h.Stop.Coordinator.Evidence.Get(ctx, q.RecordKey)
+ return recordErr == nil && recordETag != q.RecordETag || fresh.ETag != q.Target.ClaimETag || c.Holder == nil || c.Holder.Incarnation != q.Target.Incarnation || c.Holder.Stamp != q.Target.Stamp || c.Holder.Session != q.Target.Session || c.Op.ID != q.Target.Op || c.Op.Phase != q.Target.Phase || errors.Is(err, hostbox.ErrJournalUncertain)
+ }))
+ }
+ log.Printf("instance-control: %s: %v", q.Target.Host, err)
}
}
return nil
diff --git a/internal/host/rootrecovery_rebind.go b/internal/host/rootrecovery_rebind.go
index dfa562e63..4d400b6a9 100644
--- a/internal/host/rootrecovery_rebind.go
+++ b/internal/host/rootrecovery_rebind.go
@@ -11,6 +11,7 @@ import (
"net"
"net/http"
"reflect"
+ "slices"
"strconv"
"time"
@@ -85,6 +86,9 @@ func (h ClaimRootHTTP) rebindRecoveryInventory(ctx context.Context, q recoveryRe
next := hostguard.RootInstallation{Host: n.Host, Incarnation: n.Incarnation, Stamp: n.Stamp, Account: holder.Account, App: holder.App, Record: q.RecordKey, Box: n.Box, Enrollment: n.Enrollment, Migration: 1, InventoryRevision: 1}
var rebound hostbox.StopInventory
check := func(c *hostclaim.Claim) error {
+ if slices.Contains(c.Lost, n.Box) {
+ return hostbox.ErrChanged
+ }
fresh, err := hostclaim.Read(ctx, h.Stop.Coordinator.Evidence, n.Host)
if err != nil {
return err
diff --git a/internal/host/rootrestart.go b/internal/host/rootrestart.go
index a51f57722..e5cac0a64 100644
--- a/internal/host/rootrestart.go
+++ b/internal/host/rootrestart.go
@@ -4,14 +4,17 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
+ "encoding/json"
"errors"
"io"
+ "log"
"os"
"path/filepath"
"strings"
"sync"
"time"
+ "github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/hostwrite"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
@@ -19,6 +22,7 @@ import (
)
type rootRestartChecks struct {
+ targetDigest string
targetFile func(string) (*os.File, error)
dispatch func(context.Context, registry.Route) (bool, error)
resolve proofBindingResolver
@@ -36,9 +40,20 @@ type rootRestartChecks struct {
}
func productionRootRestartChecks(sys unitSystem, optDir string) rootRestartChecks {
+ var sourceMu sync.Mutex
+ var cached *proofBindingSource
source := func(ctx context.Context) (*proofBindingSource, error) {
+ sourceMu.Lock()
+ defer sourceMu.Unlock()
+ if cached != nil {
+ return cached, nil
+ }
s, err := openDeployStore(ctx)
- return productionProofBindingSource(s, sys, optDir), err
+ if err != nil {
+ return nil, err
+ }
+ cached = productionProofBindingSource(s, sys, optDir)
+ return cached, nil
}
return rootRestartChecks{targetFile: rootstate.OpenTrustedFile, started: &sync.Map{},
dispatch: func(ctx context.Context, rt registry.Route) (bool, error) {
@@ -117,7 +132,7 @@ func checkedUnitRestart(sys unitSystem, optDir string, getenv func(string) strin
if before.ActiveState == "inactive" {
return errUnitStopped
}
- if before.ActiveState != "active" || before.MainPID <= 0 || before.MainPID != binding.PID || before.InvocationID != binding.Invocation {
+ if before.ActiveState != "failed" && (before.ActiveState != "active" || before.MainPID <= 0 || before.MainPID != binding.PID || before.InvocationID != binding.Invocation) {
return os.ErrPermission
}
target, err := filepath.EvalSymlinks(release.LiveLink(optDir, candidate.Host))
@@ -143,7 +158,42 @@ func checkedUnitRestart(sys unitSystem, optDir string, getenv func(string) strin
return err
}
targetDigest := hex.EncodeToString(hash.Sum(nil))
- attempt, err := captureRollback(s, binding, c.now(), unitSettleTimeout(getenv), target, c.identity, targetDigest)
+ c.targetDigest = targetDigest
+ captured := binding
+ if before.ActiveState == "failed" {
+ previous, e := filepath.EvalSymlinks(filepath.Join(optDir, "previous", candidate.Host))
+ if e == nil && previous != target {
+ rel, e := filepath.Rel(filepath.Join(optDir, "versions"), previous)
+ if e != nil {
+ return e
+ }
+ version := strings.Split(rel, string(filepath.Separator))[0]
+ if !release.ValidVersion(version) || previous != release.VersionPath(optDir, binding.Route.Account, version, binding.Route.App) {
+ return os.ErrPermission
+ }
+ f, e := c.targetFile(previous)
+ if e != nil {
+ return e
+ }
+ h := sha256.New()
+ _, e = io.Copy(h, f)
+ f.Close()
+ if e != nil {
+ return e
+ }
+ captured.Version, captured.Digest = version, hex.EncodeToString(h.Sum(nil))
+ captured, e = c.prepareRollback(context.Background(), captured)
+ if e != nil {
+ return e
+ }
+ } else if e != nil && !errors.Is(e, os.ErrNotExist) {
+ return e
+ }
+ }
+ if err := writeRollbackState(s, rollbackHostKey(candidate.Host)+".previous", captured); err != nil {
+ return err
+ }
+ attempt, err := captureRollback(s, captured, c.now(), unitSettleTimeout(getenv), target, c.identity, targetDigest)
if err != nil {
return err
}
@@ -208,17 +258,25 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string,
return err
}
} else {
- if before.ActiveState != "active" || before.MainPID != binding.PID || before.InvocationID != binding.Invocation {
+ if before.ActiveState != "failed" && (before.ActiveState != "active" || before.MainPID != binding.PID || before.InvocationID != binding.Invocation) {
return os.ErrPermission
}
- start, err := sys.StartTime(before.MainPID)
- if err != nil || start != binding.Start {
- return os.ErrPermission
+ if before.MainPID > 0 {
+ start, err := sys.StartTime(before.MainPID)
+ if err != nil || start != binding.Start {
+ return os.ErrPermission
+ }
}
current, err := c.host(context.Background(), rt.Host)
if err != nil || current.Source != binding.Source || current.Version != binding.Version || current.Digest != binding.Digest || current.PID != binding.PID || current.Invocation != binding.Invocation {
return os.ErrPermission
}
+ if before.ActiveState == "failed" {
+ if err := sys.ResetFailed(rt.Unit); err != nil {
+ return err
+ }
+ return sys.Restart(rt.Unit)
+ }
if err := sys.TryRestart(rt.Unit); err != nil {
return err
}
@@ -233,6 +291,9 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string,
if err != nil || first.Source != binding.Source || want != release.VersionPath(optDir, first.Route.Account, first.Version, first.Route.App) {
return os.ErrPermission
}
+ if c.targetDigest != "" && first.Digest != c.targetDigest {
+ return os.ErrPermission
+ }
if rollback && (first.Version != binding.Version || first.Digest != binding.Digest) {
return os.ErrPermission
}
@@ -262,7 +323,7 @@ func restartBoundUnit(sys unitSystem, optDir string, getenv func(string) string,
}
if settled != nil {
if err := effect(func() error { return settled(binding.Route) }); err != nil {
- return errors.Join(&release.UnitNotUpError{Unit: rt.Unit, Reason: err.Error()}, err)
+ log.Print(err)
}
}
return nil
@@ -293,8 +354,24 @@ func rollbackUnitRestart(_ *registry.Registry, sys unitSystem, optDir string, ge
defer func() { result = errors.Join(result, settleRollbackAttempt(s, attempt)) }()
}
if err != nil {
- return err
+ if !errors.Is(err, os.ErrNotExist) && !attempt.Settled && (attempt.ID == "" || !c.now().After(attempt.Deadline.Add(rollbackMargin))) {
+ return err
+ }
+ raw, readErr := s.Read(rollbackHostKey(host) + ".previous")
+ if errors.Is(readErr, os.ErrNotExist) && c.host != nil {
+ binding, readErr = c.host(context.Background(), host)
+ } else if readErr == nil && json.Unmarshal(raw, &binding) != nil {
+ readErr = os.ErrPermission
+ }
+ if readErr != nil || binding.Route.Host != host {
+ return errors.Join(err, readErr)
+ }
+ before, showErr := sys.Show(binding.Route.Unit)
+ if showErr != nil || before.ActiveState != "failed" || before.MainPID != 0 {
+ return errors.Join(err, showErr)
+ }
}
+ c.targetDigest = binding.Digest
binding, err = c.rollback(context.Background(), binding)
if err != nil {
return err
@@ -356,9 +433,26 @@ func rootRestartRouteWith(reg *registry.Registry, sys unitSystem, optDir string,
return err
}
if !need {
- return nil
+ return restartOrdinaryUnit(sys, optDir, host, checks.targetFile)
}
}
return restart(*rt)
}
}
+
+func restartOrdinaryUnit(sys unitSystem, optDir, host string, open func(string) (*os.File, error)) error {
+ unit := activator.UnitName(host)
+ p, err := sys.Show(unit)
+ if err != nil || p.Name != unit || p.ExecPath != release.LiveLink(optDir, host) || p.ActiveState != "active" || open == nil {
+ return nil
+ }
+ file, err := open(p.FragmentPath)
+ if err != nil {
+ return err
+ }
+ file.Close()
+ if err := sys.TryRestart(unit); err != nil {
+ log.Print(err)
+ }
+ return nil
+}
diff --git a/internal/host/rootrollback.go b/internal/host/rootrollback.go
index cf4e7c71e..5e53dc68a 100644
--- a/internal/host/rootrollback.go
+++ b/internal/host/rootrollback.go
@@ -67,15 +67,17 @@ func readRollbackAttempt(s rootstate.Store, host string) (rollbackAttempt, error
}
func settleRollbackAttempt(s rootstate.Store, a rollbackAttempt) error {
- a.Settled = true
- if err := writeRollbackState(s, rollbackJournalKey(a.Host), a); err != nil {
- return err
+ if !a.Settled {
+ a.Settled = true
+ if err := writeRollbackState(s, rollbackJournalKey(a.Host), a); err != nil {
+ return err
+ }
}
return removeRollbackCapabilities(s, a.Host)
}
func captureRollback(s rootstate.Store, b proofBinding, now time.Time, timeout time.Duration, target string, identity hostwrite.Identity, targetDigest ...string) (rollbackAttempt, error) {
- if b.Source == "" || !b.Historical || b.Version == "" || b.Digest == "" || b.History == nil || b.History.SHA256 != b.Digest {
+ if b.Source == "" || b.Version == "" || b.Digest == "" {
return rollbackAttempt{}, os.ErrPermission
}
if prior, err := readRollbackAttempt(s, b.Route.Host); err == nil {
diff --git a/internal/host/rootrollback_process_test.go b/internal/host/rootrollback_process_test.go
index 8a026c9ee..c7477b78c 100644
--- a/internal/host/rootrollback_process_test.go
+++ b/internal/host/rootrollback_process_test.go
@@ -130,9 +130,9 @@ func TestPlan14RollbackProcessCrash(t *testing.T) {
crash()
}
}
- u.crashNew = true
+ u.crashNew = boundary != "forward running"
u.afterEffect = func() {
- if boundary == "forward restart" && len(u.effects) == 1 {
+ if (boundary == "forward restart" || boundary == "forward running") && len(u.effects) == 1 {
crash()
}
if boundary == "rollback running" && len(u.effects) == 3 {
@@ -143,7 +143,7 @@ func TestPlan14RollbackProcessCrash(t *testing.T) {
_ = release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, forward, release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c)))
t.Fatal("crash boundary not reached")
}
- for _, boundary := range []string{"captured", "forward restart", "failed journal", "rollback pending", "rollback running", "settled"} {
+ for _, boundary := range []string{"captured", "forward running", "forward restart", "failed journal", "rollback pending", "rollback running", "settled"} {
t.Run(boundary, func(t *testing.T) {
dir := t.TempDir()
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
@@ -190,10 +190,14 @@ func TestPlan14RollbackProcessCrash(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if u.exe != release.VersionPath(u.opt, "bab", "v1", "console") || u.props.MainPID <= 0 {
+ want := "v1"
+ if boundary == "forward running" {
+ want = "v2"
+ }
+ if u.exe != release.VersionPath(u.opt, "bab", want, "console") || u.props.MainPID <= 0 {
t.Fatalf("no recovered process: %+v", u)
}
- if (boundary == "rollback running" || boundary == "settled") && len(u.effects) != before {
+ if (boundary == "forward running" || boundary == "rollback running" || boundary == "settled") && len(u.effects) != before {
t.Fatalf("restarted twice: %v", u.effects)
}
}
diff --git a/internal/host/rootrollback_resume.go b/internal/host/rootrollback_resume.go
index 65560359d..41a61bcb2 100644
--- a/internal/host/rootrollback_resume.go
+++ b/internal/host/rootrollback_resume.go
@@ -34,7 +34,14 @@ func recoverRootRestarts(ctx context.Context, sys unitSystem, optDir string, get
body, err := s.Read(name)
var a rollbackAttempt
if err != nil || json.Unmarshal(body, &a) != nil || name != rollbackJournalKey(a.Host) {
- return errors.Join(err, os.ErrPermission)
+ failures = append(failures, errors.Join(err, os.ErrPermission))
+ continue
+ }
+ if a.Settled {
+ if err := removeRollbackCapabilities(s, a.Host); err != nil {
+ failures = append(failures, err)
+ }
+ continue
}
if err := resumeRootRestart(ctx, sys, optDir, getenv, checks, writes, s, a); err != nil {
failures = append(failures, err)
@@ -54,7 +61,7 @@ func resumeRootRestart(ctx context.Context, sys unitSystem, optDir string, geten
return errors.Join(err, os.ErrPermission)
}
if current.Settled {
- return settleRollbackAttempt(s, current)
+ return nil
}
a, b, err := loadRollback(s, expected.Host, checks.now())
if a.ID != expected.ID {
@@ -88,6 +95,11 @@ func resumeRootRestart(ctx context.Context, sys unitSystem, optDir string, geten
if err != nil || (live != a.Target && live != old) {
return os.ErrPermission
}
+ if !a.Recovering && !a.Failed && p.ActiveState == "active" && p.MainPID > 0 && live == a.Target {
+ checks.verifyOnly = true
+ checks.targetDigest = a.TargetDigest
+ return nil
+ }
if a.Recovering && p.ActiveState == "active" && p.MainPID > 0 && live == old {
checks.verifyOnly = true
} else if p.ActiveState != "failed" && p.MainPID != 0 && !a.Failed {
@@ -105,6 +117,13 @@ func resumeRootRestart(ctx context.Context, sys unitSystem, optDir string, geten
if err != nil {
return err
}
+ if checks.verifyOnly && !a.Recovering {
+ b, err = checks.host(ctx, a.Host)
+ if err != nil {
+ return err
+ }
+ return restartBoundUnit(sys, optDir, getenv, b, false, nil, checks)
+ }
b, err = checks.rollback(ctx, b)
if err != nil {
return err
diff --git a/internal/host/rootrollback_test.go b/internal/host/rootrollback_test.go
index 45f3b7d9d..58dc4a7ec 100644
--- a/internal/host/rootrollback_test.go
+++ b/internal/host/rootrollback_test.go
@@ -125,7 +125,7 @@ func TestPlan14RollbackCapability(t *testing.T) {
rollbackErr = rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c)(host)
return rollbackErr
}))
- positive := attack == "crash" || attack == "later process" || attack == "failed restart result" || attack == "spent"
+ positive := attack == "missing" || attack == "stale" || attack == "settled" || attack == "missing journal" || attack == "crash" || attack == "later process" || attack == "failed restart result" || attack == "spent"
if err == nil {
t.Fatal("forward failure disappeared")
}
@@ -149,7 +149,7 @@ func TestPlan14RollbackCapability(t *testing.T) {
t.Log("absent capability refused")
}
for k := range mem.files {
- if k != rollbackJournalKey(rt.Host) {
+ if k != rollbackJournalKey(rt.Host) && k != rollbackHostKey(rt.Host)+".previous" {
t.Fatalf("capability survived settlement: %s", k)
}
}
diff --git a/internal/host/rootroute.go b/internal/host/rootroute.go
index 85964855d..09b589c53 100644
--- a/internal/host/rootroute.go
+++ b/internal/host/rootroute.go
@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
+ "errors"
"github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
@@ -48,6 +49,9 @@ func sameProofCandidate(candidate, r registry.Route, declaredUnit string) bool {
func (s *proofBindingSource) needsRestart(ctx context.Context, candidate registry.Route) (bool, error) {
evidence, err := s.record(ctx, candidate.Host)
if err != nil {
+ if errors.Is(err, os.ErrNotExist) && candidate.Unit == "" {
+ return false, nil
+ }
return false, err
}
rec := evidence.record
diff --git a/internal/hostbox/coldhandover.go b/internal/hostbox/coldhandover.go
index af1a734d2..166d489a0 100644
--- a/internal/hostbox/coldhandover.go
+++ b/internal/hostbox/coldhandover.go
@@ -626,6 +626,9 @@ func (h ColdHandover) Recover(ctx context.Context) error {
}
}
if err = h.Run(ctx, j.Prior, j.Successor); err != nil {
+ if h.Guard.Root != nil && h.Guard.AbandonColdHandover(ctx, j.Barrier) == nil {
+ continue
+ }
return err
}
}
diff --git a/internal/hostbox/inventory_expected.go b/internal/hostbox/inventory_expected.go
index 9f762dcbf..20a950ef6 100644
--- a/internal/hostbox/inventory_expected.go
+++ b/internal/hostbox/inventory_expected.go
@@ -41,3 +41,13 @@ func (d *StopInventoryDB) WriteExpected(ctx context.Context, in StopInventory, e
}
func (d *StopInventoryDB) Remote() bool { return d != nil && d.remote != nil }
+
+func (d *StopInventoryDB) SnapshotRequest(ctx context.Context, target hostguard.FenceTarget) (JournalResponse, error) {
+ if d == nil {
+ return JournalResponse{}, ErrUnsupported
+ }
+ if d.remote != nil {
+ return journalCall(ctx, d.remote, JournalRequest{Op: "inventory-version", Target: target})
+ }
+ return d.inventoryVersion(ctx, target)
+}
diff --git a/internal/hostexec/plan14_fix2_linux_test.go b/internal/hostexec/plan14_fix2_linux_test.go
new file mode 100644
index 000000000..11ff43e8e
--- /dev/null
+++ b/internal/hostexec/plan14_fix2_linux_test.go
@@ -0,0 +1,105 @@
+package hostexec
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "github.com/carlosframework/platform/internal/release"
+ "os"
+ "testing"
+ "time"
+)
+
+func TestPlan14Fix2RenewReplyLostSettlesAfterExpiry(t *testing.T) {
+ b, _, remote, clock, q, p := brokerFixture(t)
+ out, err := b.Execute(t.Context(), p, q)
+ if err != nil {
+ t.Fatal(err)
+ }
+ r, err := b.read(q.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, _ := Key(q.Host, r.Authority.Box)
+ r.Expires = r.Expires.Add(Renewal)
+ raw, _ := json.Marshal(Record{Op: q.Op, Expires: r.Expires})
+ if _, err := remote.Store.Put(t.Context(), key, raw, r.LeaseETag); err != nil {
+ t.Fatal(err)
+ }
+ r.Status = "pending"
+ if err := b.write(r); err != nil {
+ t.Fatal(err)
+ }
+ q.Operation, q.Token, q.ID = "executionSettle", out.Token, "settle-lost-response"
+ if _, err := b.Execute(t.Context(), p, q); err == nil {
+ t.Fatal("unexpired unknown ETag deleted")
+ }
+ clock.advance(TTL + Renewal + time.Second)
+ if _, err := b.Execute(t.Context(), p, q); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := remote.Get(t.Context(), key); !errors.Is(err, os.ErrNotExist) {
+ t.Fatal("remote lease retained", err)
+ }
+}
+func TestPlan14Fix2TransientRenewalRetries(t *testing.T) {
+ for _, kind := range []string{"authority", "put"} {
+ t.Run(kind, func(t *testing.T) {
+ b, _, remote, _, q, p := brokerFixture(t)
+ out, err := b.Execute(t.Context(), p, q)
+ if err != nil {
+ t.Fatal(err)
+ }
+ q.Operation, q.Token, q.ID = "executionRenew", out.Token, "renew"
+ calls := 0
+ orig := b.cfg.Authority
+ if kind == "authority" {
+ b.cfg.Authority = func(ctx context.Context, q ExecutionRequest, p ControllerProcess) (ExecutionAuthority, error) {
+ calls++
+ if calls == 1 {
+ return ExecutionAuthority{}, errors.New("transient transport")
+ }
+ return orig(ctx, q, p)
+ }
+ } else {
+ remote.failAt = 2
+ }
+ renewed, err := b.Execute(t.Context(), p, q)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if renewed.Status != "active" || renewed.Token == out.Token {
+ t.Fatal("renewal not committed", renewed)
+ }
+ })
+ }
+}
+func TestPlan14Fix2LostMonitorBacksOff(t *testing.T) {
+ b, d, _, clock, q, p := brokerFixture(t)
+ if _, err := b.Execute(t.Context(), p, q); err != nil {
+ t.Fatal(err)
+ }
+ r, err := b.read(q.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := b.deny(t.Context(), r, nil); err != nil {
+ t.Fatal(err)
+ }
+ attempts := 0
+ b.cfg.Settle = func(context.Context, ExecutionRecord) error { attempts++; return release.ErrCASConflict }
+ _ = b.Monitor(t.Context())
+ writes := d.writes
+ for i := 0; i < 9; i++ {
+ clock.advance(100 * time.Millisecond)
+ _ = b.Monitor(t.Context())
+ }
+ if attempts != 1 || d.writes != writes {
+ t.Fatal("lost record hot-loop", attempts, d.writes, writes)
+ }
+ clock.advance(101 * time.Millisecond)
+ _ = b.Monitor(t.Context())
+ if attempts != 2 {
+ t.Fatal("backoff never retried", attempts)
+ }
+}
diff --git a/internal/hostexec/rootbroker.go b/internal/hostexec/rootbroker.go
index 2af2f7a54..c2ce9f4d7 100644
--- a/internal/hostexec/rootbroker.go
+++ b/internal/hostexec/rootbroker.go
@@ -1,6 +1,7 @@
package hostexec
import (
+ "bytes"
"context"
"crypto/rand"
"crypto/sha256"
@@ -53,6 +54,8 @@ type RootBroker struct {
calls map[string]chan struct{}
lifetime map[string]func()
live map[string]func()
+ retryAfter map[string]int64
+ retryDelay map[string]time.Duration
}
func NewRootBroker(cfg RootBrokerConfig) (*RootBroker, error) {
@@ -60,7 +63,7 @@ func NewRootBroker(cfg RootBrokerConfig) (*RootBroker, error) {
return nil, ErrState
}
lifetimeCtx, cancel := context.WithCancel(context.Background())
- return &RootBroker{cfg: cfg, lifetimeCtx: lifetimeCtx, cancel: cancel, closeGate: make(chan struct{}, 1), denied: make(map[string]bool), calls: make(map[string]chan struct{}), lifetime: make(map[string]func()), live: make(map[string]func())}, nil
+ return &RootBroker{cfg: cfg, lifetimeCtx: lifetimeCtx, cancel: cancel, closeGate: make(chan struct{}, 1), denied: make(map[string]bool), calls: make(map[string]chan struct{}), lifetime: make(map[string]func()), live: make(map[string]func()), retryAfter: make(map[string]int64), retryDelay: make(map[string]time.Duration)}, nil
}
func (b *RootBroker) enter(ctx context.Context) (context.Context, func(), error) {
@@ -243,7 +246,11 @@ func (b *RootBroker) Execute(ctx context.Context, peer ControllerProcess, q Exec
var authority ExecutionAuthority
var err error
if q.Operation != "executionSettle" {
- authority, err = b.cfg.Authority(ctx, q, peer)
+ cutoff := int64(0)
+ if prior, readErr := b.read(q.Host); readErr == nil && prior.Status == "active" {
+ cutoff = prior.SafeUntil
+ }
+ err = b.retry(ctx, cutoff, func(call context.Context) error { var e error; authority, e = b.cfg.Authority(call, q, peer); return e })
if err == nil && (authority.Migration == 0 || (authority.Installation == "" && !authority.ParticipationOnly) || !validID(authority.Box) || !validID(authority.Enrollment) || authority.Account == "" || authority.App == "" || authority.ClaimRevision == "" || authority.RecordRevision == "" || authority.DirectoryRevision == "") {
err = ErrState
}
@@ -305,6 +312,8 @@ func (b *RootBroker) Execute(ctx context.Context, peer ControllerProcess, q Exec
}
b.mu.Lock()
delete(b.denied, q.Host)
+ delete(b.retryAfter, q.Host)
+ delete(b.retryDelay, q.Host)
b.mu.Unlock()
r = ExecutionRecord{Schema: 1, Request: q, Peer: peer, Authority: authority, Broker: b.cfg.Identity, Token: hex.EncodeToString(token[:]), Status: "pending", Replay: history}
wall, start, e := b.cfg.Clock.Now()
@@ -427,12 +436,39 @@ func (b *RootBroker) acquire(ctx context.Context, peer ControllerProcess, q Exec
}
}
raw, _ := json.Marshal(Record{Op: q.Op, Expires: r.Expires})
- next, err := b.cfg.Remote.Put(ctx, key, raw, etag)
+ var next string
+ cutoff := r.SafeUntil
+ if r.PriorSafeUntil > 0 && r.PriorSafeUntil < cutoff {
+ cutoff = r.PriorSafeUntil
+ }
+ uncertain := false
+ err = b.retry(ctx, cutoff, func(call context.Context) error {
+ if uncertain {
+ found, current, e := b.cfg.Remote.Get(call, key)
+ if e == nil && bytes.Equal(found, raw) && current != "" {
+ next = current
+ return nil
+ }
+ if e != nil && !errors.Is(e, os.ErrNotExist) {
+ return e
+ }
+ if e == nil && current != etag {
+ return ErrLost
+ }
+ }
+ var e error
+ next, e = b.cfg.Remote.Put(call, key, raw, etag)
+ if e != nil && !errors.Is(e, release.ErrCASConflict) {
+ uncertain = true
+ }
+ return e
+ })
if err != nil || next == "" {
return ExecutionResponse{}, b.deny(ctx, r, errors.Join(err, ErrLost))
}
r.LeaseETag = next
- authority, err := b.cfg.Authority(ctx, q, peer)
+ var authority ExecutionAuthority
+ err = b.retry(ctx, cutoff, func(call context.Context) error { var e error; authority, e = b.cfg.Authority(call, q, peer); return e })
if err != nil || authority != r.Authority {
return ExecutionResponse{}, b.deny(ctx, r, errors.Join(err, ErrLost))
}
@@ -468,27 +504,35 @@ func (b *RootBroker) settle(ctx context.Context, q ExecutionRequest, r Execution
if err != nil {
return ExecutionResponse{}, err
}
- if r.LeaseETag != "" {
- if err = release.DeleteIfMatch(ctx, b.cfg.Remote, key, r.LeaseETag); err != nil {
+ deletionUncertain := r.LeaseETag == ""
+ if !deletionUncertain {
+ err = release.DeleteIfMatch(ctx, b.cfg.Remote, key, r.LeaseETag)
+ if errors.Is(err, release.ErrCASConflict) {
+ deletionUncertain = true
+ } else if err != nil && !errors.Is(err, os.ErrNotExist) {
return ExecutionResponse{}, err
}
- } else {
- raw, _, readErr := b.cfg.Remote.Get(ctx, key)
+ }
+ if deletionUncertain {
+ raw, currentETag, readErr := b.cfg.Remote.Get(ctx, key)
wall, _, clockErr := b.cfg.Clock.Now()
if clockErr != nil {
return ExecutionResponse{}, clockErr
}
+ if !wall.After(r.Expires) {
+ return ExecutionResponse{}, ErrLost
+ }
if readErr == nil {
var remote Record
- if decode(raw, &remote) != nil || remote.Expires.IsZero() || !wall.After(remote.Expires) {
+ if decode(raw, &remote) != nil || remote.Expires.IsZero() || !wall.After(remote.Expires) || currentETag == "" {
return ExecutionResponse{}, ErrLost
}
+ if err := release.DeleteIfMatch(ctx, b.cfg.Remote, key, currentETag); err != nil {
+ return ExecutionResponse{}, err
+ }
} else if !errors.Is(readErr, os.ErrNotExist) {
return ExecutionResponse{}, readErr
}
- if !wall.After(r.Expires) {
- return ExecutionResponse{}, ErrLost
- }
}
if err = b.cfg.Settle(ctx, r); err != nil {
return ExecutionResponse{}, err
@@ -534,7 +578,34 @@ func (b *RootBroker) Monitor(ctx context.Context) error {
continue
}
if r.Status == "lost" {
- errs = append(errs, b.recoverSettlement(ctx, host))
+ _, now, clockErr := b.cfg.Clock.Now()
+ if clockErr != nil {
+ errs = append(errs, clockErr)
+ continue
+ }
+ b.mu.Lock()
+ next := b.retryAfter[host]
+ b.mu.Unlock()
+ if now < next {
+ continue
+ }
+ err := b.recoverSettlement(ctx, host)
+ b.mu.Lock()
+ if err != nil {
+ delay := b.retryDelay[host] * 2
+ if delay < time.Second {
+ delay = time.Second
+ }
+ if delay > 30*time.Second {
+ delay = 30 * time.Second
+ }
+ b.retryDelay[host], b.retryAfter[host] = delay, now+int64(delay)
+ } else {
+ delete(b.retryDelay, host)
+ delete(b.retryAfter, host)
+ }
+ b.mu.Unlock()
+ errs = append(errs, err)
continue
}
_, now, clockErr := b.cfg.Clock.Now()
@@ -703,3 +774,40 @@ func (b *RootBroker) Run(ctx context.Context) {
_ = b.Monitor(ctx)
}
}
+
+func (b *RootBroker) retry(ctx context.Context, cutoff int64, fn func(context.Context) error) error {
+ _, now, err := b.cfg.Clock.Now()
+ if err != nil {
+ return err
+ }
+ slack := time.Second
+ if cutoff != 0 {
+ slack = time.Duration(cutoff - now)
+ if slack <= 0 {
+ return ErrLost
+ }
+ }
+ bounded, cancel := context.WithTimeout(ctx, slack)
+ defer cancel()
+ delay := 25 * time.Millisecond
+ for {
+ err = fn(bounded)
+ if err == nil || errors.Is(err, ErrState) || errors.Is(err, ErrInactive) || errors.Is(err, ErrLost) || errors.Is(err, ErrUnsupported) || errors.Is(err, os.ErrPermission) || errors.Is(err, os.ErrInvalid) || errors.Is(err, os.ErrNotExist) || errors.Is(err, release.ErrCASConflict) {
+ return err
+ }
+ _, now, clockErr := b.cfg.Clock.Now()
+ if clockErr != nil || cutoff != 0 && now >= cutoff {
+ return errors.Join(err, clockErr, ErrLost)
+ }
+ timer := time.NewTimer(delay)
+ select {
+ case <-bounded.Done():
+ timer.Stop()
+ return errors.Join(err, bounded.Err())
+ case <-timer.C:
+ }
+ if delay < 500*time.Millisecond {
+ delay *= 2
+ }
+ }
+}
diff --git a/internal/hostguard/coldhandover.go b/internal/hostguard/coldhandover.go
index 1b3ac00d5..b477c6f05 100644
--- a/internal/hostguard/coldhandover.go
+++ b/internal/hostguard/coldhandover.go
@@ -117,3 +117,34 @@ func (g Guard) CompletedColdPredecessor(ctx context.Context, host string, captur
})
return found, err
}
+
+func (g Guard) AbandonColdHandover(ctx context.Context, barrier FenceTarget) error {
+ if g.Root == nil {
+ return ErrUnsupported
+ }
+ return g.readLocked(ctx, barrier.Host, func(d *recordDir) error {
+ c, err := g.fresh(ctx, barrier.Host)
+ if err != nil {
+ return err
+ }
+ if c.Holder.Incarnation == barrier.Incarnation && (c.Holder.Stamp < barrier.Stamp || c.Holder.Session <= barrier.Session) {
+ return ErrFenced
+ }
+ r, err := d.read(barrier.Host)
+ if err != nil {
+ return err
+ }
+ if r.barrier == nil {
+ return nil
+ }
+ if *r.barrier != barrier {
+ return ErrFenceTarget
+ }
+ r.rootRetired = append(r.rootRetired, Captured{Incarnation: barrier.Incarnation, Stamp: barrier.Stamp})
+ r.barrier = nil
+ if r.ActiveFence != nil && *r.ActiveFence == barrier {
+ r.ActiveFence = nil
+ }
+ return d.write(r)
+ })
+}
diff --git a/internal/hostguard/guard.go b/internal/hostguard/guard.go
index 427deab80..19a831554 100644
--- a/internal/hostguard/guard.go
+++ b/internal/hostguard/guard.go
@@ -3,6 +3,7 @@ package hostguard
import (
"context"
"errors"
+ "time"
"github.com/carlosframework/platform/internal/hostclaim"
"github.com/carlosframework/platform/internal/release"
@@ -68,8 +69,24 @@ func (g Guard) locked(ctx context.Context, host string, fn func(*recordDir) erro
if dir == "" {
dir = "/run/carlos/hostguard"
}
- unlock, err := acquire(ctx, dir, host)
+ lockCtx := ctx
+ cancel := func() {}
+ if g.Root != nil {
+ lockCtx, cancel = context.WithTimeout(ctx, 250*time.Millisecond)
+ }
+ defer cancel()
+ unlock, err := acquire(lockCtx, dir, host)
if err != nil {
+ if g.Root != nil {
+ r, readErr := g.Root.read(host)
+ if readErr == nil {
+ r.quarantined = true
+ return errors.Join(err, g.Root.write(r))
+ }
+ if !errors.Is(readErr, ErrNoRecord) {
+ return errors.Join(err, readErr)
+ }
+ }
return err
}
defer unlock()
@@ -168,6 +185,15 @@ func (g Guard) With(ctx context.Context, host string, captured Captured, check f
if err := ctx.Err(); err != nil {
return err
}
+ if g.FenceView != nil {
+ v, err := g.FenceView(ctx, host, captured)
+ if err != nil {
+ return err
+ }
+ if err := v.admit(host, captured, c.Holder.Session); err != nil {
+ return err
+ }
+ }
return fn()
})
}
@@ -176,7 +202,7 @@ func (g Guard) With(ctx context.Context, host string, captured Captured, check f
// lets the captured installation run claim c's session. AdmissionFenced
// reads the same, so the two cannot disagree.
func (r record) admits(c *hostclaim.Claim, captured Captured) error {
- if r.pendingInstallation != nil || r.barrier != nil {
+ if r.quarantined || r.pendingInstallation != nil || r.barrier != nil {
return ErrFenced
}
if r.root && (r.installation.Account != c.Holder.Account || r.installation.App != c.Holder.App) {
@@ -227,7 +253,7 @@ func (g Guard) Activate(ctx context.Context, host, box string, participant func(
if err != nil {
return err
}
- if r.barrier != nil || r.Incarnation != c.Holder.Incarnation || r.ActiveStamp != c.Holder.Stamp || r.ActiveStamp == 0 || r.installation.Box != box {
+ if r.quarantined || r.pendingInstallation != nil || r.barrier != nil || r.Incarnation != c.Holder.Incarnation || r.ActiveStamp != c.Holder.Stamp || r.ActiveStamp == 0 || r.installation.Box != box {
return ErrFenced
}
return d.sync(host)
@@ -310,3 +336,18 @@ func (g Guard) RetireInstallation(ctx context.Context, host string, check func(*
return d.write(r)
})
}
+
+func (g Guard) readLocked(ctx context.Context, host string, fn func(*recordDir) error) error {
+ if g.Root == nil {
+ return g.locked(ctx, host, fn)
+ }
+ if validateFileHost(host) != nil || g.Root.store == nil {
+ return ErrInvalidInput
+ }
+ unlock, err := g.Root.store.Lock(ctx, host+".lock")
+ if err != nil {
+ return err
+ }
+ defer unlock()
+ return fn(&recordDir{root: g.Root})
+}
diff --git a/internal/hostguard/plan14_fix2_unix_test.go b/internal/hostguard/plan14_fix2_unix_test.go
new file mode 100644
index 000000000..f9b603756
--- /dev/null
+++ b/internal/hostguard/plan14_fix2_unix_test.go
@@ -0,0 +1,107 @@
+//go:build unix
+
+package hostguard
+
+import (
+ "context"
+ "errors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "testing"
+ "time"
+)
+
+func TestPlan14Fix2LegacyLockCannotStallRoot(t *testing.T) {
+ f, target, captured := fenceFixture(t)
+ f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console"
+ f.g.Root = NewRootAuthority(&rootTestStore{dir: t.TempDir()})
+ in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record", Box: target.Box, Enrollment: target.Enrollment, Migration: 1}
+ if err := f.g.EstablishRoot(t.Context(), in, allow); err != nil {
+ t.Fatal(err)
+ }
+ unlock, err := acquire(t.Context(), f.g.LockDir, target.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer unlock()
+ ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ if _, err := f.g.PendingFences(ctx); err != nil {
+ t.Fatal("read waited for legacy lock", err)
+ }
+ if _, err := f.g.BeginFence(ctx, target); err == nil {
+ t.Fatal("mutation crossed held legacy lock")
+ }
+ r, err := f.g.Root.read(target.Host)
+ if err != nil || !r.quarantined {
+ t.Fatalf("missing durable deny: %+v %v", r, err)
+ }
+ if _, err := f.g.Root.View(ctx, target.Host, captured); !errors.Is(err, ErrFenced) {
+ t.Fatal("quarantined view admitted", err)
+ }
+}
+func TestPlan14Fix2PendingIntentCanBeAbandoned(t *testing.T) {
+ f, target, captured := fenceFixture(t)
+ f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console"
+ f.g.Root = NewRootAuthority(&rootTestStore{dir: t.TempDir()})
+ in := RootInstallation{Host: target.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record", Box: target.Box, Enrollment: target.Enrollment, Migration: 1, InventoryRevision: 1}
+ intent := []byte("pending")
+ failure := errors.New("commit failed")
+ if err := f.g.CommitRootInventory(t.Context(), in, intent, allow, nil, func(RootInstallation) error { return failure }); !errors.Is(err, failure) {
+ t.Fatal(err)
+ }
+ if err := f.g.AbandonRootInventory(t.Context(), target.Host, intent, func(*hostclaim.Claim) bool { return false }); !errors.Is(err, ErrFenced) {
+ t.Fatal("abandoned committable intent", err)
+ }
+ if err := f.g.AbandonRootInventory(t.Context(), target.Host, intent, func(*hostclaim.Claim) bool { return true }); err != nil {
+ t.Fatal(err)
+ }
+ pending, err := f.g.Root.PendingInstallations()
+ if err != nil || len(pending) != 0 {
+ t.Fatal(pending, err)
+ }
+ if _, err := f.g.Root.View(t.Context(), target.Host, captured); !errors.Is(err, ErrFenced) {
+ t.Fatal("abandon discarded deny", err)
+ }
+ if err := f.g.CommitRootInventory(t.Context(), in, []byte("new"), allow, nil, func(RootInstallation) error { return nil }); err != nil {
+ t.Fatal("fresh intent refused", err)
+ }
+ if _, err := f.g.Root.View(t.Context(), target.Host, captured); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPlan14Fix2SupersededColdBarrierAbandoned(t *testing.T) {
+ f, prior, captured := fenceFixture(t)
+ f.obs.Claim.Holder.Account, f.obs.Claim.Holder.App = "bab", "console"
+ f.g.Root = NewRootAuthority(&rootTestStore{dir: t.TempDir()})
+ in := RootInstallation{Host: prior.Host, Incarnation: captured.Incarnation, Stamp: captured.Stamp, Account: "bab", App: "console", Record: "record", Box: prior.Box, Enrollment: prior.Enrollment, Migration: 1}
+ if err := f.g.EstablishRoot(t.Context(), in, allow); err != nil {
+ t.Fatal(err)
+ }
+ f.obs.Claim.Op.Kind = hostclaim.KindMoving
+ if _, err := f.g.BeginFence(t.Context(), prior); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.g.CompleteFence(t.Context(), prior, []byte("receipt"), time.Now(), func(context.Context, FenceTarget, []byte, time.Time) error { return nil }); err != nil {
+ t.Fatal(err)
+ }
+ barrier := prior
+ barrier.FenceID = "barrier"
+ barrier.Stamp = 17
+ barrier.Session = 17
+ f.obs.Claim.Holder.Stamp, f.obs.Claim.Holder.Session = 17, 17
+ if err := f.g.BeginColdHandover(t.Context(), prior, barrier, allow); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.g.AbandonColdHandover(t.Context(), barrier); !errors.Is(err, ErrFenced) {
+ t.Fatal("abandoned live admission", err)
+ }
+ f.obs.Claim.Holder.Stamp, f.obs.Claim.Holder.Session = 18, 18
+ if err := f.g.AbandonColdHandover(t.Context(), barrier); err != nil {
+ t.Fatal(err)
+ }
+ r, err := f.g.Root.read(prior.Host)
+ if err != nil || r.barrier != nil || len(r.FenceJournal) != 1 || r.FenceJournal[0].Target != prior {
+ t.Fatalf("barrier/receipt: %+v %v", r, err)
+ }
+}
diff --git a/internal/hostguard/plan14_root_fences_test.go b/internal/hostguard/plan14_root_fences_test.go
index b442da120..1ee1c82c3 100644
--- a/internal/hostguard/plan14_root_fences_test.go
+++ b/internal/hostguard/plan14_root_fences_test.go
@@ -215,7 +215,7 @@ func TestPlan14FenceViewRechecksAfterUnlockedRead(t *testing.T) {
t.Errorf("stale root snapshot admitted an already fenced session: err=%v effects=%d", err, effects)
}
} else {
- if err != nil || effects != 1 {
+ if (!errors.Is(err, ErrFenced) || effects != 0) && (err != nil || effects != 1) {
t.Fatalf("unfenced admission: err=%v effects=%d", err, effects)
}
if _, err := root.BeginFence(t.Context(), target); err != nil {
@@ -223,6 +223,13 @@ func TestPlan14FenceViewRechecksAfterUnlockedRead(t *testing.T) {
}
}
view, err := root.Root.View(t.Context(), target.Host, captured)
+ if errors.Is(err, ErrFenced) {
+ r, e := root.Root.read(target.Host)
+ if e != nil || !r.quarantined || r.installation != in {
+ t.Fatalf("quarantine changed installation: %+v %v", r, e)
+ }
+ return
+ }
if err != nil || view.Pending == nil || *view.Pending != target || view.Installation != in {
t.Fatalf("fence changed installation identity: %+v %v", view, err)
}
@@ -274,20 +281,20 @@ func TestPlan14FenceViewWhileRootWaitsForAgent(t *testing.T) {
go func() { _, err := root.Root.View(ctx, target.Host, captured); viewDone <- err }()
select {
case err := <-viewDone:
- if err != nil {
+ if !errors.Is(err, ErrFenced) {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal("fence view waited on root holding a lock needed by the agent")
}
- if n := acquisitions.Load(); n != 1 {
- t.Fatalf("view acquired a root host lock: %d acquisitions", n)
+ if n := acquisitions.Load(); n != 2 {
+ t.Fatalf("root mutation and view must each acquire the root lock: %d acquisitions", n)
}
unlock()
unlock = nil
select {
case err := <-done:
- if err != nil {
+ if !errors.Is(err, context.DeadlineExceeded) {
t.Fatal(err)
}
case <-ctx.Done():
diff --git a/internal/hostguard/record.go b/internal/hostguard/record.go
index 515990b7c..1e611edf8 100644
--- a/internal/hostguard/record.go
+++ b/internal/hostguard/record.go
@@ -26,6 +26,7 @@ type Captured struct {
}
type record struct {
+ quarantined bool
root bool
installation RootInstallation
rootRetired []Captured
diff --git a/internal/hostguard/recoveryfence.go b/internal/hostguard/recoveryfence.go
index 9ecfe29c5..9bab40b4c 100644
--- a/internal/hostguard/recoveryfence.go
+++ b/internal/hostguard/recoveryfence.go
@@ -21,7 +21,7 @@ func (g Guard) PendingFences(ctx context.Context) ([]FenceTarget, error) {
continue
}
host := strings.TrimSuffix(name, ".json")
- err := g.locked(ctx, host, func(d *recordDir) error {
+ err := g.readLocked(ctx, host, func(d *recordDir) error {
r, err := d.read(host)
if err != nil {
return err
diff --git a/internal/hostguard/retirement.go b/internal/hostguard/retirement.go
index a8b62b324..99e456595 100644
--- a/internal/hostguard/retirement.go
+++ b/internal/hostguard/retirement.go
@@ -82,7 +82,7 @@ func (g Guard) WithNoInstallation(ctx context.Context, host, incarnation string,
}
if g.Root != nil {
captured := Captured{Incarnation: incarnation, Stamp: c.Holder.Stamp}
- view, err := g.Root.View(ctx, host, captured)
+ view, err := g.Root.viewLocked(host, captured)
if err != nil {
return err
}
diff --git a/internal/hostguard/root.go b/internal/hostguard/root.go
index 579ea9da4..3ffed074f 100644
--- a/internal/hostguard/root.go
+++ b/internal/hostguard/root.go
@@ -32,6 +32,7 @@ type RootAuthority struct {
func NewRootAuthority(store rootstate.Store) *RootAuthority { return &RootAuthority{store: store} }
type rootRecord struct {
+ Quarantined bool `json:",omitempty"`
RebindCommit []byte `json:",omitempty"`
Schema int
Installation RootInstallation
@@ -60,6 +61,7 @@ func (a *RootAuthority) read(host string) (record, error) {
return record{}, ErrMalformedRecord
}
r := saved.State
+ r.quarantined = saved.Quarantined
r.root = true
r.installation = saved.Installation
r.rootRetired = saved.Retired
@@ -80,7 +82,7 @@ func (a *RootAuthority) write(r record) error {
return ErrMalformedRecord
}
r.installation.Host, r.installation.Incarnation, r.installation.Stamp = r.Host, r.Incarnation, r.ActiveStamp
- b, err := json.Marshal(rootRecord{RebindCommit: r.rebindCommit, Schema: 1, Installation: r.installation, Retired: r.rootRetired, Barrier: r.barrier, PendingInventory: r.pendingInventory, PendingInstallation: r.pendingInstallation, State: r})
+ b, err := json.Marshal(rootRecord{Quarantined: r.quarantined, RebindCommit: r.rebindCommit, Schema: 1, Installation: r.installation, Retired: r.rootRetired, Barrier: r.barrier, PendingInventory: r.pendingInventory, PendingInstallation: r.pendingInstallation, State: r})
if err != nil {
return err
}
@@ -159,10 +161,6 @@ func (g Guard) CommitRootInventory(ctx context.Context, in RootInstallation, int
if r.Incarnation == in.Incarnation && r.ActiveStamp > in.Stamp {
return ErrOldStamp
}
- if r.Incarnation != in.Incarnation || r.ActiveStamp != in.Stamp {
- r.retire()
- r.RetiredStamps = nil
- }
prior := r.installation
if prior.Incarnation == in.Incarnation && prior.Stamp == in.Stamp && in.Registration == (InstallationRegistration{}) {
in.Registration = prior.Registration
@@ -191,8 +189,13 @@ func (g Guard) CommitRootInventory(ctx context.Context, in RootInstallation, int
if err := commit(prior); err != nil {
return err
}
+ if r.Incarnation != in.Incarnation || r.ActiveStamp != in.Stamp {
+ r.retire()
+ r.RetiredStamps = nil
+ }
r.pendingInventory, r.pendingInstallation = nil, nil
r.Incarnation, r.ActiveStamp, r.installation = in.Incarnation, in.Stamp, in
+ r.quarantined = false
return d.write(r)
})
}
@@ -249,3 +252,28 @@ func (a *RootAuthority) PendingInstallation(ctx context.Context, host string) (R
}
return *r.pendingInstallation, append([]byte(nil), r.pendingInventory...), nil
}
+
+func (g Guard) AbandonRootInventory(ctx context.Context, host string, intent []byte, superseded func(*hostclaim.Claim) bool) error {
+ if g.Root == nil || superseded == nil {
+ return ErrInvalidInput
+ }
+ return g.readLocked(ctx, host, func(d *recordDir) error {
+ r, err := d.read(host)
+ if err != nil {
+ return err
+ }
+ if !bytes.Equal(r.pendingInventory, intent) || r.pendingInstallation == nil {
+ return ErrInvalidInput
+ }
+ c, err := g.fresh(ctx, host)
+ if err != nil {
+ return err
+ }
+ if !superseded(c) {
+ return ErrFenced
+ }
+ r.pendingInventory, r.pendingInstallation = nil, nil
+ r.quarantined = true
+ return d.write(r)
+ })
+}
diff --git a/internal/hostguard/rootrebind.go b/internal/hostguard/rootrebind.go
index aa58445f7..69850bea2 100644
--- a/internal/hostguard/rootrebind.go
+++ b/internal/hostguard/rootrebind.go
@@ -70,6 +70,7 @@ func (g Guard) RebindRootInventory(ctx context.Context, prior Captured, revision
r.retire()
r.Incarnation, r.ActiveStamp, r.installation = next.Incarnation, next.Stamp, next
r.pendingInventory, r.pendingInstallation = nil, nil
+ r.quarantined = false
r.rebindCommit = append([]byte(nil), intent...)
return d.write(r)
})
diff --git a/internal/hostguard/rootview.go b/internal/hostguard/rootview.go
index be7074b49..5e33329a3 100644
--- a/internal/hostguard/rootview.go
+++ b/internal/hostguard/rootview.go
@@ -18,11 +18,23 @@ func (a *RootAuthority) View(ctx context.Context, host string, captured Captured
if err := ctx.Err(); err != nil {
return FenceView{}, err
}
+ if a == nil || a.store == nil || validateFileHost(host) != nil {
+ return FenceView{}, ErrInvalidInput
+ }
+ unlock, err := a.store.Lock(ctx, host+".lock")
+ if err != nil {
+ return FenceView{}, err
+ }
+ defer unlock()
+ return a.viewLocked(host, captured)
+}
+
+func (a *RootAuthority) viewLocked(host string, captured Captured) (FenceView, error) {
r, err := a.read(host)
if err != nil {
return FenceView{}, err
}
- if r.pendingInstallation != nil {
+ if r.quarantined || r.pendingInstallation != nil {
return FenceView{}, ErrFenced
}
known := r.Incarnation == captured.Incarnation && r.ActiveStamp == captured.Stamp && captured.Stamp != 0
@@ -83,7 +95,7 @@ func (v FenceView) admit(host string, c Captured, claimSession uint64) error {
}
func (g Guard) fenceLocked(ctx context.Context, host string, c Captured, fn func(*recordDir) error) error {
if g.FenceView == nil {
- return g.locked(ctx, host, fn)
+ return g.readLocked(ctx, host, fn)
}
view, err := g.FenceView(ctx, host, c)
if err != nil {
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go
index 474b0221f..986aef08f 100644
--- a/internal/provision/plan14_root_reads_test.go
+++ b/internal/provision/plan14_root_reads_test.go
@@ -615,6 +615,12 @@ func TestPlan14RootReadCensus(t *testing.T) {
t.Fatal(err)
}
seal = "c3351f27f6467fc33b319b36a7e64b75b2d992232f956d9c6243cc49e0c05396"
+ if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 {
+ t.Fatal(errs)
+ }
+ if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-fix2.tsv"), &inv, true); err != nil {
+ t.Fatal(err)
+ }
errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
errs = append(errs, checkRootReadDeclarations(got, inv)...)
if len(errs) > 0 {
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index 350c4c5f4..8ef5e26a8 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -472,11 +472,12 @@ func Reviewed(ctx context.Context, s Store, k string) error {
// BEGIN REVIEWED INVENTORY
var plan9B8CensusInventory = []censusEntry{
- {"internal/host/proofnamed.go", "proofBindingSource.record", "key-site", "key:RecordKey", "2f0e14e715df41e50c7a95714028a57a6a256c4dbd3a4fbeeff32f9e0d263d72"},
+ {"internal/host/rootinventory.go", "ClaimRootHTTP.recoverRootInventory", "key-site", "key:RecordKey", "10d09b5ab6d07d316437233c9a65bf0bcdfe1083db4d0769fa8303b255dfd0ac"},
+ {"internal/host/proofnamed.go", "proofBindingSource.record", "key-site", "key:RecordKey", "10a99ff4b355907831a904010b7019d480ce93f2f595e99e076d3c702d10ca2b"},
{"internal/host/rootrecovery_rebind.go", "type recoveryRebindRequest", "key-site", "key:RecordKey", "b5db3e4dcd4807edea1d053eb1787fbf222be97d3f892e2ab66771457a51dc3d"},
- {"internal/host/rootrecovery_rebind.go", "ClaimRootHTTP.rebindRecoveryInventory", "key-site", "key:RecordKey", "70f47ac54692802662b353edb4d89cdb2617fc154dde6f2b698e488b8356baf1"},
+ {"internal/host/rootrecovery_rebind.go", "ClaimRootHTTP.rebindRecoveryInventory", "key-site", "key:RecordKey", "a782cf4233eeeb7063e39a1c699530a88c96755e8105f508b3e745cfd3508c9a"},
{"internal/host/claimrecovery_remote.go", "claimCrossBoxHandover.installRecoveryRemote", "key-site", "key:RecordKey", "d60c3ab46f7b909fa22c648e883e402345d2652a3c54640c3981b13c5c36e168"},
- {"internal/host/rootinventory.go", "ClaimRootHTTP.commitRootInventoryPending", "key-site", "key:RecordKey", "8035738e2f200bd436be8b06b41c3e1c1cebe1cce2aed1f87cce635da56b2210"},
+ {"internal/host/rootinventory.go", "ClaimRootHTTP.commitRootInventoryPending", "key-site", "key:RecordKey", "3e3d34311cb6685989689ea445aaa969d270e595d90b97b78dab68f21c1aff1e"},
{"internal/host/movesheldrelease_root.go", "movesHeldReleaseRoot.authority", "key-site", "key:RecordKey", "cd50ea0c21a17a1acccfe5735e0ab4147341747ec1a080c6113affc1df940901"},
{"internal/host/rootrecord_claim.go", "rootClaimRecord", "key-site", "key:RecordKey,key:SidecarRecordKey", "e7f73adcaa45e4dfefc5743f43e9ab83567ac95a74ae0557e789478a58e64da4"},
{"internal/hostbox/inventory_expected.go", "StopInventoryDB.WriteExpected", "key-site", "key:RecordKey", "8c2824998dcb510ed54c08a74846352c7918fe1ba2d7ffb3dd05596ca81b329b"},
@@ -486,10 +487,10 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/hostbox/journalservice.go", "journalRequestHash", "key-site", "key:RecordKey", "8c07978658bb060c28ad88477dcbb10d8591ef194cac283cb54eb6a4e833ac76"},
{"internal/hostbox/journalservice.go", "type JournalRequest", "key-site", "key:RecordKey", "14baf4c57a342a8b2dadaa95c965de4650ee352183de3a8c56768f47642ff5ab"},
{"internal/hostbox/journalservice.go", "validateJournalRequest", "key-site", "key:RecordKey", "724be9634772f00bf0d9da2c39c8049763c9aad6f3075c10f31c97ab69c15103"},
- {"internal/host/proofbinding.go", "proofBindingSource.binding", "key-site", "key:RecordKey", "f2563f773a9684e3329752babba671f41a7fcd1c1f81d2ce02b5865aa35dafb4"},
+ {"internal/host/proofbinding.go", "proofBindingSource.binding", "key-site", "key:RecordKey", "455e47321aad1c671f1d0ec468d0389b2cd2386b24f99cc3dd7c9251e5108bee"},
{"internal/host/proofbinding.go", "type proofBinding", "key-site", "key:RecordKey", "418f61bc5a4192d99821b315baeb1a5b4619593303f8a02e9b13ef824003a613"},
{"internal/host/rootroute.go", "proofBindingSource.claimIdentity", "key-site", "key:RecordKey", "ad6e0fc3f2cd360473a5d0a2267d2dc7944b45d80947613f791db8d573f8fee6"},
- {"internal/provision/rootrecord.go", "FindRecordByHost", "key-site", "key:ParseRecordKey,lit:instances", "10c8254cf896be034334d634151d169760a909ac1ba4c1fc334785e86e4b84d4"},
+ {"internal/provision/rootrecord.go", "IndexRecords", "key-site", "key:ParseRecordKey,lit:instances", "2e45f3c659a61943727ac1413d431c8320dbc1d4e89cb4f43df60c819764abf9"},
// Read-only discovery of signed loss records; no record write or lifecycle authority.
{"internal/console/claim_loss_pages.go", "Server.pageClaimLossRecord", "key-site", "key:LossRecordKey", "d21e2450f540edf6645c457eb08c72f470969f5ea4fcbc597157bb7dbeb27287"},
{"internal/hostbox/parkleases.go", "ReplicaParkLeaseSet", "key-site", "lit:instances", "0ca8d9b743402fda9d15cc215464423b35e21c1c2b47f91294d76188b7d43fb6"},
@@ -797,7 +798,7 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/hostevidence/signature.go", "immutable", "nonrecord", "sel:Put", "54fb2bdc230c2cb4193f3b808004b63bcfd092266846fc0c03e61617c3ad0ab5"},
{"internal/hostexec/controller.go", "Controller.run", "nonrecord", "sel:Put", "03df11b375761730700ce30c0db05c57f0828629eda0a5de126ea1a2f2f7501c"},
// D1: conditional execution lease write under control/hosts/<host>/exec/<box>.json; no deployment record key.
- {"internal/hostexec/rootbroker.go", "RootBroker.acquire", "nonrecord", "sel:Put", "2fdbeb00cd5e68d0a93c56229a90c1e334bffc9273c466c8cca83f2de4b2691e"},
+ {"internal/hostexec/rootbroker.go", "RootBroker.acquire", "nonrecord", "sel:Put", "63226d41e26fb28dfc0fd2168386cde8191c02391d07d5d449aa5f963f498147"},
{"internal/hostfence/termination.go", "TerminationWorker.persist", "nonrecord", "sel:Put", "47758114adcc3c493a9d1ead0fa7087ad0d1e49595c891bab4dfb51c9058a8a1"},
{"internal/hostfiles/contract.go", "CopyManifest", "nonrecord", "sel:Put", "9754b113ca2cad5e267402a44ac25de778e332d0c87027767f9a7daa4937d9fd"},
{"internal/hostfiles/publisher.go", "Publisher.put", "nonrecord", "sel:Put", "4bf1d0c4ab995f4e5afa8d7675cbd00446da006a82a659764f8013c011f9efaa"},
diff --git a/internal/provision/rootrecord.go b/internal/provision/rootrecord.go
index 6e7e30e21..918a282e3 100644
--- a/internal/provision/rootrecord.go
+++ b/internal/provision/rootrecord.go
@@ -2,40 +2,62 @@ package provision
import (
"context"
+
"os"
"github.com/carlosframework/platform/internal/release"
)
-func FindRecordByHost(ctx context.Context, s release.Store, host string) (*Record, string, error) {
+type RecordIndex struct {
+ keys map[string][2]string
+}
+
+func IndexRecords(ctx context.Context, s release.Store) (*RecordIndex, error) {
known, partial, err := knownAccounts(ctx, s)
if err != nil {
- return nil, "", err
+ return nil, err
}
if partial {
- return nil, "", os.ErrPermission
+ return nil, ErrRecordIndexRetry
}
keys, partial, err := listAppSubtree(ctx, s, known, "instances/")
if err != nil {
- return nil, "", err
+ return nil, err
}
if partial {
- return nil, "", os.ErrPermission
+ return nil, ErrRecordIndexRetry
}
- var account, app string
+ out := &RecordIndex{keys: make(map[string][2]string)}
for _, key := range keys {
a, p, h, ok := ParseRecordKey(key, known)
- if !ok || h != host {
+ if !ok {
continue
}
- if account != "" {
- return nil, "", os.ErrPermission
+ if _, duplicate := out.keys[h]; duplicate {
+ out.keys[h] = [2]string{}
+ } else {
+ out.keys[h] = [2]string{a, p}
}
- account, app = a, p
}
- if account == "" {
+ return out, nil
+}
+
+type recordIndexRetry struct{}
+
+func (recordIndexRetry) Error() string { return os.ErrDeadlineExceeded.Error() }
+func (recordIndexRetry) Temporary() bool { return true }
+
+var ErrRecordIndexRetry error = recordIndexRetry{}
+
+func (i *RecordIndex) Find(ctx context.Context, s release.Store, host string) (*Record, string, error) {
+ names, ok := i.keys[host]
+ if !ok {
return nil, "", os.ErrNotExist
}
+ account, app := names[0], names[1]
+ if account == "" {
+ return nil, "", os.ErrPermission
+ }
rec, etag, err := GetRecordETag(ctx, s, account, app, host)
if err != nil {
return nil, "", err
@@ -48,3 +70,11 @@ func FindRecordByHost(ctx context.Context, s release.Store, host string) (*Recor
}
return rec, etag, nil
}
+
+func FindRecordByHost(ctx context.Context, s release.Store, host string) (*Record, string, error) {
+ index, err := IndexRecords(ctx, s)
+ if err != nil {
+ return nil, "", err
+ }
+ return index.Find(ctx, s, host)
+}
diff --git a/internal/provision/testdata/plan14-root-reads-fix2.tsv b/internal/provision/testdata/plan14-root-reads-fix2.tsv
new file mode 100644
index 000000000..e3190092e
--- /dev/null
+++ b/internal/provision/testdata/plan14-root-reads-fix2.tsv
@@ -0,0 +1,60 @@
+# Plan 14 Part 1 fix2: controller-required authority changes. Prior freezes remain unchanged.
+internal/host/agent.go runAdoptUnderRootRelease authority B3,C Route restart and recovery errors continue through sweep, proof minting and held-stage work. whole:Route@10418,whole:Route@16956,whole:Route@16992,whole:Route@7846 3f2c4079d973759d3ff6af587a3a2fc6d462d73d986f4fcbdd23dd0cea55c461
+internal/host/claimstop_run.go RunInstanceControl authority C,D,G Refuse premature claim activation and continue serving recovery after per-host startup failures. 7b425e2882811cb5ec42eb14e2eaf424995e928ca0c2a63e1a156c4258e7ddb9
+internal/host/proofbinding.go productionProofBindingSource authority B3 Root-owned current and previous pointers replace health history as executable authority; cache only record discovery. file:Open@209,file:Open@379,file:Read@958 a201908eb70c2769e2e7a2f3bb1e672911128370ab5f546b5cb1f80563361a59
+internal/host/proofbinding.go proofBindingSource.binding authority B3 Verified unit identity and checked current/previous release paths bind running or failed units independently of health history. file:EvalSymlinks@2524,file:EvalSymlinks@2787,file:EvalSymlinks@3206,file:EvalSymlinks@3515,registry:Unit@461,registry:Unit@4718,registry:Unit@544,registry:Unit@717,whole:Route@5198,whole:Route@5205 455e47321aad1c671f1d0ec468d0389b2cd2386b24f99cc3dd7c9251e5108bee
+internal/host/proofnamed.go proofBindingSource.findRecord authority B3 Cache successful per-pass discovery only; retry partial listings and reread selected record ETags. c7d87c42992654bd5d36ce746c18ff7ab324bfbfeb0c66fed68decbabfd3432e
+internal/host/proofnamed.go proofBindingSource.record authority B3 Record key selection uses indexed discovery; local root-managed unit authority is unchanged. 10a99ff4b355907831a904010b7019d480ce93f2f595e99e076d3c702d10ca2b
+internal/host/proofobserve.go productionProofObserver authority B3,B4 Reuse per-pass record discovery; continue independent observation and root health writes. whole:Route@301 8818b1bc139c865263e4b30ad506ae4ef1bb6e19e01af4df69dbdde5e7fa8064
+internal/host/proofroot.go proofRoot.around authority B3,B4 Continue minting observations for unaffected routes even when adoption reports a route error. a63c9ff49cfc8a1ee46439008a0eaaff55dbccd7cde9692f9c9f0fc8a179ae72
+internal/host/rootexecution_runtime.go newClaimExecutionRuntime authority D Positive claim or fence mismatches are terminal; transport failures retain bounded retry eligibility. file:Read@205 8ce18baebc235f8e247018e879b8558855a73d18e6e1c36303ddc439d8fb0968
+internal/host/rootinventory.go ClaimRootHTTP.commitRootInventoryPending authority C Check request age before intent and recover committed inventory using revision, full inventory equality and LastRequest. file:Read@1353,file:Read@3384,registry:Account@4748,registry:Account@4763,registry:App@4784,registry:App@4795,registry:ClaimIncarnation@4254,registry:ClaimOwner@4304,registry:ClaimOwner@5287,registry:ClaimOwnership@4846,registry:ClaimOwnership@4868,registry:ClaimOwnership@5601,registry:ClaimOwnership@7602,registry:ClaimRowID@4468,registry:ClaimRowID@4545,registry:ClaimStamp@4282,registry:DBPath@4812,registry:DBPath@4826,registry:Host@4718,registry:Host@4730,whole:ClaimOwnership@4850,whole:ClaimOwnership@4878,whole:ClaimOwnership@5611,whole:ClaimOwnership@7613,whole:Route@3576,whole:Route@3959,whole:Route@3975,whole:Route@4030,whole:Route@4681,whole:Route@4694,whole:Route@4703,whole:Route@4941,whole:Route@5173,whole:Route@5849,whole:Route@5985,whole:Route@6209,whole:Route@7446,whole:Route@8480,whole:Route@8541 3e3d34311cb6685989689ea445aaa969d270e595d90b97b78dab68f21c1aff1e
+internal/host/rootinventory.go ClaimRootHTTP.recoverRootInventory authority C Retry per host; abandon superseded intents under root lock while retaining durable quarantine. file:Read@1373,file:Read@707 10d09b5ab6d07d316437233c9a65bf0bcdfe1083db4d0769fa8303b255dfd0ac
+internal/host/rootrecovery_rebind.go ClaimRootHTTP.rebindRecoveryInventory authority C Retain fresh claim and inventory checks and refuse a box listed Lost. file:Exec@2331,file:Read@1225,file:Read@1835,registry:Account@3052,registry:App@3089,registry:ClaimIncarnation@3152,registry:ClaimOwner@3332,registry:ClaimRowID@3354,registry:ClaimStamp@3180,registry:Host@3024,whole:Route@3473 a782cf4233eeeb7063e39a1c699530a88c96755e8105f508b3e745cfd3508c9a
+internal/host/rootrestart.go checkedUnitRestart authority B3 Capture root previous binding without history; failed units can reset and restart onto the authorized live build. file:EvalSymlinks@1059,file:EvalSymlinks@1842,registry:Account@1405,registry:Account@2210,registry:App@1437,registry:App@2242,registry:Host@1106,registry:Host@1898,registry:Host@2785,registry:Host@3052,registry:Host@3588,registry:Host@510,registry:Unit@3470,registry:Unit@711,whole:Route@1413,whole:Route@1445,whole:Route@2218,whole:Route@2250,whole:Route@3478,whole:Route@638,whole:Route@719 3a5759fabf75be2f000bdb184d31f82c13672683ab0dcc90b55375106226a609
+internal/host/rootrestart.go productionRootRestartChecks authority B3 One deployment discovery index per resolver/pass; fresh record bodies and root file checks at each use. file:Open@882,whole:Route@1516,whole:Route@1783,whole:Route@715 b0720f80a7c75e94161997706fb5135503163e57e2b113baf631bbd36494d2f6
+internal/host/rootrestart.go restartBoundUnit authority B3 Recheck identity, target digest and readiness; failed units can start and health-write failure cannot roll back a healthy service. file:EvalSymlinks@629,registry:Account@2334,registry:Account@3574,registry:App@2370,registry:App@3614,registry:Host@1614,registry:Host@2236,registry:Host@2728,registry:Host@3345,registry:Host@676,registry:Unit@1097,registry:Unit@1165,registry:Unit@1928,registry:Unit@1996,registry:Unit@2038,registry:Unit@3031,registry:Unit@393,whole:Route@198,whole:Route@2340,whole:Route@2376,whole:Route@316,whole:Route@3582,whole:Route@3622,whole:Route@3832 847d33cf9527f2de215b9e82cfacb006bc4355c08946687bdeedf95768248e6d
+internal/host/rootrestart.go restartOrdinaryUnit authority B3 Root-derived unit name, systemd execution path and trusted unit file preserve ordinary unit restarts without registry authority. 383608711571b3255de0422bbe236e6683af9cb6156d5d26e633e4124af11732
+internal/host/rootrestart.go rollbackUnitRestart authority B3 Capability remains a recovery aid; root previous binding or current failed-unit binding permits an independently checked start. file:Read@942,registry:Host@1234,registry:Unit@1336,registry:Unit@1637,whole:Route@1242,whole:Route@1344,whole:Route@1645 f2688aa7e8aa5f2d313c68cb8465c602bc228061862c579a3d64e618f3eb1afa
+internal/host/rootrestart.go rootRestartRouteWith authority B3 Retain main ordinary unit try-restart naming and error behavior; registry Unit never selects a named unit. whole:Route@363,whole:Route@491,whole:Route@644 07baa6584881ec41287a349f13a896add7ae54c982c610f67b5ca91913a36b36
+internal/host/rootrollback.go captureRollback authority B3 Capture unit and executable authority without a root health-history prerequisite. file:Read@699,registry:Host@336,registry:Host@818,whole:Route@338,whole:Route@820 c915c20438daff70d792a95ef8c60edd37d276b4f5e0fd5b068c7f44a8949e62
+internal/host/rootrollback.go settleRollbackAttempt authority B3 Do not rewrite settled journals; capability cleanup remains idempotent. 374524f9110f0cd9a9ade95a85d87880cdc1878275384a0e9c2b33f533ed0f99
+internal/host/rootrollback_resume.go recoverRootRestarts authority B3 Recover every attempt independently, skip settled writes and remove leftover capabilities. file:Read@519 aa7aede682a8748e427f8c34e2f99dedb3e5bf206a1e465609fa7b1bb433cdbb
+internal/host/rootrollback_resume.go resumeRootRestart authority B3 Finish verification after a successful forward restart interrupted before settlement without restarting it again. file:EvalSymlinks@1238,registry:Account@1331,registry:Account@2037,registry:App@1359,registry:App@2065,registry:Unit@1109,whole:Route@1111,whole:Route@1333,whole:Route@1361,whole:Route@2039,whole:Route@2067 6181fd6e04a3f2be0707eb188168f60a2c0f50a8f2320f06af1dd3df9baeda1a
+internal/host/rootroute.go proofBindingSource.needsRestart authority B3 Named units use authoritative deployment or root-owned local sources; ordinary recordless routes retain derived-unit dispatch. registry:Host@137,registry:Host@352,registry:Unit@209,registry:Unit@685,whole:Route@614,whole:Route@625 461e89f36f05073df34a887ec86fad0e8c6652f69d82e5a916145cda72842398
+internal/hostbox/coldhandover.go ColdHandover.Recover authority C A provably superseded cold admission can abandon its barrier without discarding its receipt. 6e37cd73c5a42d84e464f86d5c040b69250fbabf59de5929b1ac4ceae07438d4
+internal/hostbox/inventory_expected.go StopInventoryDB.SnapshotRequest authority C Expose inventory revision and LastRequest for exact root commit recovery after receipt pruning. 9ca5b0b60dbf3b0c148a0e0eb3ff1df44e0082a8e7a30b5878e9431c9d31871e
+internal/hostexec/rootbroker.go RootBroker.Execute authority D Retry transient authority reads within existing admission slack; bounded replay and permanent mismatches remain enforced. file:Read@3480,file:Read@5063 2dc3cdebdedbe96bb14733baf2219f6f4fda528b86baf8691077e0da0c9faaf9
+internal/hostexec/rootbroker.go RootBroker.Monitor authority D Exponentially back off lost-record settlement retries; active cutoff monitoring remains independent. 2e5be8edbafbf1885833ca7d639fe98546c5ffe3b3cc23f1e5cba7b1cff8e43f
+internal/hostexec/rootbroker.go RootBroker.acquire authority D Retry conditional writes and authority reads within original monotonic cutoff; reconcile uncertain writes by exact bytes. file:Read@808 63226d41e26fb28dfc0fd2168386cde8191c02391d07d5d449aa5f963f498147
+internal/hostexec/rootbroker.go RootBroker.retry authority D Bound transport retries by original monotonic slack and context; never extend a permit. 79c2cf3bbe1cc13525b985aff5a1c21f8ba10d5f58cbdd7eaa15d90487f5305d
+internal/hostexec/rootbroker.go RootBroker.settle authority D On uncertain ETag or delete conflict require both root and remote expiry, then conditionally delete the current ETag. 8151e8fbac0881adac4981eed10dad413780867f31f4c401f14746b06370b3f4
+internal/hostguard/coldhandover.go Guard.AbandonColdHandover authority C Fresh successor claim permits abandonment; retire barrier stamp and retain predecessor receipt. 02759e0dac6a4e0c9f0f8f361f315a188881c94c4962c49f362993057dbfbaf3
+internal/hostguard/guard.go Guard.Activate authority C Pending intent and durable quarantine remain denial conditions. file:Exec@723 05d728a1ef7c9a25245e7782148f5678f90928d4c804d9e83994c591619fadd1
+internal/hostguard/guard.go Guard.With authority C Recheck remote root denial immediately before the protected effect. ff8d2220879047a430940ffad88de5c0319213f33f0ad33c2f58bf7964914aef
+internal/hostguard/guard.go Guard.locked authority C Root host lock precedes bounded legacy-lock wait; failure durably quarantines the existing installation. 12e3484626a1177e0237ba76839eb1ac1b5cdb006f6329a246ec4d23f52a9a86
+internal/hostguard/guard.go Guard.readLocked authority C Read-only access takes only the root host lock. 5084d4441fb431d24703d1a1875a6dd8b3c878f6be3bf5f329d4c5a3e24fcddc
+internal/hostguard/recoveryfence.go Guard.PendingFences authority C Root-only read lock avoids an agent-held legacy lock at startup. cb28fae024ac90e8b0dc61a6bade6900be8515ada5da8a2b0e92b76dc7e92060
+internal/hostguard/retirement.go Guard.WithNoInstallation authority C Use the already-held root host lock for empty-installation authority checks. 4a6f2bd3ed932b35a551f86acaa423f5d3e72d7651afc5531744f64ce84b98b9
+internal/hostguard/root.go Guard.AbandonRootInventory authority C Clear only an exact pending intent proved superseded, retaining quarantine and all prior denials. 189b68eb13d8e9549ad410a46a4c3a285b7c50cfbc49492a7b3efaff0f19036b
+internal/hostguard/root.go Guard.CommitRootInventory authority C Write coherent pending identity; retire old stamp only at commit and clear quarantine only after authorization. 662d753b26b3a45372435e2425231d0ee4a2098c2a89feef484e8c1f797f8b24
+internal/hostguard/root.go RootAuthority.read authority C Read additive root-owned quarantine state together with installation and retained denials. file:Read@175 716dc51839dc602064aeb0ab3e975a49b122d55b4882dbc79b9c85207a5dd11d
+internal/hostguard/rootrebind.go Guard.RebindRootInventory authority C Clear quarantine only on a fresh fully verified committed rebind. 90f9415f47af01755418101139d9be27ee91ce5ff2cc52212ac13dfcd18a69ea
+internal/hostguard/rootview.go RootAuthority.View authority C Root host lock serializes views; quarantine and pending intent deny admission. 05e48dfee1fe85bac1e2876b48eabc20447c61952ee14fce74a281af483aa8f6
+internal/hostguard/rootview.go RootAuthority.viewLocked authority C Read the root view only with the caller already holding the root host lock; avoid nested acquisition. 3adc7fa8c0a52ebd1e4de9e2ec6d7166549b4dc1b5dd43f6c9a37c01508ff7c1
+internal/provision/rootrecord.go FindRecordByHost authority B3,C Discovery delegates to the collision-checking index and still reads the selected record afresh. 7bb3556a3818ab57cb94b5c4104de708afc7c0b6ac4d88e476509903b23793b0
+internal/provision/rootrecord.go IndexRecords authority B3,C Discover hosts once, reject duplicate keys, and classify partial discovery as retry rather than permission refusal. 2e45f3c659a61943727ac1413d431c8320dbc1d4e89cb4f43df60c819764abf9
+internal/provision/rootrecord.go RecordIndex.Find authority B3,C Use cached key identity but read and validate current record body and ETag. 57369bc683b52f238bae08c56eb54fd65b1617c104e93f0787ba7b4260a41f93
+internal/release/adopt.go adoptHost authority B3,E1 Incomplete record discovery is retryable and does not poison the build for other hosts in the pass. registry:App@10271,registry:App@10431,registry:App@11460,registry:App@11917,registry:App@13283,registry:App@17583,registry:App@22862,registry:App@30469,registry:App@3114,registry:App@6423,registry:App@7697,registry:App@7891,registry:Hibernate@31148,registry:Host@10151,registry:Host@10184,registry:Host@10284,registry:Host@10444,registry:Host@10546,registry:Host@10578,registry:Host@10643,registry:Host@11093,registry:Host@11226,registry:Host@11546,registry:Host@12007,registry:Host@12755,registry:Host@13056,registry:Host@13269,registry:Host@13585,registry:Host@17305,registry:Host@17413,registry:Host@17551,registry:Host@18250,registry:Host@18408,registry:Host@18870,registry:Host@18922,registry:Host@19042,registry:Host@19129,registry:Host@20235,registry:Host@20430,registry:Host@2075,registry:Host@21090,registry:Host@21475,registry:Host@21616,registry:Host@2264,registry:Host@22830,registry:Host@22957,registry:Host@2345,registry:Host@24299,registry:Host@24949,registry:Host@28182,registry:Host@28800,registry:Host@29125,registry:Host@29199,registry:Host@29452,registry:Host@30626,registry:Host@30733,registry:Host@30961,registry:Host@31104,registry:Host@3127,registry:Host@31849,registry:Host@31975,registry:Host@32063,registry:Host@4248,registry:Host@5211,registry:Host@5316,registry:Host@6535,registry:Host@6583,registry:Host@6700,registry:Host@6732,registry:Host@7583,registry:Host@7751,registry:Host@7877,registry:Host@7996,registry:Host@8474,registry:Host@8861,registry:Host@9087,registry:Host@9183,registry:Kind@552,registry:Unit@577,whole:Route@10266,whole:Route@10426,whole:Route@11450,whole:Route@11907,whole:Route@13786,whole:Route@14878,whole:Route@15181,whole:Route@17573,whole:Route@19964,whole:Route@2176,whole:Route@22852,whole:Route@27753,whole:Route@3109,whole:Route@3554,whole:Route@6418,whole:Route@9408 01ddb201c381aad5cffc92b215949e6934df31677ecd1e3803c4d248a408448d
+internal/release/sweep.go LivePins authority B3,E2 Pin root previous pointers as well as current pointers so rollback artifacts survive sweeping. file:ReadDir@100,file:ReadDir@499,file:Readlink@652 79dee4be204bc6c3a4a67ece14f3101e9712b940b4662232b07dfcbcd65f158b
+internal/release/versions.go PointLive authority B3,E2 Controller fix2 reclassifies this legacy primitive: durably retain the old root release pointer before swapping current. file:Open@798,file:Readlink@362,file:Stat@121 599ab4719cb0a340395b73a02171249b046738e8addbd7377f67c7f2b16ad749
+internal/rootparticipation/ledger.go Ledger.Acquire authority D Clear pending acquisition on a returned commit failure while preserving durable record-based exclusion. file:Read@345 990f117b72da3fca7705d7aeb80d0c54a79a63132011f8c49fc0c38a3b599cdc
+internal/rootparticipation/production.go ActivationReady authority D,G Refuse claims activation without complete census and registered verifier; Part 3 G remains mandatory. file:Open@61 d2b720a29f82edfa4bf744a3c07bf7726a9335ef1772165a438aa6f76008bef3
+internal/rootparticipation/production.go OpenForUpdate authority D Unactivated absent census does not block update or recovery; retained census remains authoritative. file:Open@56,file:Read@193 08e361dedc72eecb20902435c9c532316a971d223466990ff758ef92b60b0b53
+@coverage internal/host 8afb6c437cb0315157792d28f86645c75663ce15286af1104c891cee31e62bf7
+@coverage internal/hostbox 74453bbc80561f91e84ccc2cb1b31239c0a1fe232b40d3a6892dbe242303ada5
+@coverage internal/hostexec 5c9f06a0d7bc9441f8357409b3b999467fd904287aee20e479255a2f2c81f614
+@coverage internal/hostguard 9847d32d54210b93592e5426c6023a88c4ebbec9543f49611fb407866dfe052d
+@coverage internal/provision 88c7b3001bc9829d9baeaefb67d3c8a20a6a074eab71ab403d653150ca6e3b1f
+@coverage internal/release 83b2be809f0ec1d02e57708db1bb64e9c814393effe06b6ad649999c41683f10
+@coverage internal/rootparticipation 7c88084fa906ffb0e57e23b717ea176bf6f7cabd292151e93acf680e3905bd28
diff --git a/internal/release/adopt.go b/internal/release/adopt.go
index 52c0a762f..a543f803f 100644
--- a/internal/release/adopt.go
+++ b/internal/release/adopt.go
@@ -1620,7 +1620,10 @@ func adoptHost(ctx context.Context, work hostwrite.Work, reg *registry.Registry,
// restore, the summary line, above all the start — must key on
// whether the rollback actually HAPPENED, not on whether one was
// attempted.
- poisoned[ver] = true
+ var temporary interface{ Temporary() bool }
+ if !errors.As(rerr, &temporary) || !temporary.Temporary() {
+ poisoned[ver] = true
+ }
var notUp *UnitNotUpError
isNotUp := errors.As(rerr, &notUp)
rolledBack := false
diff --git a/internal/release/sweep.go b/internal/release/sweep.go
index 5e4bf1a5e..9c740ce8c 100644
--- a/internal/release/sweep.go
+++ b/internal/release/sweep.go
@@ -197,5 +197,18 @@ func LivePins(optDir string) (map[string]bool, error) {
pins[ver] = true
}
}
+ entries, err = os.ReadDir(filepath.Join(optDir, "previous"))
+ if err != nil && !os.IsNotExist(err) {
+ return nil, err
+ }
+ for _, e := range entries {
+ target, err := os.Readlink(filepath.Join(optDir, "previous", e.Name()))
+ if err != nil {
+ return nil, err
+ }
+ if ver := versionOfTarget(optDir, target); ver != "" {
+ pins[ver] = true
+ }
+ }
return pins, nil
}
diff --git a/internal/release/versions.go b/internal/release/versions.go
index 742148727..91068977c 100644
--- a/internal/release/versions.go
+++ b/internal/release/versions.go
@@ -291,6 +291,33 @@ func PointLive(optDir, host, acct, ver, app string) error {
return err
}
link := LiveLink(optDir, host)
+ if old, err := os.Readlink(link); err == nil && old != target {
+ dir := filepath.Join(optDir, "previous")
+ if err := safefile.MkdirAll(dir, 0755); err != nil {
+ return err
+ }
+ previous := filepath.Join(dir, host)
+ pending := previous + ".tmp"
+ _ = os.Remove(pending)
+ if err := os.Symlink(old, pending); err != nil {
+ return err
+ }
+ if err := os.Rename(pending, previous); err != nil {
+ os.Remove(pending)
+ return err
+ }
+ f, err := os.Open(dir)
+ if err != nil {
+ return err
+ }
+ err = f.Sync()
+ f.Close()
+ if err != nil {
+ return err
+ }
+ } else if err != nil && !os.IsNotExist(err) {
+ return err
+ }
tmp := link + ".tmp"
_ = os.Remove(tmp) // a leftover temp link from a crashed swap
if err := os.Symlink(target, tmp); err != nil {
diff --git a/internal/rootparticipation/ledger.go b/internal/rootparticipation/ledger.go
index 10bbb0e02..36a6cb222 100644
--- a/internal/rootparticipation/ledger.go
+++ b/internal/rootparticipation/ledger.go
@@ -93,11 +93,9 @@ func (l Ledger) Acquire(ctx context.Context, commit func() error) error {
if e = l.writeCensus(c); e != nil {
return e
}
- if e = commit(); e != nil {
- return e
- }
+ e = commit()
c.Pending = ""
- return l.writeCensus(c)
+ return errors.Join(e, l.writeCensus(c))
}
func (l Ledger) idle(ctx context.Context, c Census) error {
if l.VerifyLaunches == nil {
diff --git a/internal/rootparticipation/plan14_fix2_test.go b/internal/rootparticipation/plan14_fix2_test.go
new file mode 100644
index 000000000..acbe08c8f
--- /dev/null
+++ b/internal/rootparticipation/plan14_fix2_test.go
@@ -0,0 +1,21 @@
+package rootparticipation
+
+import (
+ "errors"
+ "testing"
+)
+
+func TestPlan14Fix2AcquisitionCommitFailureClearsPending(t *testing.T) {
+ l := censusFixture(t)
+ failure := errors.New("commit failed")
+ if err := l.Acquire(t.Context(), func() error { return failure }); !errors.Is(err, failure) {
+ t.Fatal(err)
+ }
+ c, err := l.Census()
+ if err != nil || c.Pending != "" {
+ t.Fatalf("pending acquisition survived failed commit: %+v %v", c, err)
+ }
+ if err := l.BeginUpdate(t.Context(), "next"); err != nil {
+ t.Fatal("failed acquisition wedged update", err)
+ }
+}
diff --git a/internal/rootparticipation/production.go b/internal/rootparticipation/production.go
index 7932f52e2..810d55942 100644
--- a/internal/rootparticipation/production.go
+++ b/internal/rootparticipation/production.go
@@ -67,12 +67,17 @@ func (l Ledger) UpdateRequired() (bool, error) {
func OpenForUpdate() (Ledger, bool, error) {
l, err := Open()
- if errors.Is(err, os.ErrNotExist) && !hostwrite.Enabled(os.Getenv) {
+ if errors.Is(err, os.ErrNotExist) {
return Ledger{}, false, nil
}
if err != nil {
return Ledger{}, false, err
}
+ _, err = l.State.Read("census.json")
+ if errors.Is(err, os.ErrNotExist) {
+ l.State.Close()
+ return Ledger{}, false, nil
+ }
required, err := l.UpdateRequired()
if err != nil {
l.State.Close()
@@ -80,3 +85,19 @@ func OpenForUpdate() (Ledger, bool, error) {
}
return l, required, nil
}
+
+func ActivationReady(ctx context.Context) error {
+ l, err := Open()
+ if err != nil {
+ return err
+ }
+ defer l.State.Close()
+ c, err := l.Census()
+ if err != nil {
+ return err
+ }
+ if l.VerifyLaunches == nil {
+ return ErrUnavailable
+ }
+ return l.VerifyLaunches(ctx, c.LaunchRevision)
+}