amadan
host: the root pass records healthy live builds before release.Pass, mints after (H4)
On the first pass after a requirement was switched on, release.Pass could adopt a proven head over a build that had run well for hours before proofRoot looked at it, so that build never entered the healthy history and a rollback to it would wait for a dev proof. proofRoot is split into recordLive (observe and record, before the pass) and mintAndLog (mint, marker, hourly line, after it); around runs them about the pass. The second half reuses the first half's observation when the live link and the unit's process are unchanged, and observes again otherwise.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+256
−22
internal/host/agent.go+13 −9internal/host/proofobserve.go+5 −1internal/host/proofroot.go+103 −12internal/host/proofroot_test.go+135 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go| index be5a2eec4..d06d80ed4 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -3873,14 +3873,13 @@ func RunAdopt(args []string) { |
| if !*stageOnly { |
| passOpts = append(passOpts, release.NagExpiredCanaries(os.Stdout)) |
| } |
| - if err := release.Pass(ctx, reg, store, pub, release.OptDir(), restart, passOpts...); err != nil { |
| - log.Fatalf("adopt: %v", err) |
| - } |
| - |
| - // D3 (proofroot.go): record live builds as healthy, mint on a proving |
| - // box, and on the hourly pass say when a requirement is off. After the |
| - // pass, so it looks at what this pass left live. |
| - proofRoot{ |
| + // D3 (proofroot.go) wraps the pass. Before it, record every named-unit |
| + // route's live build that has run well as healthy, so the build live when |
| + // a requirement is switched on is in the history before this pass can |
| + // adopt a proven head over it (final review H4). After it, mint on a |
| + // proving box and, on the hourly pass, say when a requirement is off; |
| + // a route the pass changed is looked at again first. |
| + pr := proofRoot{ |
| obs: productionProofObserver(release.OptDir()), |
| reg: reg, |
| store: store, |
| @@ -3889,7 +3888,12 @@ func RunAdopt(args []string) { |
| readKey: readProvingKey, |
| hourly: !*stageOnly, |
| logf: log.Printf, |
| - }.run(ctx) |
| + } |
| + if err := pr.around(ctx, func() error { |
| + return release.Pass(ctx, reg, store, pub, release.OptDir(), restart, passOpts...) |
| + }); err != nil { |
| + log.Fatalf("adopt: %v", err) |
| + } |
| // Step 4: the advisories. The two capability advisories that fall out of |
| // READING manifests (declared-but-ungranted, and capabilities on an |
diff --git a/internal/host/proofobserve.go b/internal/host/proofobserve.go| index b190f9f12..715233b2d 100644 |
| --- a/internal/host/proofobserve.go |
| +++ b/internal/host/proofobserve.go |
| @@ -27,6 +27,9 @@ type liveBuild struct { |
| SHA256 string |
| Version string |
| ActiveSince time.Time |
| + // The process observed, so a later look can tell it is the same one. |
| + invocation string |
| + pid int |
| } |
| type proofObserver struct { |
| @@ -85,7 +88,8 @@ func (o proofObserver) observe(rt registry.Route, after time.Duration) (liveBuil |
| return liveBuild{}, fmt.Errorf("%s restarted during the check", rt.Unit) |
| } |
| ver, _ := release.LiveVersion(o.optDir, rt.Host) |
| - return liveBuild{SHA256: exe, Version: ver, ActiveSince: before.ActiveEnterTimestamp}, nil |
| + return liveBuild{SHA256: exe, Version: ver, ActiveSince: before.ActiveEnterTimestamp, |
| + invocation: before.InvocationID, pid: before.MainPID}, nil |
| } |
| // recordSettled is checkedUnitRestart's settled hook on the root pass: hash |
diff --git a/internal/host/proofroot.go b/internal/host/proofroot.go| index b0b72c1ab..5f2c1fb1e 100644 |
| --- a/internal/host/proofroot.go |
| +++ b/internal/host/proofroot.go |
| @@ -1,13 +1,18 @@ |
| // The root pass's D3 work (spec 2026-10-06-console-release-defences-design.md |
| -// §4.3, §4.5): after release.Pass, look at every named-unit route's live build. |
| +// §4.3, §4.5): look at every named-unit route's live build, once before |
| +// release.Pass and again, only where the pass changed it, after. |
| // |
| // - A build meeting observe's conditions (bound, active in one invocation |
| // for the policy's after, hashed from its running executable, ready) |
| // enters the app's healthy history, on every box, policy file or not. |
| // That is what puts the build already live when a requirement is switched |
| -// on into the history without a restart. |
| -// - On a proving box (a prove block lists the app) the same build is |
| -// minted a proof, unless a refusal record names it or the bucket already |
| +// on into the history without a restart. recordLive does it BEFORE |
| +// release.Pass (final review H4): on the first pass after a requirement |
| +// is switched on, the pass can adopt a proven head and replace the build |
| +// that has been running well for hours, and that build must already be in |
| +// the history, or a later rollback to it would wait for a dev proof. |
| +// - On a proving box (a prove block lists the app) the build the pass left |
| +// live is minted a proof (mintAndLog, after release.Pass), unless a refusal record names it or the bucket already |
| // has one. Minting needs the root-only key, which is why it is here. |
| // - A pass that reads a require entry leaves a proof-required/<app> marker |
| // in the bake directory. On the hourly pass, an app with that marker, |
| @@ -46,11 +51,98 @@ type proofRoot struct { |
| logf func(format string, a ...any) |
| } |
| +// liveObservation is one look at a route's live build: the live link's |
| +// target when it was taken, and observe's answer. |
| +type liveObservation struct { |
| + target string |
| + build liveBuild |
| + err error |
| +} |
| + |
| +// proofObservations is recordLive's result, handed to mintAndLog so a route |
| +// the pass did not touch is observed once per pass, not twice. |
| +type proofObservations struct { |
| + pol *release.ProofPolicy |
| + perr error |
| + byHost map[string]liveObservation |
| +} |
| + |
| +// run is both halves back to back, for a caller with no pass between them. |
| func (r proofRoot) run(ctx context.Context) { |
| + r.mintAndLog(ctx, r.recordLive(ctx)) |
| +} |
| + |
| +// around is RunAdopt's order: record the live builds, run the pass, then |
| +// mint on what the pass left live. A pass error is returned without minting. |
| +func (r proofRoot) around(ctx context.Context, pass func() error) error { |
| + obs := r.recordLive(ctx) |
| + if err := pass(); err != nil { |
| + return err |
| + } |
| + r.mintAndLog(ctx, obs) |
| + return nil |
| +} |
| + |
| +// recordLive reads the policy, observes every named-unit route's live build, |
| +// and records each one that qualifies as healthy. |
| +func (r proofRoot) recordLive(ctx context.Context) proofObservations { |
| pol, perr := release.LoadProofPolicy(r.policyPath) |
| if perr != nil { |
| r.logf("proof: %v; nothing is minted until it parses", perr) |
| } |
| + obs := proofObservations{pol: pol, perr: perr, byHost: map[string]liveObservation{}} |
| + routes, err := r.reg.List() |
| + if err != nil { |
| + r.logf("proof: list routes: %v", err) |
| + return obs |
| + } |
| + for _, rt := range routes { |
| + if !release.Adoptable(rt) || rt.Unit == "" { |
| + continue |
| + } |
| + obs.byHost[rt.Host] = r.observeAndRecord(rt, pol.ProveAfter()) |
| + } |
| + return obs |
| +} |
| + |
| +// observeAndRecord observes rt's live build and, when it qualifies, adds it |
| +// to the app's healthy history. |
| +func (r proofRoot) observeAndRecord(rt registry.Route, after time.Duration) liveObservation { |
| + target, _ := os.Readlink(release.LiveLink(r.obs.optDir, rt.Host)) |
| + b, err := r.obs.observe(rt, after) |
| + o := liveObservation{target: target, build: b, err: err} |
| + if err != nil { |
| + return o |
| + } |
| + if err := release.RecordHealthy(r.bakeDir, release.RouteAccount(rt), rt.App, release.HealthyBuild{ |
| + SHA256: b.SHA256, Version: b.Version, How: "live", At: r.obs.now().UTC()}); err != nil { |
| + r.logf("proof: %s: recording the healthy build: %v", rt.Host, err) |
| + } |
| + return o |
| +} |
| + |
| +// stillLive reports whether o still describes rt's live build after the |
| +// pass: the live link names the same target and, for a build that passed, |
| +// the unit is the same process (a rollback can put the link back where it |
| +// was and restart the unit; that process has not run for after). |
| +func (r proofRoot) stillLive(rt registry.Route, o liveObservation) bool { |
| + target, err := os.Readlink(release.LiveLink(r.obs.optDir, rt.Host)) |
| + if err != nil || target != o.target { |
| + return false |
| + } |
| + if o.err != nil { |
| + return true |
| + } |
| + p, err := r.obs.sys.Show(rt.Unit) |
| + return err == nil && p.ActiveState == "active" && p.InvocationID == o.build.invocation && p.MainPID == o.build.pid |
| +} |
| + |
| +// mintAndLog is the second half: mint on a proving box, leave the |
| +// requirement marker, and on the hourly pass say when a requirement is off. |
| +// A route whose live build the pass changed is observed again (and recorded, |
| +// if it qualifies); every other route reuses recordLive's observation. |
| +func (r proofRoot) mintAndLog(ctx context.Context, obs proofObservations) { |
| + pol, perr := obs.pol, obs.perr |
| routes, err := r.reg.List() |
| if err != nil { |
| r.logf("proof: list routes: %v", err) |
| @@ -65,19 +157,18 @@ func (r proofRoot) run(ctx context.Context) { |
| } |
| acct := release.RouteAccount(rt) |
| seen[acctApp{acct, rt.App}] = true |
| - b, err := r.obs.observe(rt, after) |
| - if err != nil { |
| + o, ok := obs.byHost[rt.Host] |
| + if !ok || !r.stillLive(rt, o) { |
| + o = r.observeAndRecord(rt, after) |
| + } |
| + if o.err != nil { |
| if pol.Proves(rt.App) { |
| - r.logf("proof: %s: not minting yet: %v", rt.Host, err) |
| + r.logf("proof: %s: not minting yet: %v", rt.Host, o.err) |
| } |
| continue |
| } |
| - if err := release.RecordHealthy(r.bakeDir, acct, rt.App, release.HealthyBuild{ |
| - SHA256: b.SHA256, Version: b.Version, How: "live", At: r.obs.now().UTC()}); err != nil { |
| - r.logf("proof: %s: recording the healthy build: %v", rt.Host, err) |
| - } |
| if perr == nil && pol.Proves(rt.App) { |
| - r.mint(ctx, pol, rt, acct, b) |
| + r.mint(ctx, pol, rt, acct, o.build) |
| } |
| } |
| if perr != nil { |
diff --git a/internal/host/proofroot_test.go b/internal/host/proofroot_test.go| index 6557a29d5..03826398e 100644 |
| --- a/internal/host/proofroot_test.go |
| +++ b/internal/host/proofroot_test.go |
| @@ -3,6 +3,8 @@ package host |
| import ( |
| "context" |
| "crypto/ed25519" |
| + "crypto/sha256" |
| + "encoding/hex" |
| "fmt" |
| "os" |
| "path/filepath" |
| @@ -303,3 +305,136 @@ func TestProofRootRequireEntryLeavesTheMarker(t *testing.T) { |
| t.Fatalf("logs = %v, want the hourly requirement-off line once the file is gone", e.logs) |
| } |
| } |
| + |
| +// Final review H4: the first adoption after a requirement is switched on. |
| +// A has run well for two hours with an empty healthy history; B is promoted |
| +// with a valid dev proof. One RunAdopt-shaped run (proofRoot.around a real |
| +// release.Pass) adopts B, and A, the build B replaced, is in the history |
| +// too, so a later rollback to A does not wait for a dev proof. B enters it |
| +// through the settle hook. |
| +func TestProofRootRecordsTheReplacedBuildBeforeAdoption(t *testing.T) { |
| + ctx := context.Background() |
| + opt, bake, etc := t.TempDir(), t.TempDir(), t.TempDir() |
| + s := release.NewDirStore(t.TempDir()) |
| + plat := release.AdoptPlatform() |
| + pub, priv, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for ver, body := range map[string]string{"a": "build a", "b": "build b"} { |
| + m := release.Manifest{App: "console", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}} |
| + if err := release.Ship(ctx, s, "bab", m, map[string][]byte{plat: []byte(body)}); err != nil { |
| + t.Fatalf("Ship %s: %v", ver, err) |
| + } |
| + } |
| + if err := release.StageArtifact(opt, "bab", "a", "console", []byte("build a")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PointLive(opt, observedHost, "bab", "a", "console"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.Promote(ctx, s, priv, "bab", "console", "canary/console", "b"); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg := restartTestRegistry(t) |
| + rt := registry.Route{Host: observedHost, Kind: "instance", App: "console", |
| + Channel: "canary/console", Unit: "console.service", Status: "ready"} |
| + if err := reg.Upsert(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + shaA := sha256Hex("build a") |
| + shaB := sha256Hex("build b") |
| + |
| + seed, proverPub, err := release.GenerateKey() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + proverKey, err := release.ParseSigningKey(seed, "test") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) |
| + p, err := release.SignProof(proverKey, release.Proof{App: "console", Platform: plat, SHA256: shaB, |
| + Version: "b", Prover: "dev", ActiveSince: now.Add(-time.Hour), ProvenAt: now}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PutProof(ctx, s, "bab", p); err != nil { |
| + t.Fatal(err) |
| + } |
| + policyPath := filepath.Join(etc, "proof-policy.json") |
| + if err := os.WriteFile(policyPath, []byte(fmt.Sprintf( |
| + `{"schema":1,"require":[{"app":"console","prover":"dev","key":%q}]}`, proverPub)), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + link := release.LiveLink(opt, observedHost) |
| + units := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "a1", MainPID: 7, |
| + ExecPath: link, ActiveEnterTimestamp: now.Add(-2 * time.Hour)}}} |
| + procExe := func(int) string { return link } // the process runs whatever the link names |
| + clock := func() time.Time { return now } |
| + root := proofRoot{ |
| + obs: proofObserver{sys: units, procExe: procExe, optDir: opt, now: clock, |
| + ready: func(registry.Route) func() error { return nil }}, |
| + reg: reg, store: s, bakeDir: bake, policyPath: policyPath, readKey: readProvingKey, |
| + logf: func(f string, a ...any) { t.Logf(f, a...) }, |
| + } |
| + // The restart starts B's process and settles, as checkedUnitRestart |
| + // would, and its settled hook records what it runs. |
| + restart := func(host string) error { |
| + units.shows, units.n = []unitProps{{ActiveState: "active", InvocationID: "b1", MainPID: 8, |
| + ExecPath: link, ActiveEnterTimestamp: now}}, 0 |
| + recordSettled(units, procExe, opt, bake, clock)(rt) |
| + return nil |
| + } |
| + if err := root.around(ctx, func() error { |
| + return release.Pass(ctx, reg, s, pub, opt, restart, release.BakeDir(bake), |
| + release.WithProofGate(release.NewProofGate(policyPath, bake))) |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if v, _ := release.LiveVersion(opt, observedHost); v != "b" { |
| + t.Fatalf("live = %q, want b adopted on its proof", v) |
| + } |
| + got := map[string]string{} |
| + for _, h := range release.ReadHealthy(bake, "bab", "console") { |
| + got[h.SHA256] = h.How |
| + } |
| + if got[shaA] != "live" { |
| + t.Fatalf("history = %+v, want a (sha256 %s) recorded live before the pass replaced it", got, shaA) |
| + } |
| + if got[shaB] != "settle" { |
| + t.Fatalf("history = %+v, want b (sha256 %s) recorded by the settle hook", got, shaB) |
| + } |
| +} |
| + |
| +// A route the pass did not touch is observed once: the second half reuses |
| +// the first half's look when the link and the process are unchanged, and |
| +// looks again when the process was restarted under the same link. |
| +func TestProofRootReusesAnUnchangedObservation(t *testing.T) { |
| + e := newProofRootEnv(t, provePolicy) |
| + units := e.root.obs.sys.(*fakeUnits) |
| + obs := e.root.recordLive(context.Background()) |
| + if o := obs.byHost[observedHost]; o.err != nil { |
| + t.Fatalf("first half: %v", o.err) |
| + } |
| + rt := registry.Route{Host: observedHost, App: "console", Unit: "console.service"} |
| + if !e.root.stillLive(rt, obs.byHost[observedHost]) { |
| + t.Fatal("an untouched route is not still live") |
| + } |
| + units.shows = []unitProps{{ActiveState: "active", InvocationID: "i2", MainPID: 9, |
| + ExecPath: units.shows[0].ExecPath, ActiveEnterTimestamp: e.now}} |
| + units.n = 0 |
| + if e.root.stillLive(rt, obs.byHost[observedHost]) { |
| + t.Fatal("a restarted process under the same link counts as still live") |
| + } |
| + e.root.mintAndLog(context.Background(), obs) |
| + if e.proof(t) != nil { |
| + t.Fatal("minted from the first half's look at a process that has since restarted") |
| + } |
| +} |
| + |
| +func sha256Hex(s string) string { |
| + h := sha256.Sum256([]byte(s)) |
| + return hex.EncodeToString(h[:]) |
| +} |