amadan

carlos / platform Public

release: adoptHost requires a dev proof for listed apps, checked before staging and on the staged bytes

D3's ProofGate (plan 3 task 4): admit before ensureStaged with a 30 s miss cache and a waiting list, bind on the staged bytes before PointLive. A staged-bytes mismatch is a RefusalNotProven record (recovered, still on the old build) with its own gate line, alert and status row; the wait is cleared on every early return that stops the gate being asked (ruling 5); any policy load error holds every named-unit route.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 8c0d7201385fcfefa5c78854935b3798904a0f67 parent 15779f7
8 files changed, +962 −4
  • cli.go +4 −0
  • internal/host/alertsink.go +16 −0
  • internal/host/refusalwatch_test.go +37 −0
  • internal/release/adopt.go +46 −2
  • internal/release/proofgate.go +336 −0
  • internal/release/proofgate_test.go +494 −0
  • internal/release/refusal.go +6 −2
  • statuslive_test.go +23 −0
diff --git a/cli.go b/cli.go
index ffeeab224..fb212becf 100644
--- a/cli.go
+++ b/cli.go
@@ -3460,6 +3460,10 @@ func printLiveBoxes(w io.Writer, boxes []status.Box, now time.Time) {
fmt.Fprintf(w, " refused %s %s since %s: wrong program (%s)\n", r.Host, r.Version, r.RefusedAt.UTC().Format(time.RFC3339), strings.TrimPrefix(r.Reason, "wrong program: "))
continue
}
+ if r.Kind == release.RefusalNotProven {
+ fmt.Fprintf(w, " refused %s %s since %s: staged bytes do not match dev's proof (%s)\n", r.Host, r.Version, r.RefusedAt.UTC().Format(time.RFC3339), strings.TrimPrefix(r.Reason, "not proven: "))
+ continue
+ }
failed := ""
if r.RecoveryError != "" {
failed = " (rollback failed: " + r.RecoveryError + ")"
diff --git a/internal/host/alertsink.go b/internal/host/alertsink.go
index f3fde47ad..6000c57aa 100644
--- a/internal/host/alertsink.go
+++ b/internal/host/alertsink.go
@@ -174,6 +174,22 @@ func formatReleaseAlert(r release.Refusal) (subject, msg string) {
r.Host, r.Version, r.RefusedAt.Format(time.RFC3339), r.Reason)
return subject, msg
}
+ if r.Kind == release.RefusalNotProven {
+ // D3's binding check: the same shape as the wrong program. Nothing
+ // started and nothing was rolled back, and "--retry" would re-sign
+ // the pointer over the same staged bytes.
+ still := r.RestoredVersion
+ if still == "" {
+ still = "nothing"
+ }
+ subject = fmt.Sprintf("%s: %s refused, staged bytes do not match dev's proof (still on %s)", r.Host, r.Version, still)
+ if len(subject) > 100 { // SNS subject limit
+ subject = subject[:100]
+ }
+ msg = fmt.Sprintf("%s refused %s at %s: %s\n\nThe staged bytes do not match dev's proof, so %s was never pointed live. The box is still on %s. Roll the exact bytes dev proved.",
+ r.Host, r.Version, r.RefusedAt.Format(time.RFC3339), strings.TrimPrefix(r.Reason, "not proven: "), r.Version, still)
+ return subject, msg
+ }
if r.Recovered {
subject = fmt.Sprintf("%s: %s did not start, rolled back to %s", r.Host, r.Version, r.RestoredVersion)
} else {
diff --git a/internal/host/refusalwatch_test.go b/internal/host/refusalwatch_test.go
index 419712022..8aa25967a 100644
--- a/internal/host/refusalwatch_test.go
+++ b/internal/host/refusalwatch_test.go
@@ -122,3 +122,40 @@ func TestRefusalWatchReportsAWrongProgram(t *testing.T) {
t.Fatalf("rows = %+v", rows)
}
}
+
+// D3's staged-bytes refusal is not a rollback either (plan 3 ruling 3): the
+// box never left its old build. The alert says the staged bytes do not match
+// dev's proof, names the build the box is still on, and asks for the exact
+// bytes dev proved; never --retry, never a rollback.
+func TestRefusalWatchReportsANotProven(t *testing.T) {
+ bake := t.TempDir()
+ now := time.Date(2026, 10, 6, 9, 31, 0, 0, time.UTC)
+ rec := release.Refusal{Host: "console.example.com", App: "console", Version: "v2", RefusedAt: now.Add(-time.Minute),
+ Reason: "not proven: staged sha256 aaaa, dev proved bbbb", Kind: release.RefusalNotProven, Recovered: true, RestoredVersion: "v1"}
+ if err := release.WriteRefusal(bake, rec); err != nil {
+ t.Fatal(err)
+ }
+ subject, msg := formatReleaseAlert(rec)
+ if subject != "console.example.com: v2 refused, staged bytes do not match dev's proof (still on v1)" {
+ t.Fatalf("subject = %q", subject)
+ }
+ for _, want := range []string{"staged sha256 aaaa, dev proved bbbb", "still on v1", "Roll the exact bytes dev proved"} {
+ if !strings.Contains(msg, want) {
+ t.Fatalf("message = %q, want %q in it", msg, want)
+ }
+ }
+ for _, never := range []string{"--retry", "rolled back", "rollback"} {
+ if strings.Contains(msg, never) || strings.Contains(subject, never) {
+ t.Fatalf("subject %q, message %q: %q has no place in a not-proven alert", subject, msg, never)
+ }
+ }
+ var posts []string
+ sinks := newAlertSinks("https://hook.example", func(_ string, b []byte) error { posts = append(posts, string(b)); return nil }, "", nil, &alertCounters{})
+ rows := (&refusalWatch{}).tick(context.Background(), bake, sinks, now)
+ if len(posts) != 1 || !strings.Contains(posts[0], `"refusal_kind":"not-proven"`) {
+ t.Fatalf("posts = %v", posts)
+ }
+ if len(rows) != 1 || rows[0].Kind != release.RefusalNotProven || rows[0].RecoveryError != "" {
+ t.Fatalf("rows = %+v", rows)
+ }
+}
diff --git a/internal/release/adopt.go b/internal/release/adopt.go
index 17f858825..09fa8461b 100644
--- a/internal/release/adopt.go
+++ b/internal/release/adopt.go
@@ -72,6 +72,10 @@ type passConfig struct {
// running (reset-failed + restart), because a deadline failure can leave
// the rejected build running unready and `start` is a no-op on it.
rollbackRestart func(host string) error
+
+ // proof is D3's gate (WithProofGate, proofgate.go). nil, the default for
+ // every existing caller and test, means no proof is read or required.
+ proof *ProofGate
}
// NagExpiredCanaries opts a Pass/PassFilter call into spec §5's nag: one
@@ -259,6 +263,14 @@ func WithRollbackRestart(fn func(host string) error) PassOption {
return func(c *passConfig) { c.rollbackRestart = fn }
}
+// WithProofGate arms D3 (proofgate.go): a build of an app the policy file
+// lists is adopted only with a verified proof for its bytes, or from this
+// box's healthy history. Pass the SAME gate to every pass of a process, so
+// its 30 s miss cache and its waiting list survive between ticks.
+func WithProofGate(g *ProofGate) PassOption {
+ return func(c *passConfig) { c.proof = g }
+}
+
// stageBlocked reports whether err is the version store refusing a write to
// THIS uid, rather than any of the many other ways staging fails. errors.Is
// unwraps the *os.PathError that mkdir/openat return.
@@ -317,6 +329,9 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551
for _, opt := range opts {
opt(&cfg)
}
+ // D3: re-read the proof policy at the start of every pass, so an edit
+ // reaches the agent and root alike without a restart (spec §4.5).
+ cfg.proof.BeginPass()
routes, err := reg.List()
if err != nil {
return err
@@ -957,6 +972,7 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
backfillManifest(ctx, s, optDir, RouteAccount(rt), rt.App, cur, rt.Host)
}
if cur == ver {
+ cfg.proof.clear(rt.Host) // live on its head: nothing left to wait for
// A refusal that no longer blocks the pointer is stale even though
// nothing moves: v2 was refused and rolled back to v1, then the
// channel went back to v1. Without this every later pass returns
@@ -979,12 +995,20 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
// deliberately unratcheted (pipelines spec §3 amendment: ratcheting it
// would make it wait out the very window it exists to skip). Either one
// alone releases the head.
+ //
+ // Each return from here to admit stops D3's gate being asked about this
+ // head, so each one also takes the host off the proof waiting list
+ // (cfg.proof.clear). A wait left behind for a head the channel has moved
+ // off would stay due for ever, and the fast loop would sweep the host
+ // every tick for the whole hold (plan 3 ruling 5).
if hold := Hold(bake, RouteAccount(rt), rt.App, channel, ver, holds, ignore || ptr.FastTrack, now); hold != "" {
fmt.Printf(" %s: %s (%s) is %s\n", rt.Host, ver, channel, hold)
+ cfg.proof.clear(rt.Host)
return
}
if poisoned[ver] {
fmt.Printf(" %s: skipping %s (failed on another host this pass)\n", rt.Host, ver)
+ cfg.proof.clear(rt.Host)
return
}
// D1's refusal (refusal.go): this box rolled this exact promotion back
@@ -994,19 +1018,33 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
if r, err := ReadRefusal(cfg.bakeDir, rt.Host); err != nil {
fmt.Printf(" %s: reading refusal record: %v\n", rt.Host, err)
} else if r.Blocks(ptr) {
- if r.Kind == RefusalWrongProgram {
+ switch r.Kind {
+ case RefusalWrongProgram:
// --retry would re-stage the same bytes and be refused again,
// and re-shipping this version cannot help: its manifest is
// immutable, so the consent has to ride on a new version.
fmt.Printf(" %s: refused: %s is the wrong program for this route (%s); still on %s; ship a new version with carlos ship --program-change and promote it, if the change is intended\n",
rt.Host, ver, r.Reason, r.RestoredVersion)
- } else {
+ case RefusalNotProven:
+ // D3's binding check (proofgate.go bind). Nothing was started
+ // or rolled back, and --retry would re-sign the pointer over
+ // the same staged bytes and be refused again.
+ fmt.Printf(" %s: refused: %s's staged bytes do not match dev's proof (%s); still on %s; roll the exact bytes dev proved\n",
+ rt.Host, ver, strings.TrimPrefix(r.Reason, "not proven: "), orNothing(r.RestoredVersion))
+ default:
fmt.Printf(" %s: refused: %s did not come up on this box at %s (%s); retry with: carlos promote --retry %s %s\n",
rt.Host, ver, r.RefusedAt.Format(time.RFC3339), r.Reason, ver, channel)
}
+ cfg.proof.clear(rt.Host)
return
}
}
+ // D3's cheap gate (proofgate.go, spec §4.5 check 1): before ensureStaged,
+ // so an unproven build is neither downloaded nor relayed to root.
+ verdict, admitted := cfg.proof.admit(ctx, s, optDir, rt, ptr, cur)
+ if !admitted {
+ return
+ }
m, err := ensureStaged(ctx, s, optDir, RouteAccount(rt), rt.App, ver, rt.Host)
if err != nil {
fmt.Printf(" %s: %v\n", rt.Host, err)
@@ -1061,6 +1099,12 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri
}
}
+ // D3's binding check (spec §4.5 check 2): the staged bytes themselves
+ // must be the proven ones, or a build this box has run well.
+ if !cfg.proof.bind(optDir, rt, ptr, verdict, cur, now) {
+ return
+ }
+
// The capability converge for the NEW version, BEFORE PointLive (spec
// §4d). m is the manifest ensureStaged has in hand — never a re-read of
// the staged sibling, whose write is best-effort: a sibling that failed
diff --git a/internal/release/proofgate.go b/internal/release/proofgate.go
new file mode 100644
index 000000000..803415ffb
--- /dev/null
+++ b/internal/release/proofgate.go
@@ -0,0 +1,336 @@
+// The dev-proof gate, the requiring half of D3 (spec
+// 2026-10-06-console-release-defences-design.md §4.5).
+//
+// adoptHost asks it twice. admit runs before ensureStaged, so an unproven
+// build is neither downloaded nor relayed to a root pass: it takes the
+// manifest's sha256 for this platform and admits the build when that sha256
+// is in this box's healthy history, or a configured prover's proof for it
+// verifies. bind runs after ensureStaged and before PointLive and hashes the
+// staged file, because a manifest can name bytes the box is not running (a
+// reused version name, a stale cache): the staged hash must be the proof's or
+// in the healthy history, or the build is refused with a refusal record of
+// kind RefusalNotProven.
+//
+// A missing proof is not a refusal. The box keeps looking, so the build adopts
+// soon after its proof arrives. A miss is cached per (route, sha256) for
+// ProofMissTTL, and the gate keeps a waiting list that the agent's fast loop
+// (fastMembers) and `carlos status --live` read. A host leaves the list as
+// soon as the gate stops being asked about it (adoptHost clears it on every
+// early return before admit), so the list never holds a head the channel has
+// moved off.
+//
+// One gate lives for the life of its process and is shared by every pass in
+// it. BeginPass re-reads the policy file, so an edit reaches the agent and
+// root alike without a restart. An absent file is D3 off: nothing is read or
+// required. Any other failure to load it (it does not parse, or cannot be
+// read) is D3 on and waiting for every named-unit route, because the box
+// cannot tell which apps were listed.
+package release
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+const (
+ ProofMissTTL = 30 * time.Second
+ // proofPolicyLogEvery bounds how often a broken policy file is printed
+ // by a long-lived process. A root pass is a fresh process and prints it
+ // every pass; the agent's 2-second loop prints it once a minute.
+ proofPolicyLogEvery = time.Minute
+)
+
+// ProofWait is a host held on its current build because its channel head is
+// not proven (or the policy file cannot be loaded).
+type ProofWait struct {
+ Host, App, Version, SHA256, Prover, Reason string
+ Since time.Time
+}
+
+type promotionKey struct {
+ acct, app, version string
+ promotedAt int64
+}
+
+type missKey struct{ host, sha string }
+
+type ProofGate struct {
+ policyPath string
+ bakeDir string
+ now func() time.Time
+
+ mu sync.Mutex
+ policy *ProofPolicy
+ policyErr error
+ lastPolicyErr string
+ lastPolicyLog time.Time
+ shas map[promotionKey]string
+ misses map[missKey]time.Time
+ waits map[string]ProofWait
+}
+
+func NewProofGate(policyPath, bakeDir string) *ProofGate {
+ return &ProofGate{policyPath: policyPath, bakeDir: bakeDir, now: time.Now,
+ shas: map[promotionKey]string{}, misses: map[missKey]time.Time{}, waits: map[string]ProofWait{}}
+}
+
+// WithClock replaces the gate's clock (tests).
+func (g *ProofGate) WithClock(now func() time.Time) *ProofGate {
+ g.now = now
+ return g
+}
+
+// BeginPass re-reads the policy file. PassFilter calls it once per pass.
+func (g *ProofGate) BeginPass() {
+ if g == nil {
+ return
+ }
+ pol, err := LoadProofPolicy(g.policyPath)
+ now := g.now()
+ say := false
+ g.mu.Lock()
+ g.policy, g.policyErr = pol, err
+ if err != nil {
+ msg := err.Error()
+ say = msg != g.lastPolicyErr || now.Sub(g.lastPolicyLog) >= proofPolicyLogEvery
+ if say {
+ g.lastPolicyErr, g.lastPolicyLog = msg, now
+ }
+ } else {
+ g.lastPolicyErr = ""
+ }
+ g.mu.Unlock()
+ if say {
+ fmt.Printf(" %v; every named-unit route waits until it is fixed\n", err)
+ }
+}
+
+func (g *ProofGate) state() (*ProofPolicy, error) {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ return g.policy, g.policyErr
+}
+
+// Waits is the waiting list, sorted by host.
+func (g *ProofGate) Waits() []ProofWait {
+ if g == nil {
+ return nil
+ }
+ g.mu.Lock()
+ out := make([]ProofWait, 0, len(g.waits))
+ for _, w := range g.waits {
+ out = append(out, w)
+ }
+ g.mu.Unlock()
+ sort.Slice(out, func(i, j int) bool { return out[i].Host < out[j].Host })
+ return out
+}
+
+// WaitingDue reports whether host is waiting and its cached miss has run
+// out, which is when the fast loop should look at it again.
+func (g *ProofGate) WaitingDue(host string) bool {
+ if g == nil {
+ return false
+ }
+ now := g.now()
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ w, ok := g.waits[host]
+ if !ok {
+ return false
+ }
+ until, cached := g.misses[missKey{host, w.SHA256}]
+ return !cached || !now.Before(until)
+}
+
+// clear takes host off the waiting list and forgets its cached misses. A
+// miss that outlived its wait would hold the host silently, off the list,
+// if its channel came back to the same build inside ProofMissTTL.
+func (g *ProofGate) clear(host string) {
+ if g == nil {
+ return
+ }
+ g.mu.Lock()
+ delete(g.waits, host)
+ for k := range g.misses {
+ if k.host == host {
+ delete(g.misses, k)
+ }
+ }
+ g.mu.Unlock()
+}
+
+// hold records host as waiting, caches the miss, and prints the line once
+// per change of version, sha256 or reason.
+func (g *ProofGate) hold(rt registry.Route, ver, sha, prover, reason, cur string) {
+ cur = orNothing(cur)
+ now := g.now()
+ g.mu.Lock()
+ prev, had := g.waits[rt.Host]
+ same := had && prev.Version == ver && prev.SHA256 == sha
+ w := ProofWait{Host: rt.Host, App: rt.App, Version: ver, SHA256: sha, Prover: prover, Reason: reason, Since: now}
+ if same {
+ w.Since = prev.Since
+ }
+ g.waits[rt.Host] = w
+ g.misses[missKey{rt.Host, sha}] = now.Add(ProofMissTTL)
+ g.mu.Unlock()
+ if same && prev.Reason == reason {
+ return
+ }
+ if sha == "" {
+ fmt.Printf(" %s: %s is waiting: %s; staying on %s\n", rt.Host, ver, reason, cur)
+ return
+ }
+ fmt.Printf(" %s: %s (sha256 %s) is not proven on %s; staying on %s (%s)\n", rt.Host, ver, sha, prover, cur, reason)
+}
+
+// orNothing names the build a host is on for a "staying on" or "still on"
+// line: a first adoption has none.
+func orNothing(ver string) string {
+ if ver == "" {
+ return "nothing"
+ }
+ return ver
+}
+
+// proofVerdict is admit's answer, carried to bind.
+type proofVerdict struct {
+ required bool // the route's app is listed in a require entry
+ sha string // the manifest's sha256 for this platform
+ proof *Proof // the verified proof; nil when admitted by the healthy history
+}
+
+// admit is the cheap gate (spec §4.5 check 1). false means adoptHost returns.
+func (g *ProofGate) admit(ctx context.Context, s Store, optDir string, rt registry.Route, ptr *ChannelPointer, cur string) (proofVerdict, bool) {
+ if g == nil {
+ return proofVerdict{}, true
+ }
+ pol, perr := g.state()
+ if perr != nil {
+ // Any load error is D3 on, never "no policy" (spec §4.5): every
+ // named-unit route waits, since the box cannot tell which apps were
+ // listed. The reason is the load error itself, which names the file
+ // and says whether it did not parse or could not be read.
+ if rt.Unit == "" {
+ return proofVerdict{}, true
+ }
+ g.hold(rt, ptr.Version, "", "", perr.Error(), cur)
+ return proofVerdict{}, false
+ }
+ reqs := pol.Requires(rt.App)
+ if len(reqs) == 0 {
+ g.clear(rt.Host)
+ return proofVerdict{}, true
+ }
+ acct := RouteAccount(rt)
+ sha, err := g.manifestSHA(ctx, s, optDir, acct, rt.App, ptr)
+ if err != nil {
+ fmt.Printf(" %s: %s: reading its manifest for the proof check: %v\n", rt.Host, ptr.Version, err)
+ return proofVerdict{}, false
+ }
+ v := proofVerdict{required: true, sha: sha}
+ if sha == "" {
+ return v, true // no build for this platform: ensureStaged says so
+ }
+ if IsHealthy(g.bakeDir, acct, rt.App, sha) {
+ g.clear(rt.Host)
+ return v, true
+ }
+ g.mu.Lock()
+ until, cached := g.misses[missKey{rt.Host, sha}]
+ g.mu.Unlock()
+ if cached && g.now().Before(until) {
+ return v, false
+ }
+ var reasons, names []string
+ for _, e := range reqs {
+ names = append(names, e.Prover)
+ p, err := GetProof(ctx, s, acct, rt.App, sha, e.Prover)
+ if errors.Is(err, os.ErrNotExist) {
+ reasons = append(reasons, "no proof from "+e.Prover)
+ continue
+ }
+ if err == nil {
+ err = p.Check(e.PublicKey(), rt.App, adoptPlatform, sha, e.Prover)
+ }
+ if err != nil {
+ reasons = append(reasons, fmt.Sprintf("proof from %s: %v", e.Prover, err))
+ continue
+ }
+ g.clear(rt.Host)
+ v.proof = p
+ return v, true
+ }
+ g.hold(rt, ptr.Version, sha, strings.Join(names, " or "), strings.Join(reasons, "; "), cur)
+ return v, false
+}
+
+// manifestSHA is the manifest's sha256 for this platform, from the staged
+// sibling when there is one and the bucket otherwise, cached per promotion.
+func (g *ProofGate) manifestSHA(ctx context.Context, s Store, optDir, acct, app string, ptr *ChannelPointer) (string, error) {
+ k := promotionKey{acct, app, ptr.Version, ptr.PromotedAt.Unix()}
+ g.mu.Lock()
+ sha, ok := g.shas[k]
+ g.mu.Unlock()
+ if ok {
+ return sha, nil
+ }
+ m, err := ReadStagedManifest(optDir, acct, ptr.Version, app)
+ if err != nil || m == nil {
+ m, err = GetManifest(ctx, s, acct, app, ptr.Version)
+ if err != nil {
+ return "", err
+ }
+ }
+ sha = m.Artifacts[adoptPlatform].SHA256
+ g.mu.Lock()
+ g.shas[k] = sha
+ g.mu.Unlock()
+ return sha, nil
+}
+
+// bind is the staged-bytes check (spec §4.5 check 2). false means adoptHost
+// returns. A mismatch is a RefusalNotProven record naming both hashes:
+// nothing was restarted, so it is Recovered and still on cur, the same shape
+// as D2's wrong-program record.
+func (g *ProofGate) bind(optDir string, rt registry.Route, ptr *ChannelPointer, v proofVerdict, cur string, now time.Time) bool {
+ if g == nil || !v.required {
+ return true
+ }
+ acct := RouteAccount(rt)
+ got, err := FileSHA256(VersionPath(optDir, acct, ptr.Version, rt.App))
+ if err != nil {
+ fmt.Printf(" %s: %s: hashing the staged build for the proof check: %v; not pointed live\n", rt.Host, ptr.Version, err)
+ return false
+ }
+ if v.proof != nil && got == v.proof.SHA256 {
+ return true
+ }
+ if IsHealthy(g.bakeDir, acct, rt.App, got) {
+ return true
+ }
+ reason := fmt.Sprintf("staged sha256 %s, manifest sha256 %s", got, v.sha)
+ if v.proof != nil {
+ reason = fmt.Sprintf("staged sha256 %s, %s proved %s", got, v.proof.Prover, v.proof.SHA256)
+ }
+ fmt.Printf(" %s: %s: the staged bytes do not match the proof (%s); not pointed live\n", rt.Host, ptr.Version, reason)
+ if g.bakeDir != "" {
+ if err := WriteRefusal(g.bakeDir, Refusal{Host: rt.Host, App: rt.App, Version: ptr.Version, SHA256: got,
+ Platform: adoptPlatform, PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(),
+ Reason: "not proven: " + reason,
+ Kind: RefusalNotProven, Recovered: true, RestoredVersion: cur}); err != nil {
+ fmt.Printf(" %s: writing refusal record: %v\n", rt.Host, err)
+ }
+ }
+ g.clear(rt.Host)
+ return false
+}
diff --git a/internal/release/proofgate_test.go b/internal/release/proofgate_test.go
new file mode 100644
index 000000000..4d4552ac9
--- /dev/null
+++ b/internal/release/proofgate_test.go
@@ -0,0 +1,494 @@
+package release
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+const proofHost = "console.example.com"
+
+// proofEnv is startUnitFixture (live "old", channel canary/x promoted to
+// "new" whose bytes are "bad build") plus a prover key, a policy path, a bake
+// directory and one gate shared by every pass, as the agent shares one. It
+// keeps the release key too, so a test can promote again.
+type proofEnv struct {
+ ctx context.Context
+ reg *registry.Registry
+ store Store
+ priv ed25519.PrivateKey
+ pub ed25519.PublicKey
+ opt, bake string
+ policy string
+ prover ed25519.PrivateKey
+ proverPub string
+ newSHA string
+ clock time.Time
+ gate *ProofGate
+}
+
+func newProofEnv(t *testing.T) *proofEnv {
+ t.Helper()
+ // startUnitFixture's setup, inlined so the release key is kept.
+ s := NewDirStore(t.TempDir())
+ ctx := context.Background()
+ priv, pub := testKeys(t)
+ opt := t.TempDir()
+ reg := testRegistry(t)
+ shipRelease(t, s, "bab", "console", "old", adoptPlatform, []byte("good build"))
+ shipRelease(t, s, "bab", "console", "new", adoptPlatform, []byte("bad build"))
+ if err := StageArtifact(opt, "bab", "old", "console", []byte("good build")); err != nil {
+ t.Fatal(err)
+ }
+ addRoute(t, reg, proofHost, "console", "canary/x", false)
+ if err := PointLive(opt, proofHost, "bab", "old", "console"); err != nil {
+ t.Fatal(err)
+ }
+ if err := Promote(ctx, s, priv, "bab", "console", "canary/x", "new"); err != nil {
+ t.Fatalf("Promote new: %v", err)
+ }
+ prover, _, pubB64 := proverKey(t)
+ e := &proofEnv{ctx: ctx, reg: reg, store: s, priv: priv, pub: pub, opt: opt, bake: t.TempDir(),
+ policy: filepath.Join(t.TempDir(), "proof-policy.json"), prover: prover, proverPub: pubB64,
+ newSHA: etagOf([]byte("bad build")), clock: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)}
+ e.gate = NewProofGate(e.policy, e.bake).WithClock(func() time.Time { return e.clock })
+ return e
+}
+
+func (e *proofEnv) writePolicy(t *testing.T, body string) {
+ t.Helper()
+ if err := os.WriteFile(e.policy, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (e *proofEnv) requireDev(t *testing.T) {
+ e.writePolicy(t, fmt.Sprintf(`{"schema":1,"require":[{"app":"console","prover":"dev","key":%q}]}`, e.proverPub))
+}
+
+func (e *proofEnv) pass(t *testing.T, s Store) string {
+ t.Helper()
+ return e.passWith(t, s, func(string) error { return nil })
+}
+
+func (e *proofEnv) passWith(t *testing.T, s Store, restart func(string) error) string {
+ t.Helper()
+ var out bytes.Buffer
+ withCapturedStdout(t, &out, func() {
+ if err := Pass(e.ctx, e.reg, s, e.pub, e.opt, restart,
+ BakeDir(e.bake), WithProofGate(e.gate)); err != nil {
+ t.Fatalf("Pass: %v", err)
+ }
+ })
+ return out.String()
+}
+
+func (e *proofEnv) live() string { v, _ := LiveVersion(e.opt, proofHost); return v }
+
+func (e *proofEnv) signed(t *testing.T, key ed25519.PrivateKey, app, sha, prover string) *Proof {
+ t.Helper()
+ p, err := SignProof(key, Proof{App: app, Platform: adoptPlatform, SHA256: sha, Version: "new", Prover: prover,
+ ActiveSince: e.clock.Add(-time.Hour), ProvenAt: e.clock})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return p
+}
+
+// putAt writes p at the dev key for sha, whatever p itself says.
+func (e *proofEnv) putAt(t *testing.T, sha string, p *Proof) {
+ t.Helper()
+ b, err := json.MarshalIndent(p, "", " ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := e.store.Put(e.ctx, ProofKey("bab", "console", sha, "dev"), b, ""); err != nil {
+ t.Fatal(err)
+ }
+}
+
+type proofReadCountingStore struct {
+ Store
+ mu sync.Mutex
+ gets []string
+}
+
+func (c *proofReadCountingStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ c.mu.Lock()
+ c.gets = append(c.gets, key)
+ c.mu.Unlock()
+ return c.Store.Get(ctx, key)
+}
+
+func (c *proofReadCountingStore) proofGets() int {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ n := 0
+ for _, k := range c.gets {
+ if strings.Contains(k, "/proofs/") {
+ n++
+ }
+ }
+ return n
+}
+
+func TestProofNoPolicyMakesNoProofRead(t *testing.T) {
+ e := newProofEnv(t)
+ cs := &proofReadCountingStore{Store: e.store}
+ e.pass(t, cs)
+ if e.live() != "new" {
+ t.Fatalf("live = %q, want new: with no policy file adoption is unchanged", e.live())
+ }
+ if n := cs.proofGets(); n != 0 {
+ t.Fatalf("%d proof GETs with no policy file, want 0 (spec §4.6)", n)
+ }
+}
+
+func TestProofMissingStaysAndIsNotARefusal(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ out := e.pass(t, e.store)
+ if e.live() != "old" {
+ t.Fatalf("live = %q, want old", e.live())
+ }
+ if !strings.Contains(out, "is not proven on dev; staying on old") {
+ t.Fatalf("printed %q, want the not-proven line", out)
+ }
+ if r, _ := ReadRefusal(e.bake, proofHost); r != nil {
+ t.Fatalf("a missing proof wrote a refusal: %+v", r)
+ }
+ w := e.gate.Waits()
+ if len(w) != 1 || w[0].Host != proofHost || w[0].SHA256 != e.newSHA || w[0].Prover != "dev" {
+ t.Fatalf("waits = %+v", w)
+ }
+}
+
+func TestProofValidAdopts(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev"))
+ e.pass(t, e.store)
+ if e.live() != "new" {
+ t.Fatalf("live = %q, want new", e.live())
+ }
+ if w := e.gate.Waits(); len(w) != 0 {
+ t.Fatalf("waits = %+v after adopting", w)
+ }
+}
+
+func TestProofRejectsBadProofs(t *testing.T) {
+ for name, mk := range map[string]func(t *testing.T, e *proofEnv) *Proof{
+ "bad signature": func(t *testing.T, e *proofEnv) *Proof {
+ other, _, _ := proverKey(t)
+ return e.signed(t, other, "console", e.newSHA, "dev")
+ },
+ "wrong sha256": func(t *testing.T, e *proofEnv) *Proof {
+ return e.signed(t, e.prover, "console", etagOf([]byte("other bytes")), "dev")
+ },
+ "wrong app": func(t *testing.T, e *proofEnv) *Proof {
+ return e.signed(t, e.prover, "other", e.newSHA, "dev")
+ },
+ "wrong prover": func(t *testing.T, e *proofEnv) *Proof {
+ return e.signed(t, e.prover, "console", e.newSHA, "staging")
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ e.putAt(t, e.newSHA, mk(t, e))
+ out := e.pass(t, e.store)
+ if e.live() != "old" || !strings.Contains(out, "is not proven on dev") {
+ t.Fatalf("live = %q, printed %q; want old and the not-proven line", e.live(), out)
+ }
+ })
+ }
+}
+
+// Review focus 3: rolling back to a build this box ran never waits on dev.
+func TestProofHealthyHistoryAdmitsWithoutProof(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ if err := RecordHealthy(e.bake, "bab", "console", HealthyBuild{SHA256: e.newSHA, Version: "new", How: "settle", At: e.clock}); err != nil {
+ t.Fatal(err)
+ }
+ cs := &proofReadCountingStore{Store: e.store}
+ e.pass(t, cs)
+ if e.live() != "new" {
+ t.Fatalf("live = %q, want new from the healthy history", e.live())
+ }
+ if n := cs.proofGets(); n != 0 {
+ t.Fatalf("%d proof GETs for a build in the healthy history, want 0", n)
+ }
+}
+
+// Review focus 2: the manifest's sha256 is proven, but the staged file (a
+// cache hit, which ensureStaged does not re-hash) holds other bytes.
+//
+// Ruling 3: the record is its own kind, recovered and still on the old
+// build, because nothing was restarted. The next pass's gate line says the
+// staged bytes do not match the proof and never suggests --retry, which
+// would re-sign the pointer over the same stale bytes and be refused again.
+func TestProofBindingRefusesStagedBytesThatDiffer(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ stale := []byte("not the shipped bytes")
+ if err := StageArtifact(e.opt, "bab", "new", "console", stale); err != nil {
+ t.Fatal(err)
+ }
+ e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev"))
+ out := e.pass(t, e.store)
+ if e.live() != "old" || !strings.Contains(out, "not pointed live") {
+ t.Fatalf("live = %q, printed %q; want old and the binding refusal", e.live(), out)
+ }
+ r, _ := ReadRefusal(e.bake, proofHost)
+ if r == nil || !strings.Contains(r.Reason, etagOf(stale)) || !strings.Contains(r.Reason, e.newSHA) {
+ t.Fatalf("refusal = %+v, want one naming both hashes", r)
+ }
+ if r.Kind != RefusalNotProven || !r.Recovered || r.RestoredVersion != "old" || r.RecoveryError != "" {
+ t.Fatalf("refusal = %+v, want kind %q, recovered, restored to old", r, RefusalNotProven)
+ }
+ out = e.pass(t, e.store)
+ if e.live() != "old" {
+ t.Fatalf("live = %q on the second pass, want old", e.live())
+ }
+ want := "refused: new's staged bytes do not match dev's proof"
+ if !strings.Contains(out, want) || !strings.Contains(out, "still on old") || !strings.Contains(out, "roll the exact bytes dev proved") {
+ t.Fatalf("second pass printed %q, want the not-proven gate line", out)
+ }
+ if strings.Contains(out, "--retry") || strings.Contains(out, "did not come up") {
+ t.Fatalf("second pass printed %q: a not-proven refusal is not a failed start", out)
+ }
+}
+
+func TestProofMissIsCachedThirtySeconds(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ cs := &proofReadCountingStore{Store: e.store}
+ e.pass(t, cs)
+ e.clock = e.clock.Add(10 * time.Second)
+ e.pass(t, cs)
+ if n := cs.proofGets(); n != 1 {
+ t.Fatalf("%d proof GETs inside 30s, want 1", n)
+ }
+ if e.gate.WaitingDue(proofHost) {
+ t.Fatal("due inside its 30s miss")
+ }
+ e.clock = e.clock.Add(21 * time.Second)
+ if !e.gate.WaitingDue(proofHost) {
+ t.Fatal("not due after 31s")
+ }
+ e.pass(t, cs)
+ if n := cs.proofGets(); n != 2 {
+ t.Fatalf("%d proof GETs after 31s, want 2", n)
+ }
+}
+
+// Review focus 1: a typo keeps the gate closed for a named-unit route.
+func TestProofPolicyUnparseableKeepsGateClosed(t *testing.T) {
+ e := newProofEnv(t)
+ if err := e.reg.SetUnit(proofHost, "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ e.writePolicy(t, `{"schema":1,"require":[`)
+ out := e.pass(t, e.store)
+ if e.live() != "old" {
+ t.Fatalf("live = %q, want old: an unparseable policy must not open the gate", e.live())
+ }
+ if !strings.Contains(out, "does not parse") {
+ t.Fatalf("printed %q, want the policy error", out)
+ }
+ if w := e.gate.Waits(); len(w) != 1 || !strings.Contains(w[0].Reason, "does not parse") {
+ t.Fatalf("waits = %+v", w)
+ }
+}
+
+// The Task 2 carry: an error that is not a parse error (here, a policy path
+// that is a directory and cannot be read) is closed too, never "no policy".
+func TestProofPolicyUnreadableKeepsGateClosed(t *testing.T) {
+ e := newProofEnv(t)
+ if err := e.reg.SetUnit(proofHost, "console.service"); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Mkdir(e.policy, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ e.pass(t, e.store)
+ if e.live() != "old" {
+ t.Fatalf("live = %q, want old: an unreadable policy must not open the gate", e.live())
+ }
+ if w := e.gate.Waits(); len(w) != 1 || w[0].Host != proofHost {
+ t.Fatalf("waits = %+v, want the named-unit route waiting", w)
+ }
+}
+
+// The agent's gate lives across passes: a policy edit is seen on the next
+// pass without restarting anything.
+func TestProofPolicyChangeSeenNextPass(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ e.pass(t, e.store)
+ if e.live() != "old" {
+ t.Fatalf("live = %q, want old while required", e.live())
+ }
+ e.writePolicy(t, `{"schema":1}`)
+ e.pass(t, e.store)
+ if e.live() != "new" {
+ t.Fatalf("live = %q, want new once the require entry is gone", e.live())
+ }
+}
+
+// Ruling 5 (preflight 4.6): a host waiting on one head whose channel moves
+// to a build that stops before the gate (bake hold, poisoned this pass, a
+// refusal record) is no longer waiting. Otherwise WaitingDue stays true and
+// the fast loop sweeps it every tick for the whole hold.
+func TestProofWaitClearedWhenTheGateIsNotConsulted(t *testing.T) {
+ assertCleared := func(t *testing.T, e *proofEnv, out string) {
+ t.Helper()
+ if w := e.gate.Waits(); len(w) != 0 {
+ t.Fatalf("waits = %+v, printed %q; want none once the gate is not consulted", w, out)
+ }
+ if e.gate.WaitingDue(proofHost) {
+ t.Fatal("still due once the gate is not consulted")
+ }
+ }
+ waitOnNew := func(t *testing.T) *proofEnv {
+ t.Helper()
+ e := newProofEnv(t)
+ e.requireDev(t)
+ e.pass(t, e.store)
+ if w := e.gate.Waits(); len(w) != 1 || w[0].Version != "new" {
+ t.Fatalf("waits = %+v, want one on new", w)
+ }
+ e.clock = e.clock.Add(ProofMissTTL + time.Second)
+ return e
+ }
+
+ t.Run("bake hold", func(t *testing.T) {
+ e := waitOnNew(t)
+ // beta has a 24h window, and a production app's windows apply.
+ if err := SetProduction(e.ctx, e.store, e.priv, "bab", "console", true, "test"); err != nil {
+ t.Fatal(err)
+ }
+ shipRelease(t, e.store, "bab", "console", "newer", adoptPlatform, []byte("newer build"))
+ if err := Promote(e.ctx, e.store, e.priv, "bab", "console", "beta", "newer"); err != nil {
+ t.Fatal(err)
+ }
+ addRoute(t, e.reg, proofHost, "console", "beta", false)
+ out := e.pass(t, e.store)
+ if !strings.Contains(out, "bake window") {
+ t.Fatalf("printed %q, want the bake hold", out)
+ }
+ assertCleared(t, e, out)
+ })
+
+ t.Run("poisoned", func(t *testing.T) {
+ e := waitOnNew(t)
+ // A second, hibernating route on the same channel goes after the
+ // awake one, so it meets the build the awake one poisoned.
+ const sleeper = "sleeper.example.com"
+ addRoute(t, e.reg, sleeper, "console", "canary/x", true)
+ if err := PointLive(e.opt, sleeper, "bab", "old", "console"); err != nil {
+ t.Fatal(err)
+ }
+ e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev"))
+ // The sleeper is the one whose wait must go: make it wait first.
+ e.gate.hold(registry.Route{Host: sleeper, App: "console"}, "new", e.newSHA, "dev", "no proof from dev", "old")
+ e.clock = e.clock.Add(ProofMissTTL + time.Second)
+ out := e.passWith(t, e.store, func(string) error { return errors.New("Job for console.service failed") })
+ if !strings.Contains(out, "failed on another host this pass") {
+ t.Fatalf("printed %q, want the poisoned skip", out)
+ }
+ for _, w := range e.gate.Waits() {
+ if w.Host == sleeper {
+ t.Fatalf("waits = %+v, printed %q; the poisoned host is still waiting", e.gate.Waits(), out)
+ }
+ }
+ if e.gate.WaitingDue(sleeper) {
+ t.Fatal("poisoned host still due")
+ }
+ })
+
+ t.Run("refusal gate", func(t *testing.T) {
+ e := waitOnNew(t)
+ ptr, err := Channel(e.ctx, e.store, e.pub, "bab", "console", "canary/x")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := WriteRefusal(e.bake, Refusal{Host: proofHost, App: "console", Version: "new",
+ PointerPromotedAt: ptr.PromotedAt, RefusedAt: e.clock, Reason: "exited: status=2"}); err != nil {
+ t.Fatal(err)
+ }
+ out := e.pass(t, e.store)
+ if !strings.Contains(out, "refused: new") {
+ t.Fatalf("printed %q, want the refusal gate", out)
+ }
+ assertCleared(t, e, out)
+ })
+}
+
+// A host found live on its head (another process, root say, moved it) is
+// not waiting for anything.
+func TestProofWaitClearedOnceLiveOnTheHead(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ e.pass(t, e.store)
+ if len(e.gate.Waits()) != 1 {
+ t.Fatalf("waits = %+v, want one", e.gate.Waits())
+ }
+ if err := StageArtifact(e.opt, "bab", "new", "console", []byte("bad build")); err != nil {
+ t.Fatal(err)
+ }
+ if err := PointLive(e.opt, proofHost, "bab", "new", "console"); err != nil {
+ t.Fatal(err)
+ }
+ e.pass(t, e.store)
+ if w := e.gate.Waits(); len(w) != 0 || e.gate.WaitingDue(proofHost) {
+ t.Fatalf("waits = %+v once live on the head, want none", w)
+ }
+}
+
+// A host that stops waiting forgets its cached miss too: when the channel
+// comes back to the same build inside 30 s, the gate looks again and the
+// host is listed as waiting, rather than held silently off the list.
+func TestProofClearDropsTheCachedMiss(t *testing.T) {
+ e := newProofEnv(t)
+ e.requireDev(t)
+ cs := &proofReadCountingStore{Store: e.store}
+ e.pass(t, cs)
+ e.gate.clear(proofHost)
+ e.clock = e.clock.Add(time.Second)
+ e.pass(t, cs)
+ if n := cs.proofGets(); n != 2 {
+ t.Fatalf("%d proof GETs, want 2: a cleared host's miss is not cached", n)
+ }
+ if w := e.gate.Waits(); len(w) != 1 {
+ t.Fatalf("waits = %+v, want the host listed again", w)
+ }
+}
+
+// Every gate method is nil-safe: no WithProofGate is today's behaviour.
+func TestProofGateNilIsOff(t *testing.T) {
+ var g *ProofGate
+ g.BeginPass()
+ g.clear("x")
+ if g.Waits() != nil || g.WaitingDue("x") {
+ t.Fatal("a nil gate reports a wait")
+ }
+ if _, ok := g.admit(context.Background(), nil, "", registry.Route{Host: "x"}, &ChannelPointer{}, ""); !ok {
+ t.Fatal("a nil gate refused admission")
+ }
+ if !g.bind("", registry.Route{Host: "x"}, &ChannelPointer{}, proofVerdict{required: true}, "", time.Time{}) {
+ t.Fatal("a nil gate refused the staged bytes")
+ }
+}
diff --git a/internal/release/refusal.go b/internal/release/refusal.go
index 9a5cbce56..2dc4691b8 100644
--- a/internal/release/refusal.go
+++ b/internal/release/refusal.go
@@ -35,15 +35,19 @@ const refusalSchema = 1
// before Kind existed reads as one) is D1: the unit did not come up and was
// rolled back. RefusalWrongProgram is D2: the staged binary is a different Go
// program from any this route has run, so it was never pointed live and
-// nothing restarted. A "--retry" cures the first and loops on the second.
+// nothing restarted. RefusalNotProven is D3 (proofgate.go): the staged bytes
+// are not the ones the proof covers, so they too were never pointed live. A
+// "--retry" cures the first and loops on the other two.
const (
RefusalNotUp = ""
RefusalWrongProgram = "wrong-program"
+ RefusalNotProven = "not-proven"
)
type Refusal struct {
Schema int `json:"schema"`
- // Kind is RefusalNotUp or RefusalWrongProgram. Omitted when not-up.
+ // Kind is RefusalNotUp, RefusalWrongProgram or RefusalNotProven. Omitted
+ // when not-up.
Kind string `json:"kind,omitempty"`
Host string `json:"host"`
App string `json:"app"`
diff --git a/statuslive_test.go b/statuslive_test.go
index 21a4c0e3f..574666806 100644
--- a/statuslive_test.go
+++ b/statuslive_test.go
@@ -204,6 +204,29 @@ func TestPrintLiveBoxesShowsAWrongProgramRefusal(t *testing.T) {
}
}
+// A not-proven refusal (D3, plan 3 ruling 3) is neither a failed start nor a
+// rollback: the staged bytes did not match dev's proof.
+func TestPrintLiveBoxesShowsANotProvenRefusal(t *testing.T) {
+ now := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, []status.Box{
+ {GeneratedAt: now, Label: "ada-stargazing", Metrics: &status.BoxMetrics{
+ Tunnel: "connected",
+ ReleaseRefusals: []status.ReleaseRefusal{{
+ Host: "console.example.com", Version: "v2", Kind: "not-proven",
+ RefusedAt: time.Date(2026, 10, 6, 9, 30, 40, 0, time.UTC), Reason: "not proven: staged sha256 aaaa, dev proved bbbb",
+ }},
+ }},
+ }, now)
+ want := "refused console.example.com v2 since 2026-10-06T09:30:40Z: staged bytes do not match dev's proof (staged sha256 aaaa, dev proved bbbb)\n"
+ if !strings.Contains(buf.String(), want) {
+ t.Fatalf("missing %q in:\n%s", want, buf.String())
+ }
+ if strings.Contains(buf.String(), "rollback") {
+ t.Fatalf("a not-proven row mentions a rollback:\n%s", buf.String())
+ }
+}
+
func TestHumanBytesAndFloat(t *testing.T) {
for _, c := range []struct {
in *int64