amadan

carlos / platform Public

D3-held fix round 1: bind inside the publication with a re-hash at the swap; move-scoped held refusals

Finding 1: publishHeld runs bind inside guard (after the authority's
re-check, under its freeze), swaps the link, and binds again on the bytes
the link now names; a refusal there puts the link back (or removes it)
and is ErrHeldNotProven. heldBeforeSwap is the test seam for a
replacement between bind and the swap.

Finding 2: a held binding refusal carries its move (Refusal.MoveOp, from
rt.HoldOp, now required on a held stage) and is never written over an
adoption refusal. A successful held stage retires its move's record; the
target's moves pass retires records of moves no longer preparing or
ready (with evictArchives); root's held stage pass retires those whose
claim no longer names the move into this box. Proof minting skips
move-scoped records. bind is bindRecording with WriteRefusal, so
adoption's behaviour and records are unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude b65dcedbb472039eb4d7cbbc9c0ef7221d4af17c parent c634951
12 files changed, +464 −13
  • internal/host/heldstage.go +31 −0
  • internal/host/heldstage_test.go +1 −1
  • internal/host/movespass.go +1 −0
  • internal/host/movesreport.go +20 −0
  • internal/host/movesstage.go +1 −1
  • internal/host/movesstage_proof_test.go +95 −0
  • internal/host/proofroot.go +3 −1
  • internal/host/proofroot_test.go +15 −0
  • internal/release/materializeheld.go +114 −7
  • internal/release/materializeheld_proof_test.go +169 −2
  • internal/release/proofgate.go +8 −1
  • internal/release/refusal.go +6 −0
diff --git a/internal/host/heldstage.go b/internal/host/heldstage.go
index dfe5a201a..5bd1ca769 100644
--- a/internal/host/heldstage.go
+++ b/internal/host/heldstage.go
@@ -191,4 +191,35 @@ func heldStagePass(ctx context.Context, dataDir, optDir string, reg *registry.Re
if err := release.ServeHeldStageRequests(ctx, store, optDir, dataDir, proof, heldStageAuthority(reg, store, box)); err != nil {
log.Printf("adopt: held stage requests: %v", err)
}
+ retireEndedHeldRefusals(ctx, store, dataDir, box)
+}
+
+// retireEndedHeldRefusals is root's half of a held refusal's lifecycle
+// (D3-held): every refusal record a move's held stage left (MoveOp set)
+// whose move the host claim no longer names as moving or landing into this
+// box is removed, so an aborted or abandoned move leaves nothing behind.
+// The agent retires the ones it can (movesWorker.retireHeldRefusals), but
+// not a record root wrote. A claim that cannot be read keeps the record for
+// the next pass. Adoption refusals are never touched, and no claim is read
+// unless a held refusal exists.
+func retireEndedHeldRefusals(ctx context.Context, store release.Store, dataDir, box string) {
+ retired, err := release.RetireHeldRefusals(dataDir, func(r release.Refusal) bool {
+ rctx, cancel := context.WithTimeout(ctx, heldStageReadTimeout)
+ defer cancel()
+ obs, err := hostclaim.Read(rctx, store, r.Host)
+ if err != nil {
+ return true
+ }
+ if obs.Absent || obs.Claim == nil {
+ return false
+ }
+ op := obs.Claim.Op
+ return op.ID == r.MoveOp && (op.Kind == hostclaim.KindMoving || op.Kind == hostclaim.KindLanding) && op.Intent["target"] == box
+ })
+ for _, host := range retired {
+ log.Printf("adopt: %s: retired the held refusal of a move that is over", host)
+ }
+ if err != nil {
+ log.Printf("adopt: held refusals: %v", err)
+ }
}
diff --git a/internal/host/heldstage_test.go b/internal/host/heldstage_test.go
index c1744adde..e8a6b5eff 100644
--- a/internal/host/heldstage_test.go
+++ b/internal/host/heldstage_test.go
@@ -355,7 +355,7 @@ func TestHeldPublicationIsGuardedByTheAuthority(t *testing.T) {
if err := auth(req, nil); err != nil {
t.Fatalf("authority before staging: %v", err)
}
- rt := registry.Route{Host: req.Host, Account: req.Account, App: req.App}
+ rt := registry.Route{Host: req.Host, Account: req.Account, App: req.App, HoldOp: req.Op}
if _, err := release.StageHeld(ctx, s, opt, rt, req.Version); err != nil {
t.Fatalf("StageHeld: %v", err)
}
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index a8bbd7bc3..6a548d652 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -529,6 +529,7 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
}
default:
w.evictArchives(nil) // no move into this box: no verdict to keep
+ w.retireHeldRefusals(nil)
}
w.liftFences(recs)
return nil
diff --git a/internal/host/movesreport.go b/internal/host/movesreport.go
index 7ddde5b17..21c763f5c 100644
--- a/internal/host/movesreport.go
+++ b/internal/host/movesreport.go
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
+ "io/fs"
"log"
"os"
"path/filepath"
@@ -122,6 +123,7 @@ func loadMoveFleet(ctx context.Context, s release.Store) (moveFleet, error) {
func (w *movesWorker) targetReportPass(ctx context.Context, s release.Store, recs []provision.Record, fleet moveFleet) {
live := map[string]bool{}
defer w.evictArchives(live)
+ defer w.retireHeldRefusals(live)
for _, rec := range recs {
m := rec.Move
if m == nil || m.Legacy() || m.To != w.cfg.Region || (m.State != provision.MovePreparing && m.State != provision.MoveReady) {
@@ -148,6 +150,24 @@ func (w *movesWorker) evictArchives(live map[string]bool) {
}
}
+// retireHeldRefusals retires the held-stage refusal (D3-held) of every
+// move no longer preparing or ready into this box, with the rest of its
+// staging state: live is the same set evictArchives keeps. A record root
+// wrote cannot be removed here (the agent does not own it), which is not
+// worth a line: the root stage pass retires those (retireEndedHeldRefusals).
+func (w *movesWorker) retireHeldRefusals(live map[string]bool) {
+ if w.cfg.DataDir == "" {
+ return
+ }
+ retired, err := release.RetireHeldRefusals(w.cfg.DataDir, func(r release.Refusal) bool { return live[r.MoveOp] })
+ for _, host := range retired {
+ log.Printf("moves: %s: retired the held refusal of a move that is over", host)
+ }
+ if err != nil && !errors.Is(err, fs.ErrPermission) {
+ w.logTrouble("held refusals", err)
+ }
+}
+
// verifiedArchive is what T5 learned from one published archive, kept per
// exact version so a pass does not open the same archive again.
type verifiedArchive struct {
diff --git a/internal/host/movesstage.go b/internal/host/movesstage.go
index 9d826bba7..c666f13b4 100644
--- a/internal/host/movesstage.go
+++ b/internal/host/movesstage.go
@@ -46,7 +46,7 @@ func heldReleaseReady(reg *registry.Registry, optDir, dataDir, box string, proof
logMoveStage(rec.Host, version, err)
return moves.Check{Reason: t7Failed}
}
- rt := registry.Route{Host: rec.Host, Kind: "instance", App: rec.App, Account: rec.Account}
+ rt := registry.Route{Host: rec.Host, Kind: "instance", App: rec.App, Account: rec.Account, HoldOp: rec.Move.ID}
_, err := heldMaterialize(ctx, s, optDir, rt, version, proof, func(publish func() error) error { return auth(req, publish) })
switch {
case errors.Is(err, release.ErrHeldNotProven):
diff --git a/internal/host/movesstage_proof_test.go b/internal/host/movesstage_proof_test.go
index b751659fe..9bc7186e0 100644
--- a/internal/host/movesstage_proof_test.go
+++ b/internal/host/movesstage_proof_test.go
@@ -5,13 +5,16 @@ import (
"crypto/ed25519"
"crypto/sha256"
"encoding/hex"
+ "errors"
"fmt"
"os"
"path/filepath"
"runtime"
+ "strings"
"testing"
"time"
+ "github.com/carlosframework/platform/internal/hostclaim"
"github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/release"
)
@@ -143,3 +146,95 @@ func TestHeldReleaseReadyAppliesTheProofGate(t *testing.T) {
}
})
}
+
+// heldRefusalFixture writes three refusal records under bake: a held one
+// for hsOp at hsHost, a held one for another move at another host, and an
+// adoption refusal (no MoveOp) at a third host.
+func heldRefusalFixture(t *testing.T, bake string) {
+ t.Helper()
+ at := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
+ for _, r := range []release.Refusal{
+ {Host: hsHost, App: "titogo", Version: "v7", RefusedAt: at, Kind: release.RefusalNotProven, MoveOp: hsOp},
+ {Host: "other.example.com", App: "titogo", Version: "v7", RefusedAt: at, Kind: release.RefusalNotProven, MoveOp: hsOther},
+ {Host: "adopted.example.com", App: "titogo", Version: "v7", RefusedAt: at, Kind: release.RefusalNotProven},
+ } {
+ if err := release.WriteRefusal(bake, r); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func refusalHosts(t *testing.T, bake string) string {
+ t.Helper()
+ recs, err := release.ListRefusals(bake)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var hosts []string
+ for _, r := range recs {
+ hosts = append(hosts, r.Host)
+ }
+ return strings.Join(hosts, ",")
+}
+
+// Fix round 1, finding 2, the agent's half: when a move stops being one
+// this box prepares (aborted, abandoned, or past ready), the target's pass
+// retires its held refusal with the rest of the move's staging state. A
+// live move's refusal and an adoption refusal stay.
+func TestTargetPassRetiresAnEndedMovesHeldRefusal(t *testing.T) {
+ data := t.TempDir()
+ heldRefusalFixture(t, data)
+ w := &movesWorker{cfg: moveCfg{DataDir: data}}
+ w.retireHeldRefusals(map[string]bool{hsOp: true})
+ if got := refusalHosts(t, data); got != "adopted.example.com,"+hsHost {
+ t.Fatalf("left %q, want the adoption refusal and the live move's", got)
+ }
+ w.retireHeldRefusals(nil) // no move into this box any more
+ if got := refusalHosts(t, data); got != "adopted.example.com" {
+ t.Fatalf("left %q, want only the adoption refusal", got)
+ }
+}
+
+// Fix round 1, finding 2, root's half: the agent cannot remove a record
+// root wrote, so the root stage pass retires every held refusal whose move
+// the host claim no longer names as moving or landing into this box. A
+// claim it cannot read keeps the record; an adoption refusal is never read
+// past its MoveOp.
+func TestRootRetiresAnAbandonedMovesHeldRefusal(t *testing.T) {
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ data := t.TempDir()
+ heldRefusalFixture(t, data)
+ hsClaim(t, s, movingOp()) // hsHost: still moving into hsBox under hsOp
+ // other.example.com has no claim at all: its move is over.
+ retireEndedHeldRefusals(ctx, s, data, hsBox)
+ if got := refusalHosts(t, data); got != "adopted.example.com,"+hsHost {
+ t.Fatalf("left %q, want the adoption refusal and the live move's", got)
+ }
+
+ // The move is abandoned: the claim goes back to serving.
+ obs, err := hostclaim.Read(ctx, s, hsHost)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseMove(hsOp, "ie-box")); err != nil {
+ t.Fatal(err)
+ }
+ retireEndedHeldRefusals(ctx, s, data, hsBox)
+ if got := refusalHosts(t, data); got != "adopted.example.com" {
+ t.Fatalf("left %q, want only the adoption refusal", got)
+ }
+
+ // A claim read that fails keeps the record.
+ heldRefusalFixture(t, data)
+ retireEndedHeldRefusals(ctx, failingClaimStore{s}, data, hsBox)
+ if got := refusalHosts(t, data); got != "adopted.example.com,"+hsHost+",other.example.com" {
+ t.Fatalf("left %q after failed claim reads, want all three", got)
+ }
+}
+
+type failingClaimStore struct{ release.Store }
+
+func (failingClaimStore) Get(context.Context, string) ([]byte, string, error) {
+ return nil, "", errors.New("bucket unreachable")
+}
diff --git a/internal/host/proofroot.go b/internal/host/proofroot.go
index 5f2c1fb1e..263bd08fd 100644
--- a/internal/host/proofroot.go
+++ b/internal/host/proofroot.go
@@ -232,7 +232,9 @@ func (r proofRoot) mint(ctx context.Context, pol *release.ProofPolicy, rt regist
return
}
for _, ref := range refusals {
- if ref.SHA256 == b.SHA256 {
+ // A move's held refusal (MoveOp set, D3-held) is about that move's
+ // held stage, and is retired with it; it never stops a mint.
+ if ref.MoveOp == "" && ref.SHA256 == b.SHA256 {
r.logf("proof: %s: not minting %s: this box refused it (%s)", rt.Host, b.Version, ref.Reason)
return
}
diff --git a/internal/host/proofroot_test.go b/internal/host/proofroot_test.go
index 03826398e..d950c1f2d 100644
--- a/internal/host/proofroot_test.go
+++ b/internal/host/proofroot_test.go
@@ -134,6 +134,21 @@ func TestProofRootDoesNotMintARefusedBuild(t *testing.T) {
}
}
+// D3-held fix round 1: a move's held refusal (MoveOp set) is scoped to
+// that move's held stage and never stops this box minting a proof for a
+// build it runs.
+func TestProofRootMintsPastAMovesHeldRefusal(t *testing.T) {
+ e := newProofRootEnv(t, provePolicy)
+ if err := release.WriteRefusal(e.bake, release.Refusal{Host: "held.example.com", App: "console", Version: "v1", SHA256: e.sha,
+ Kind: release.RefusalNotProven, MoveOp: "mv_abcdefghijklmnopqrstuvwxyz"}); err != nil {
+ t.Fatal(err)
+ }
+ e.root.run(context.Background())
+ if e.proof(t) == nil {
+ t.Fatalf("a move's held refusal stopped the mint; logs: %v", e.logs)
+ }
+}
+
func TestProofRootNeverOverwritesAProof(t *testing.T) {
e := newProofRootEnv(t, provePolicy)
first := e.signed(t, e.now.Add(-time.Minute))
diff --git a/internal/release/materializeheld.go b/internal/release/materializeheld.go
index 1c4d5bdd6..219f13c99 100644
--- a/internal/release/materializeheld.go
+++ b/internal/release/materializeheld.go
@@ -51,6 +51,10 @@ var (
// a variable so a test can make it fail the way a read-only mount does.
var materializeHeldStage = ensureStaged
+// heldBeforeSwap runs inside the publication between bind and the link
+// swap. A test seam: the window a replaced staged file lands in.
+var heldBeforeSwap = func() {}
+
// MaterializeHeld is StageHeld then PublishHeld: version of rt's app is
// staged, and live/<host> is pointed at it inside guard. A held row runs
// exactly the pinned release, so there is no channel resolution, no
@@ -135,7 +139,19 @@ func PublishHeld(ctx context.Context, s Store, optDir string, rt registry.Route,
return publishHeld(optDir, rt, version, proof, ptr, v, guard)
}
-// publishHeld is PublishHeld after admit: bind, then guard and PointLive.
+// publishHeld is PublishHeld after admit. Everything runs inside guard,
+// after the authority's re-check and under whatever it holds (for the root
+// relay, the registry's holds frozen): bind hashes the staged bytes, the
+// link is swapped, and bind runs again on the bytes the link now names. The
+// guard's re-check reads the bucket and waits for the freeze, and the
+// staged file can be replaced meanwhile or after the first hash, so only a
+// hash taken after the swap vouches for what the link points at. When that
+// second bind refuses, the link goes back to the version it named before
+// (or away, when there was none) and the stage is ErrHeldNotProven.
+//
+// A binding refusal is recorded scoped to the move (rt.HoldOp, Refusal's
+// MoveOp), never over an adoption refusal for the host; a publication that
+// succeeds retires its move's earlier refusal.
func publishHeld(optDir string, rt registry.Route, version string, proof *ProofGate, ptr *ChannelPointer, v proofVerdict, guard func(publish func() error) error) error {
if err := heldStageable(rt, version); err != nil {
return err
@@ -143,19 +159,107 @@ func publishHeld(optDir string, rt registry.Route, version string, proof *ProofG
if guard == nil {
return errHeldNoGuard
}
- cur, _ := LiveVersion(optDir, rt.Host)
- if !proof.bind(optDir, rt, ptr, v, cur, proof.now()) {
- return fmt.Errorf("%w: %s at %s: its staged bytes are not the proven ones", ErrHeldNotProven, rt.Host, version)
- }
- return guard(func() error {
+ record := heldRefusalWriter(proof.bakeDir, rt.HoldOp)
+ notProven := fmt.Errorf("%w: %s at %s: its staged bytes are not the proven ones", ErrHeldNotProven, rt.Host, version)
+ err := guard(func() error {
+ cur, err := LiveVersion(optDir, rt.Host)
+ if err != nil {
+ return err
+ }
+ if !proof.bindRecording(optDir, rt, ptr, v, cur, proof.now(), record) {
+ return notProven
+ }
+ heldBeforeSwap()
if err := PointLive(optDir, rt.Host, RouteAccount(rt), version, rt.App); err != nil {
if StageBlocked(err) {
return fmt.Errorf("%w: %w", ErrStageBlocked, err)
}
return err
}
+ if !proof.bindRecording(optDir, rt, ptr, v, cur, proof.now(), record) {
+ if rerr := restoreLive(optDir, rt, cur); rerr != nil {
+ return errors.Join(notProven, fmt.Errorf("putting live/%s back on %q: %w", rt.Host, cur, rerr))
+ }
+ return notProven
+ }
return nil
})
+ if err != nil {
+ return err
+ }
+ if proof.bakeDir != "" {
+ if err := RetireHeldRefusal(proof.bakeDir, rt.Host, rt.HoldOp); err != nil {
+ fmt.Printf(" %s: retiring move %s's held refusal: %v\n", rt.Host, rt.HoldOp, err)
+ }
+ }
+ return nil
+}
+
+// restoreLive puts live/<host> back on cur, or removes it when cur is "".
+func restoreLive(optDir string, rt registry.Route, cur string) error {
+ if cur == "" {
+ if err := os.Remove(LiveLink(optDir, rt.Host)); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ return nil
+ }
+ return PointLive(optDir, rt.Host, RouteAccount(rt), cur, rt.App)
+}
+
+// heldRefusalWriter is a held stage's refusal writer for bindRecording: the
+// record carries the move's op, and an adoption refusal standing for the
+// host (MoveOp "") is left as it is rather than overwritten.
+func heldRefusalWriter(bakeDir, op string) func(Refusal) error {
+ return func(r Refusal) error {
+ old, err := ReadRefusal(bakeDir, r.Host)
+ if err != nil {
+ return err
+ }
+ if old != nil && old.MoveOp == "" {
+ return fmt.Errorf("an adoption refusal for %s stands, so move %s's is not recorded over it", r.Host, op)
+ }
+ r.MoveOp = op
+ return WriteRefusal(bakeDir, r)
+ }
+}
+
+// RetireHeldRefusal removes host's refusal record when it is move op's held
+// refusal, and leaves any other record (another move's, or adoption's).
+func RetireHeldRefusal(bakeDir, host, op string) error {
+ if op == "" {
+ return nil
+ }
+ r, err := ReadRefusal(bakeDir, host)
+ if err != nil || r == nil || r.MoveOp != op {
+ return err
+ }
+ return ClearRefusal(bakeDir, host)
+}
+
+// RetireHeldRefusals removes every held refusal (MoveOp set) whose move
+// live says is over, and returns the hosts it removed. Adoption refusals
+// are never read past their MoveOp. With no refusal directory it reads
+// nothing. Errors from individual removals are joined; the rest go ahead.
+func RetireHeldRefusals(bakeDir string, live func(Refusal) bool) ([]string, error) {
+ recs, err := ListRefusals(bakeDir)
+ if err != nil {
+ return nil, err
+ }
+ var (
+ retired []string
+ errs []error
+ )
+ for _, r := range recs {
+ if r.MoveOp == "" || live(r) {
+ continue
+ }
+ if err := RetireHeldRefusal(bakeDir, r.Host, r.MoveOp); err != nil {
+ errs = append(errs, fmt.Errorf("%s: %w", r.Host, err))
+ continue
+ }
+ retired = append(retired, r.Host)
+ }
+ return retired, errors.Join(errs...)
}
// heldProofAdmit is D3's cheap check (ProofGate.admit, unchanged) for a
@@ -192,6 +296,9 @@ func heldStageable(rt registry.Route, version string) error {
if !controlname.ValidHostShape(rt.Host) || !ValidApp(rt.App) || !ValidVersion(version) {
return fmt.Errorf("release: held stage of %q/%q at %q: malformed", rt.Host, rt.App, version)
}
+ if rt.HoldOp == "" {
+ return fmt.Errorf("release: held stage of %q at %q names no move", rt.Host, version)
+ }
return nil
}
@@ -298,7 +405,7 @@ func serveHeldStageRequest(ctx context.Context, s Store, optDir, dataDir, reqDir
if err := authorize(req, nil); err != nil {
return drop("refused: %v", err)
}
- rt := registry.Route{Host: req.Host, Account: req.Account, App: req.App}
+ rt := registry.Route{Host: req.Host, Account: req.Account, App: req.App, HoldOp: req.Op}
guard := func(publish func() error) error { return authorize(req, publish) }
if _, err := MaterializeHeld(ctx, s, optDir, rt, req.Version, proof, guard); err != nil {
if errors.Is(err, ErrStaleHeldStage) {
diff --git a/internal/release/materializeheld_proof_test.go b/internal/release/materializeheld_proof_test.go
index 426ce872b..5b5ce10b5 100644
--- a/internal/release/materializeheld_proof_test.go
+++ b/internal/release/materializeheld_proof_test.go
@@ -215,8 +215,12 @@ func TestHeldStageProofBindingRefusesOtherStagedBytes(t *testing.T) {
if err := StageArtifact(e.opt, "bab", "v1", "console", stale); err != nil {
t.Fatal(err)
}
- err, guarded, out := e.materialize(t, e.store)
- e.refusedUntouched(t, err, guarded)
+ // bind runs inside the publication (fix round 1), so the guard is asked
+ // and the refusal comes from within it.
+ err, _, out := e.materialize(t, e.store)
+ if !errors.Is(err, ErrHeldNotProven) || e.live() != "v0" {
+ t.Fatalf("MaterializeHeld: %v, live %q; want ErrHeldNotProven and v0", err, e.live())
+ }
if !strings.Contains(out, "not pointed live") {
t.Fatalf("printed %q, want the binding refusal", out)
}
@@ -294,3 +298,166 @@ func TestServeHeldStageRequestsAppliesTheProofGate(t *testing.T) {
t.Fatalf("live = %q, want the proven v1", e.live())
}
}
+
+// materializeWith is materialize with the caller's guard.
+func (e *heldProofEnv) materializeWith(t *testing.T, guard func(publish func() error) error) (error, string) {
+ t.Helper()
+ var (
+ err error
+ out bytes.Buffer
+ )
+ withCapturedStdout(t, &out, func() {
+ _, err = MaterializeHeld(e.ctx, e.store, e.opt, heldRoute(), "v1", e.gate, guard)
+ })
+ return err, out.String()
+}
+
+// Fix round 1, finding 1: the authority's re-check (bucket reads, the
+// registry freeze) runs between staging and publication, and the staged
+// bytes can be replaced meanwhile. bind runs inside the publication, after
+// the re-check, so bytes replaced while the guard ran are refused.
+func TestHeldStageProofBindsInsideThePublication(t *testing.T) {
+ e := newHeldProofEnv(t)
+ e.require(t, "console")
+ e.prove(t, e.pinnedSHA)
+ stale := []byte("replaced while the guard re-checked")
+ err, _ := e.materializeWith(t, func(publish func() error) error {
+ if err := StageArtifact(e.opt, "bab", "v1", "console", stale); err != nil {
+ t.Fatal(err)
+ }
+ return publish()
+ })
+ if !errors.Is(err, ErrHeldNotProven) || e.live() != "v0" {
+ t.Fatalf("MaterializeHeld: %v, live %q; want ErrHeldNotProven and v0", err, e.live())
+ }
+}
+
+// Fix round 1, finding 1: bytes replaced between bind and the link swap
+// are caught by a re-hash at the swap, and the link goes back to what it
+// was (or away, when there was nothing).
+func TestHeldStageProofRehashesAtTheSwap(t *testing.T) {
+ for name, hadLive := range map[string]bool{"was on v0": true, "had no link": false} {
+ t.Run(name, func(t *testing.T) {
+ e := newHeldProofEnv(t)
+ if !hadLive {
+ if err := os.Remove(LiveLink(e.opt, heldTestHost)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ e.require(t, "console")
+ e.prove(t, e.pinnedSHA)
+ prev := heldBeforeSwap
+ t.Cleanup(func() { heldBeforeSwap = prev })
+ swapped := false
+ heldBeforeSwap = func() {
+ swapped = true
+ if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("replaced after bind")); err != nil {
+ t.Fatal(err)
+ }
+ }
+ err, guarded, _ := e.materialize(t, e.store)
+ if !swapped || !guarded {
+ t.Fatalf("the seam ran %v, the guard ran %v", swapped, guarded)
+ }
+ want := "v0"
+ if !hadLive {
+ want = ""
+ }
+ if !errors.Is(err, ErrHeldNotProven) || e.live() != want {
+ t.Fatalf("MaterializeHeld: %v, live %q; want ErrHeldNotProven and %q", err, e.live(), want)
+ }
+ })
+ }
+}
+
+// Fix round 1, finding 2: a held binding refusal carries its move's op.
+func TestHeldStageProofRefusalCarriesTheMove(t *testing.T) {
+ e := newHeldProofEnv(t)
+ e.require(t, "console")
+ e.prove(t, e.pinnedSHA)
+ if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("not the shipped bytes")); err != nil {
+ t.Fatal(err)
+ }
+ e.materialize(t, e.store)
+ r, _ := ReadRefusal(e.bake, heldTestHost)
+ if r == nil || r.MoveOp != heldTestOp || r.Kind != RefusalNotProven {
+ t.Fatalf("refusal = %+v, want a not-proven record naming op %s", r, heldTestOp)
+ }
+}
+
+// Fix round 1, finding 2: an adoption refusal already standing for the host
+// is never overwritten by a held one.
+func TestHeldStageProofLeavesAnAdoptionRefusal(t *testing.T) {
+ e := newHeldProofEnv(t)
+ e.require(t, "console")
+ e.prove(t, e.pinnedSHA)
+ adoption := Refusal{Host: heldTestHost, App: "console", Version: "v0", SHA256: "aa", Reason: "did not come up",
+ PointerPromotedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), RefusedAt: time.Date(2026, 10, 1, 0, 1, 0, 0, time.UTC)}
+ if err := WriteRefusal(e.bake, adoption); err != nil {
+ t.Fatal(err)
+ }
+ before, _ := os.ReadFile(filepath.Join(RefusalDir(e.bake), heldTestHost+".json"))
+ if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("not the shipped bytes")); err != nil {
+ t.Fatal(err)
+ }
+ err, _, _ := e.materialize(t, e.store)
+ if !errors.Is(err, ErrHeldNotProven) {
+ t.Fatalf("MaterializeHeld: %v, want ErrHeldNotProven", err)
+ }
+ after, _ := os.ReadFile(filepath.Join(RefusalDir(e.bake), heldTestHost+".json"))
+ if !bytes.Equal(before, after) {
+ t.Fatalf("the adoption refusal changed:\n%s\nwant\n%s", after, before)
+ }
+}
+
+// Fix round 1, finding 2: a later held stage of the same move that
+// succeeds retires that move's refusal.
+func TestHeldStageProofSuccessRetiresTheMovesRefusal(t *testing.T) {
+ e := newHeldProofEnv(t)
+ e.require(t, "console")
+ e.prove(t, e.pinnedSHA)
+ if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("not the shipped bytes")); err != nil {
+ t.Fatal(err)
+ }
+ e.materialize(t, e.store)
+ if r, _ := ReadRefusal(e.bake, heldTestHost); r == nil {
+ t.Fatal("no refusal to retire")
+ }
+ if err := StageArtifact(e.opt, "bab", "v1", "console", []byte("pinned build")); err != nil {
+ t.Fatal(err)
+ }
+ if err, _, _ := e.materialize(t, e.store); err != nil || e.live() != "v1" {
+ t.Fatalf("retry: %v, live %q", err, e.live())
+ }
+ if r, _ := ReadRefusal(e.bake, heldTestHost); r != nil {
+ t.Fatalf("the move's refusal outlived its successful retry: %+v", r)
+ }
+}
+
+// RetireHeldRefusals removes a held refusal whose move is over and nothing
+// else: not a live move's, never an adoption refusal.
+func TestRetireHeldRefusals(t *testing.T) {
+ bake := t.TempDir()
+ at := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
+ for _, r := range []Refusal{
+ {Host: "live.example.com", App: "a", Version: "v1", RefusedAt: at, Kind: RefusalNotProven, MoveOp: "mv_live"},
+ {Host: "over.example.com", App: "a", Version: "v1", RefusedAt: at, Kind: RefusalNotProven, MoveOp: "mv_over"},
+ {Host: "adopted.example.com", App: "a", Version: "v1", RefusedAt: at, Kind: RefusalNotProven},
+ } {
+ if err := WriteRefusal(bake, r); err != nil {
+ t.Fatal(err)
+ }
+ }
+ retired, err := RetireHeldRefusals(bake, func(r Refusal) bool { return r.MoveOp == "mv_live" })
+ if err != nil || len(retired) != 1 || retired[0] != "over.example.com" {
+ t.Fatalf("retired %v, %v; want over.example.com", retired, err)
+ }
+ left, _ := ListRefusals(bake)
+ if len(left) != 2 || left[0].Host != "adopted.example.com" || left[1].Host != "live.example.com" {
+ t.Fatalf("left %+v", left)
+ }
+ // With no refusal directory it reads nothing and fails nothing.
+ if retired, err := RetireHeldRefusals(t.TempDir(), func(Refusal) bool { return false }); err != nil || len(retired) != 0 {
+ t.Fatalf("empty: %v, %v", retired, err)
+ }
+}
diff --git a/internal/release/proofgate.go b/internal/release/proofgate.go
index 2245770c4..470b411ef 100644
--- a/internal/release/proofgate.go
+++ b/internal/release/proofgate.go
@@ -323,6 +323,13 @@ func (g *ProofGate) manifestSHA(ctx context.Context, s Store, optDir, acct, app
// nothing was restarted, so it is Recovered and still on cur, the same shape
// as D2's wrong-program record.
func (g *ProofGate) bind(optDir string, rt registry.Route, ptr *ChannelPointer, v proofVerdict, cur string, now time.Time) bool {
+ return g.bindRecording(optDir, rt, ptr, v, cur, now, func(r Refusal) error { return WriteRefusal(g.bakeDir, r) })
+}
+
+// bindRecording is bind with the refusal record's writer as a parameter:
+// adoption's bind writes it with WriteRefusal, and a move's held stage
+// (materializeheld.go) scopes it to the move first.
+func (g *ProofGate) bindRecording(optDir string, rt registry.Route, ptr *ChannelPointer, v proofVerdict, cur string, now time.Time, record func(Refusal) error) bool {
if g == nil || !v.required {
return true
}
@@ -344,7 +351,7 @@ func (g *ProofGate) bind(optDir string, rt registry.Route, ptr *ChannelPointer,
}
fmt.Printf(" %s: %s: the staged bytes do not match the proof (%s); not pointed live\n", rt.Host, ptr.Version, reason)
if g.bakeDir != "" {
- if err := WriteRefusal(g.bakeDir, Refusal{Host: rt.Host, App: rt.App, Version: ptr.Version, SHA256: got,
+ if err := record(Refusal{Host: rt.Host, App: rt.App, Version: ptr.Version, SHA256: got,
Platform: adoptPlatform, PointerPromotedAt: ptr.PromotedAt, RefusedAt: now.UTC(),
Reason: "not proven: " + reason,
Kind: RefusalNotProven, Recovered: true, RestoredVersion: cur}); err != nil {
diff --git a/internal/release/refusal.go b/internal/release/refusal.go
index 2dc4691b8..87c06b394 100644
--- a/internal/release/refusal.go
+++ b/internal/release/refusal.go
@@ -66,6 +66,12 @@ type Refusal struct {
RestoredVersion string `json:"restored_version,omitempty"`
Recovered bool `json:"recovered"`
RecoveryError string `json:"recovery_error,omitempty"`
+ // MoveOp is set only on a move's held-stage refusal (materializeheld.go,
+ // ruling D3-held): the op of the move whose held stage was refused. It
+ // scopes the record to that move, which retires it (a later held stage
+ // of the move that succeeds, or the move ending), and it is never set on
+ // an adoption refusal, so adoption's records are byte-identical.
+ MoveOp string `json:"move_op,omitempty"`
}
func RefusalDir(bakeDir string) string { return filepath.Join(bakeDir, "refusals") }