| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "runtime" |
| 13 | "sort" |
| 14 | "strings" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/moves" |
| 20 | "github.com/carlosframework/platform/internal/registry" |
| 21 | "github.com/carlosframework/platform/internal/release" |
| 22 | ) |
| 23 | |
| 24 | // offProofGate is D3's gate on a box with no proof policy: the held stage |
| 25 | // behaves exactly as it did before D3-held. |
| 26 | func offProofGate(t *testing.T) *release.ProofGate { |
| 27 | t.Helper() |
| 28 | return release.NewProofGate(filepath.Join(t.TempDir(), "proof-policy.json"), t.TempDir()) |
| 29 | } |
| 30 | |
| 31 | // heldProofBox is a target box with a proof policy file and its bake |
| 32 | // directory, for T7 under D3-held. |
| 33 | type heldProofBox struct { |
| 34 | policy, bake string |
| 35 | prover ed25519.PrivateKey |
| 36 | proverPub string |
| 37 | } |
| 38 | |
| 39 | func newHeldProofBox(t *testing.T) *heldProofBox { |
| 40 | t.Helper() |
| 41 | seed, pub, err := release.GenerateKey() |
| 42 | if err != nil { |
| 43 | t.Fatal(err) |
| 44 | } |
| 45 | priv, err := release.ParseSigningKey(seed, "test") |
| 46 | if err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | return &heldProofBox{policy: filepath.Join(t.TempDir(), "proof-policy.json"), bake: t.TempDir(), prover: priv, proverPub: pub} |
| 50 | } |
| 51 | |
| 52 | func (b *heldProofBox) require(t *testing.T, app string) { |
| 53 | t.Helper() |
| 54 | body := fmt.Sprintf(`{"schema":1,"require":[{"app":%q,"prover":"dev","key":%q}]}`, app, b.proverPub) |
| 55 | if err := os.WriteFile(b.policy, []byte(body), 0o644); err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | func (b *heldProofBox) gate() *release.ProofGate { return release.NewProofGate(b.policy, b.bake) } |
| 61 | |
| 62 | func (b *heldProofBox) prove(t *testing.T, s release.Store, app string, body []byte) { |
| 63 | t.Helper() |
| 64 | sum := sha256.Sum256(body) |
| 65 | p, err := release.SignProof(b.prover, release.Proof{App: app, Platform: release.AdoptPlatform(), SHA256: hex.EncodeToString(sum[:]), |
| 66 | Version: "v7", Prover: "dev", ActiveSince: time.Now().Add(-time.Hour).UTC().Truncate(time.Second), ProvenAt: time.Now().UTC().Truncate(time.Second)}) |
| 67 | if err != nil { |
| 68 | t.Fatal(err) |
| 69 | } |
| 70 | if err := release.PutProof(context.Background(), s, "bab", p); err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | // shipPinnedV7 ships bab/app v7 for this platform and promotes it to |
| 76 | // canary/x, so a held stage of it is authorized. |
| 77 | func shipPinnedV7(t *testing.T, s release.Store) { |
| 78 | t.Helper() |
| 79 | plat := runtime.GOOS + "-" + runtime.GOARCH |
| 80 | if err := release.Ship(context.Background(), s, "bab", release.Manifest{App: "app", Version: "v7", Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}}, |
| 81 | map[string][]byte{plat: []byte("pinned")}); err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | promoteHeld(t, s, "app", "v7") |
| 85 | } |
| 86 | |
| 87 | // D3-held, through the target's report: on a box whose policy lists the |
| 88 | // app, an unproven pinned release is T7's not-proven sentence in move |
| 89 | // status, nothing is staged or relayed and live/<host> is untouched. With |
| 90 | // a proof it goes live; with a policy that lists another app T7 is |
| 91 | // unchanged. |
| 92 | func TestHeldReleaseReadyAppliesTheProofGate(t *testing.T) { |
| 93 | t.Parallel() |
| 94 | ctx := context.Background() |
| 95 | |
| 96 | t.Run("covered and unproven is refused", func(t *testing.T) { |
| 97 | h := newPhase1Harness(t) |
| 98 | h.start(t, "manual") |
| 99 | shipPinnedV7(t, h.s) |
| 100 | h.pass(t) |
| 101 | box := newHeldProofBox(t) |
| 102 | box.require(t, "app") |
| 103 | data, opt := t.TempDir(), t.TempDir() |
| 104 | tc := h.targetCfg() |
| 105 | tc.DataDir = data |
| 106 | tc.ReleaseReady = heldReleaseReady(testRegistry(t), opt, data, "uk-box", hsPub, hsRatchet(t), box.gate(), stageRequester(data)) |
| 107 | h.targetPass(t, tc) |
| 108 | if f := failing(h.targetReport(t)); f[moves.CheckRelease] != t7NotProven { |
| 109 | t.Fatalf("T7 = %q, want %q", f[moves.CheckRelease], t7NotProven) |
| 110 | } |
| 111 | if v, _ := release.LiveVersion(opt, p1Host); v != "" { |
| 112 | t.Fatalf("live = %q, want no live link", v) |
| 113 | } |
| 114 | if _, err := os.Stat(filepath.Join(opt, "versions")); !os.IsNotExist(err) { |
| 115 | t.Fatalf("an unproven release was staged: %v", err) |
| 116 | } |
| 117 | if _, err := os.Stat(filepath.Join(release.HeldStageDir(data), p1Host+".json")); !os.IsNotExist(err) { |
| 118 | t.Fatalf("an unproven release was relayed to the root pass: %v", err) |
| 119 | } |
| 120 | }) |
| 121 | |
| 122 | t.Run("covered and proven publishes", func(t *testing.T) { |
| 123 | h := newPhase1Harness(t) |
| 124 | rec := h.start(t, "manual") |
| 125 | shipPinnedV7(t, h.s) |
| 126 | h.pass(t) |
| 127 | box := newHeldProofBox(t) |
| 128 | box.require(t, "app") |
| 129 | box.prove(t, h.s, "app", []byte("pinned")) |
| 130 | opt := t.TempDir() |
| 131 | if c := heldReleaseReady(testRegistry(t), opt, t.TempDir(), "uk-box", hsPub, hsRatchet(t), box.gate(), nil)(ctx, h.s, rec, "v7"); !c.OK { |
| 132 | t.Fatalf("T7 = %+v", c) |
| 133 | } |
| 134 | if v, _ := release.LiveVersion(opt, p1Host); v != "v7" { |
| 135 | t.Fatalf("live = %q, want v7", v) |
| 136 | } |
| 137 | }) |
| 138 | |
| 139 | t.Run("uncovered is unchanged", func(t *testing.T) { |
| 140 | h := newPhase1Harness(t) |
| 141 | rec := h.start(t, "manual") |
| 142 | shipPinnedV7(t, h.s) |
| 143 | h.pass(t) |
| 144 | box := newHeldProofBox(t) |
| 145 | box.require(t, "other") |
| 146 | opt := t.TempDir() |
| 147 | if c := heldReleaseReady(testRegistry(t), opt, t.TempDir(), "uk-box", hsPub, hsRatchet(t), box.gate(), nil)(ctx, h.s, rec, "v7"); !c.OK { |
| 148 | t.Fatalf("T7 = %+v", c) |
| 149 | } |
| 150 | if v, _ := release.LiveVersion(opt, p1Host); v != "v7" { |
| 151 | t.Fatalf("live = %q, want v7", v) |
| 152 | } |
| 153 | }) |
| 154 | } |
| 155 | |
| 156 | // heldRefusalFixture writes three refusal records under bake: a held one |
| 157 | // for hsOp at hsHost, a held one for another move at another host (both in |
| 158 | // the held namespace), and an adoption refusal (no MoveOp) at a third host. |
| 159 | func heldRefusalFixture(t *testing.T, bake string) { |
| 160 | t.Helper() |
| 161 | at := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 162 | for _, r := range []release.Refusal{ |
| 163 | {Host: hsHost, App: "titogo", Version: "v7", RefusedAt: at, Kind: release.RefusalNotProven, MoveOp: hsOp}, |
| 164 | {Host: "other.example.com", App: "titogo", Version: "v7", RefusedAt: at, Kind: release.RefusalNotProven, MoveOp: hsOther}, |
| 165 | } { |
| 166 | if err := release.WriteRefusal(release.HeldRefusalBase(bake), r); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | } |
| 170 | if err := release.WriteRefusal(bake, release.Refusal{Host: "adopted.example.com", App: "titogo", Version: "v7", RefusedAt: at, Kind: release.RefusalNotProven}); err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // refusalHosts lists the hosts with a refusal record in either namespace, |
| 176 | // sorted. |
| 177 | func refusalHosts(t *testing.T, bake string) string { |
| 178 | t.Helper() |
| 179 | recs, err := release.ListRefusals(bake) |
| 180 | if err != nil { |
| 181 | t.Fatal(err) |
| 182 | } |
| 183 | held, err := release.ListHeldRefusals(bake) |
| 184 | if err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | var hosts []string |
| 188 | for _, r := range append(recs, held...) { |
| 189 | hosts = append(hosts, r.Host) |
| 190 | } |
| 191 | sort.Strings(hosts) |
| 192 | return strings.Join(hosts, ",") |
| 193 | } |
| 194 | |
| 195 | // Fix round 1, finding 2, the agent's half: when a move stops being one |
| 196 | // this box prepares (aborted, abandoned, or past ready), the target's pass |
| 197 | // retires its held refusal with the rest of the move's staging state. A |
| 198 | // live move's refusal and an adoption refusal stay. |
| 199 | func TestTargetPassRetiresAnEndedMovesHeldRefusal(t *testing.T) { |
| 200 | data := t.TempDir() |
| 201 | heldRefusalFixture(t, data) |
| 202 | w := &movesWorker{cfg: moveCfg{DataDir: data}} |
| 203 | w.retireHeldRefusals(map[string]bool{hsOp: true}) |
| 204 | if got := refusalHosts(t, data); got != "adopted.example.com,"+hsHost { |
| 205 | t.Fatalf("left %q, want the adoption refusal and the live move's", got) |
| 206 | } |
| 207 | w.retireHeldRefusals(nil) // no move into this box any more |
| 208 | if got := refusalHosts(t, data); got != "adopted.example.com" { |
| 209 | t.Fatalf("left %q, want only the adoption refusal", got) |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | // Fix round 1, finding 2, root's half: the agent cannot remove a record |
| 214 | // root wrote, so the root stage pass retires every held refusal whose move |
| 215 | // the host claim no longer names as moving or landing into this box. A |
| 216 | // claim it cannot read keeps the record; an adoption refusal is never read |
| 217 | // past its MoveOp. |
| 218 | func TestRootRetiresAnAbandonedMovesHeldRefusal(t *testing.T) { |
| 219 | ctx := context.Background() |
| 220 | s := release.NewDirStore(t.TempDir()) |
| 221 | data := t.TempDir() |
| 222 | heldRefusalFixture(t, data) |
| 223 | hsClaim(t, s, movingOp()) // hsHost: still moving into hsBox under hsOp |
| 224 | // other.example.com has no claim at all: its move is over. |
| 225 | retireEndedHeldRefusals(ctx, s, data, hsBox) |
| 226 | if got := refusalHosts(t, data); got != "adopted.example.com,"+hsHost { |
| 227 | t.Fatalf("left %q, want the adoption refusal and the live move's", got) |
| 228 | } |
| 229 | |
| 230 | // The move is abandoned: the claim goes back to serving. |
| 231 | obs, err := hostclaim.Read(ctx, s, hsHost) |
| 232 | if err != nil { |
| 233 | t.Fatal(err) |
| 234 | } |
| 235 | if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseMove(hsOp, "ie-box")); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | retireEndedHeldRefusals(ctx, s, data, hsBox) |
| 239 | if got := refusalHosts(t, data); got != "adopted.example.com" { |
| 240 | t.Fatalf("left %q, want only the adoption refusal", got) |
| 241 | } |
| 242 | |
| 243 | // A claim read that fails keeps the record. |
| 244 | heldRefusalFixture(t, data) |
| 245 | retireEndedHeldRefusals(ctx, failingClaimStore{s}, data, hsBox) |
| 246 | if got := refusalHosts(t, data); got != "adopted.example.com,"+hsHost+",other.example.com" { |
| 247 | t.Fatalf("left %q after failed claim reads, want all three", got) |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | type failingClaimStore struct{ release.Store } |
| 252 | |
| 253 | func (failingClaimStore) Get(context.Context, string) ([]byte, string, error) { |
| 254 | return nil, "", errors.New("bucket unreachable") |
| 255 | } |
| 256 | |
| 257 | // Final review finding 1, through heldReleaseReady: an existing link is |
| 258 | // re-checked every time. Bytes replaced after the publication, or a proof |
| 259 | // policy that no longer accepts them, fail T7 with the not-proven sentence |
| 260 | // and nothing is restaged or republished. |
| 261 | func TestT7RechecksAnExistingLink(t *testing.T) { |
| 262 | t.Parallel() |
| 263 | ctx := context.Background() |
| 264 | h := newPhase1Harness(t) |
| 265 | rec := h.start(t, "manual") |
| 266 | shipPinnedV7(t, h.s) |
| 267 | h.pass(t) |
| 268 | box := newHeldProofBox(t) |
| 269 | box.require(t, "app") |
| 270 | box.prove(t, h.s, "app", []byte("pinned")) |
| 271 | reg, data, opt := testRegistry(t), t.TempDir(), t.TempDir() |
| 272 | t7 := func() moves.Check { |
| 273 | return heldReleaseReady(reg, opt, data, "uk-box", hsPub, hsRatchet(t), box.gate(), nil)(ctx, h.s, rec, "v7") |
| 274 | } |
| 275 | if c := t7(); !c.OK { |
| 276 | t.Fatalf("T7 = %+v", c) |
| 277 | } |
| 278 | if c := t7(); !c.OK { |
| 279 | t.Fatalf("T7 on the published link = %+v", c) |
| 280 | } |
| 281 | path := release.VersionPath(opt, "bab", "v7", "app") |
| 282 | if err := os.WriteFile(path, []byte("replaced after publication"), 0o755); err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | if c := t7(); c.OK || c.Reason != t7NotProven { |
| 286 | t.Fatalf("T7 with replaced bytes = %+v, want %q", c, t7NotProven) |
| 287 | } |
| 288 | if got, _ := os.ReadFile(path); string(got) != "replaced after publication" { |
| 289 | t.Fatalf("T7 restaged the release: %q", got) |
| 290 | } |
| 291 | if err := os.WriteFile(path, []byte("pinned"), 0o755); err != nil { |
| 292 | t.Fatal(err) |
| 293 | } |
| 294 | if c := t7(); !c.OK { |
| 295 | t.Fatalf("T7 with the bytes restored = %+v", c) |
| 296 | } |
| 297 | // The policy rotates the prover's key: the proof no longer verifies. |
| 298 | other := newHeldProofBox(t) |
| 299 | other.policy = box.policy |
| 300 | other.require(t, "app") |
| 301 | if c := t7(); c.OK || c.Reason != t7NotProven { |
| 302 | t.Fatalf("T7 under a rotated key = %+v, want %q", c, t7NotProven) |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | // Final review finding 1, the review's interleaving with the real lock: a |
| 307 | // publication holds host's held-stage lock while live/<host> names v7, and |
| 308 | // then rolls the link back. T7, read inside that window, waits for the |
| 309 | // lock, finds the link back on v0 and does not pass (here the stage |
| 310 | // relays, so it is waiting for the root stage pass). |
| 311 | func TestT7WaitsOutAPublicationsRollback(t *testing.T) { |
| 312 | ctx := context.Background() |
| 313 | h := newPhase1Harness(t) |
| 314 | rec := h.start(t, "manual") |
| 315 | shipPinnedV7(t, h.s) |
| 316 | h.pass(t) |
| 317 | box := newHeldProofBox(t) |
| 318 | box.require(t, "app") |
| 319 | box.prove(t, h.s, "app", []byte("pinned")) |
| 320 | reg, data, opt := testRegistry(t), t.TempDir(), t.TempDir() |
| 321 | for v, body := range map[string]string{"v0": "earlier", "v7": "pinned"} { |
| 322 | if err := release.StageArtifact(opt, "bab", v, "app", []byte(body)); err != nil { |
| 323 | t.Fatal(err) |
| 324 | } |
| 325 | } |
| 326 | if err := release.PointLive(opt, p1Host, "bab", "v0", "app"); err != nil { |
| 327 | t.Fatal(err) |
| 328 | } |
| 329 | prev := heldMaterialize |
| 330 | t.Cleanup(func() { heldMaterialize = prev }) |
| 331 | heldMaterialize = func(context.Context, release.Store, ed25519.PublicKey, *release.DigestRatchet, string, registry.Route, string, *release.ProofGate, func(func() error) error) (*release.Manifest, error) { |
| 332 | return nil, fmt.Errorf("%w: read-only", release.ErrStageBlocked) |
| 333 | } |
| 334 | swapped, rolledBack := make(chan struct{}), make(chan time.Time, 1) |
| 335 | go func() { |
| 336 | err := withHeldStageLock(data, p1Host, func() error { |
| 337 | if err := release.PointLive(opt, p1Host, "bab", "v7", "app"); err != nil { |
| 338 | return err |
| 339 | } |
| 340 | close(swapped) |
| 341 | time.Sleep(300 * time.Millisecond) |
| 342 | err := release.PointLive(opt, p1Host, "bab", "v0", "app") |
| 343 | rolledBack <- time.Now() |
| 344 | return err |
| 345 | }) |
| 346 | if err != nil { |
| 347 | t.Error(err) |
| 348 | } |
| 349 | }() |
| 350 | <-swapped |
| 351 | c := heldReleaseReady(reg, opt, data, "uk-box", hsPub, hsRatchet(t), box.gate(), stageRequester(data))(ctx, h.s, rec, "v7") |
| 352 | answered := time.Now() |
| 353 | if c.OK { |
| 354 | t.Fatalf("T7 passed on a link the publication rolled back: %+v", c) |
| 355 | } |
| 356 | if at := <-rolledBack; answered.Before(at) { |
| 357 | t.Fatalf("T7 answered %v before the rollback at %v: it did not wait for the lock", answered, at) |
| 358 | } |
| 359 | if c.Reason != t7Waiting { |
| 360 | t.Fatalf("T7 = %+v, want %q", c, t7Waiting) |
| 361 | } |
| 362 | } |
| 363 | |
amadan