| 1 | package release |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ed25519" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "sync" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | ) |
| 19 | |
| 20 | const proofHost = "console.example.com" |
| 21 | |
| 22 | // proofEnv is startUnitFixture (live "old", channel canary/x promoted to |
| 23 | // "new" whose bytes are "bad build") plus a prover key, a policy path, a bake |
| 24 | // directory and one gate shared by every pass, as the agent shares one. It |
| 25 | // keeps the release key too, so a test can promote again. |
| 26 | type proofEnv struct { |
| 27 | ctx context.Context |
| 28 | reg *registry.Registry |
| 29 | store Store |
| 30 | priv ed25519.PrivateKey |
| 31 | pub ed25519.PublicKey |
| 32 | opt, bake string |
| 33 | policy string |
| 34 | prover ed25519.PrivateKey |
| 35 | proverPub string |
| 36 | newSHA string |
| 37 | clock time.Time |
| 38 | gate *ProofGate |
| 39 | } |
| 40 | |
| 41 | func newProofEnv(t *testing.T) *proofEnv { |
| 42 | t.Helper() |
| 43 | // startUnitFixture's setup, inlined so the release key is kept. |
| 44 | s := NewDirStore(t.TempDir()) |
| 45 | ctx := context.Background() |
| 46 | priv, pub := testKeys(t) |
| 47 | opt := t.TempDir() |
| 48 | reg := testRegistry(t) |
| 49 | shipRelease(t, s, "bab", "console", "old", adoptPlatform, []byte("good build")) |
| 50 | shipRelease(t, s, "bab", "console", "new", adoptPlatform, []byte("bad build")) |
| 51 | if err := StageArtifact(opt, "bab", "old", "console", []byte("good build")); err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | addRoute(t, reg, proofHost, "console", "canary/x", false) |
| 55 | if err := PointLive(opt, proofHost, "bab", "old", "console"); err != nil { |
| 56 | t.Fatal(err) |
| 57 | } |
| 58 | if err := Promote(ctx, s, priv, "bab", "console", "canary/x", "new"); err != nil { |
| 59 | t.Fatalf("Promote new: %v", err) |
| 60 | } |
| 61 | prover, _, pubB64 := proverKey(t) |
| 62 | e := &proofEnv{ctx: ctx, reg: reg, store: s, priv: priv, pub: pub, opt: opt, bake: t.TempDir(), |
| 63 | policy: filepath.Join(t.TempDir(), "proof-policy.json"), prover: prover, proverPub: pubB64, |
| 64 | newSHA: etagOf([]byte("bad build")), clock: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)} |
| 65 | e.gate = NewProofGate(e.policy, e.bake).WithClock(func() time.Time { return e.clock }) |
| 66 | return e |
| 67 | } |
| 68 | |
| 69 | func (e *proofEnv) writePolicy(t *testing.T, body string) { |
| 70 | t.Helper() |
| 71 | if err := os.WriteFile(e.policy, []byte(body), 0o644); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | func (e *proofEnv) requireDev(t *testing.T) { |
| 77 | e.writePolicy(t, fmt.Sprintf(`{"schema":1,"require":[{"app":"console","prover":"dev","key":%q}]}`, e.proverPub)) |
| 78 | } |
| 79 | |
| 80 | func (e *proofEnv) pass(t *testing.T, s Store) string { |
| 81 | t.Helper() |
| 82 | return e.passWith(t, s, func(string) error { return nil }) |
| 83 | } |
| 84 | |
| 85 | func (e *proofEnv) passWith(t *testing.T, s Store, restart func(string) error) string { |
| 86 | t.Helper() |
| 87 | var out bytes.Buffer |
| 88 | withCapturedStdout(t, &out, func() { |
| 89 | if err := Pass(e.ctx, e.reg, s, e.pub, e.opt, restart, |
| 90 | BakeDir(e.bake), WithProofGate(e.gate)); err != nil { |
| 91 | t.Fatalf("Pass: %v", err) |
| 92 | } |
| 93 | }) |
| 94 | return out.String() |
| 95 | } |
| 96 | |
| 97 | func (e *proofEnv) live() string { v, _ := LiveVersion(e.opt, proofHost); return v } |
| 98 | |
| 99 | func (e *proofEnv) signed(t *testing.T, key ed25519.PrivateKey, app, sha, prover string) *Proof { |
| 100 | t.Helper() |
| 101 | p, err := SignProof(key, Proof{App: app, Platform: adoptPlatform, SHA256: sha, Version: "new", Prover: prover, |
| 102 | ActiveSince: e.clock.Add(-time.Hour), ProvenAt: e.clock}) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | return p |
| 107 | } |
| 108 | |
| 109 | // putAt writes p at the dev key for sha, whatever p itself says. |
| 110 | func (e *proofEnv) putAt(t *testing.T, sha string, p *Proof) { |
| 111 | t.Helper() |
| 112 | b, err := json.MarshalIndent(p, "", " ") |
| 113 | if err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | if _, err := e.store.Put(e.ctx, ProofKey("bab", "console", sha, "dev"), b, ""); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | // manifestDownStore answers every other key from the wrapped store but fails |
| 122 | // reads of one manifest with an error that is not a not-found, as a bucket |
| 123 | // that is unreachable for that object would. |
| 124 | type manifestDownStore struct { |
| 125 | Store |
| 126 | key string |
| 127 | } |
| 128 | |
| 129 | func (m manifestDownStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 130 | if key == m.key { |
| 131 | return nil, "", errors.New("bucket unreachable for the manifest") |
| 132 | } |
| 133 | return m.Store.Get(ctx, key) |
| 134 | } |
| 135 | |
| 136 | // stageStaleCopyKept stages stale as the binary of acct/app/ver, with the |
| 137 | // canonical manifest (the store's exact bytes) as its sibling, and returns s |
| 138 | // wrapped so that manifest cannot be read from the bucket. |
| 139 | // |
| 140 | // This is how stale bytes reach the binding check since staging hashes a copy |
| 141 | // it reuses (spec section 4.2): an unreferenced copy that differs from its |
| 142 | // release is restaged, so the check only still sees different bytes where the |
| 143 | // canonical manifest cannot be read, a case ensureStagedChecked leaves as |
| 144 | // staged. The sibling lets the proof gate still learn the manifest's sha256. |
| 145 | func stageStaleCopyKept(t *testing.T, ctx context.Context, s Store, opt, acct, app, ver string, stale []byte) Store { |
| 146 | t.Helper() |
| 147 | key := ManifestKey(acct, app, ver) |
| 148 | raw, _, err := s.Get(ctx, key) |
| 149 | if err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | if err := StageArtifact(opt, acct, ver, app, stale); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | if err := StageManifest(opt, acct, ver, app, raw); err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | return manifestDownStore{Store: s, key: key} |
| 159 | } |
| 160 | |
| 161 | type proofReadCountingStore struct { |
| 162 | Store |
| 163 | mu sync.Mutex |
| 164 | gets []string |
| 165 | } |
| 166 | |
| 167 | func (c *proofReadCountingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 168 | c.mu.Lock() |
| 169 | c.gets = append(c.gets, key) |
| 170 | c.mu.Unlock() |
| 171 | return c.Store.Get(ctx, key) |
| 172 | } |
| 173 | |
| 174 | func (c *proofReadCountingStore) proofGets() int { |
| 175 | c.mu.Lock() |
| 176 | defer c.mu.Unlock() |
| 177 | n := 0 |
| 178 | for _, k := range c.gets { |
| 179 | if strings.Contains(k, "/proofs/") { |
| 180 | n++ |
| 181 | } |
| 182 | } |
| 183 | return n |
| 184 | } |
| 185 | |
| 186 | func TestProofNoPolicyMakesNoProofRead(t *testing.T) { |
| 187 | e := newProofEnv(t) |
| 188 | cs := &proofReadCountingStore{Store: e.store} |
| 189 | e.pass(t, cs) |
| 190 | if e.live() != "new" { |
| 191 | t.Fatalf("live = %q, want new: with no policy file adoption is unchanged", e.live()) |
| 192 | } |
| 193 | if n := cs.proofGets(); n != 0 { |
| 194 | t.Fatalf("%d proof GETs with no policy file, want 0 (spec §4.6)", n) |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | func TestProofMissingStaysAndIsNotARefusal(t *testing.T) { |
| 199 | e := newProofEnv(t) |
| 200 | e.requireDev(t) |
| 201 | out := e.pass(t, e.store) |
| 202 | if e.live() != "old" { |
| 203 | t.Fatalf("live = %q, want old", e.live()) |
| 204 | } |
| 205 | if !strings.Contains(out, "is not proven on dev; staying on old") { |
| 206 | t.Fatalf("printed %q, want the not-proven line", out) |
| 207 | } |
| 208 | if r, _ := ReadRefusal(e.bake, proofHost); r != nil { |
| 209 | t.Fatalf("a missing proof wrote a refusal: %+v", r) |
| 210 | } |
| 211 | w := e.gate.Waits() |
| 212 | if len(w) != 1 || w[0].Host != proofHost || w[0].SHA256 != e.newSHA || w[0].Prover != "dev" { |
| 213 | t.Fatalf("waits = %+v", w) |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | func TestProofValidAdopts(t *testing.T) { |
| 218 | e := newProofEnv(t) |
| 219 | e.requireDev(t) |
| 220 | e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev")) |
| 221 | e.pass(t, e.store) |
| 222 | if e.live() != "new" { |
| 223 | t.Fatalf("live = %q, want new", e.live()) |
| 224 | } |
| 225 | if w := e.gate.Waits(); len(w) != 0 { |
| 226 | t.Fatalf("waits = %+v after adopting", w) |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | func TestProofRejectsBadProofs(t *testing.T) { |
| 231 | for name, mk := range map[string]func(t *testing.T, e *proofEnv) *Proof{ |
| 232 | "bad signature": func(t *testing.T, e *proofEnv) *Proof { |
| 233 | other, _, _ := proverKey(t) |
| 234 | return e.signed(t, other, "console", e.newSHA, "dev") |
| 235 | }, |
| 236 | "wrong sha256": func(t *testing.T, e *proofEnv) *Proof { |
| 237 | return e.signed(t, e.prover, "console", etagOf([]byte("other bytes")), "dev") |
| 238 | }, |
| 239 | "wrong app": func(t *testing.T, e *proofEnv) *Proof { |
| 240 | return e.signed(t, e.prover, "other", e.newSHA, "dev") |
| 241 | }, |
| 242 | "wrong prover": func(t *testing.T, e *proofEnv) *Proof { |
| 243 | return e.signed(t, e.prover, "console", e.newSHA, "staging") |
| 244 | }, |
| 245 | } { |
| 246 | t.Run(name, func(t *testing.T) { |
| 247 | e := newProofEnv(t) |
| 248 | e.requireDev(t) |
| 249 | e.putAt(t, e.newSHA, mk(t, e)) |
| 250 | out := e.pass(t, e.store) |
| 251 | if e.live() != "old" || !strings.Contains(out, "is not proven on dev") { |
| 252 | t.Fatalf("live = %q, printed %q; want old and the not-proven line", e.live(), out) |
| 253 | } |
| 254 | }) |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // Review focus 3: rolling back to a build this box ran never waits on dev. |
| 259 | func TestProofHealthyHistoryAdmitsWithoutProof(t *testing.T) { |
| 260 | e := newProofEnv(t) |
| 261 | e.requireDev(t) |
| 262 | if err := RecordHealthy(e.bake, "bab", "console", HealthyBuild{SHA256: e.newSHA, Version: "new", How: "settle", At: e.clock}); err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | cs := &proofReadCountingStore{Store: e.store} |
| 266 | e.pass(t, cs) |
| 267 | if e.live() != "new" { |
| 268 | t.Fatalf("live = %q, want new from the healthy history", e.live()) |
| 269 | } |
| 270 | if n := cs.proofGets(); n != 0 { |
| 271 | t.Fatalf("%d proof GETs for a build in the healthy history, want 0", n) |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | // Review focus 2: the manifest's sha256 is proven, but the staged file holds |
| 276 | // other bytes. They reach the binding check through a kept stale copy: the |
| 277 | // bucket cannot answer for the manifest, so staging leaves the copy as staged |
| 278 | // (stageStaleCopyKept). |
| 279 | // |
| 280 | // Ruling 3: the record is its own kind, recovered and still on the old |
| 281 | // build, because nothing was restarted. The next pass's gate line says the |
| 282 | // staged bytes do not match the proof and never suggests --retry, which |
| 283 | // would re-sign the pointer over the same stale bytes and be refused again. |
| 284 | func TestProofBindingRefusesStagedBytesThatDiffer(t *testing.T) { |
| 285 | e := newProofEnv(t) |
| 286 | e.requireDev(t) |
| 287 | stale := []byte("not the shipped bytes") |
| 288 | down := stageStaleCopyKept(t, e.ctx, e.store, e.opt, "bab", "console", "new", stale) |
| 289 | e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev")) |
| 290 | out := e.pass(t, down) |
| 291 | if e.live() != "old" || !strings.Contains(out, "not pointed live") { |
| 292 | t.Fatalf("live = %q, printed %q; want old and the binding refusal", e.live(), out) |
| 293 | } |
| 294 | r, _ := ReadRefusal(e.bake, proofHost) |
| 295 | if r == nil || !strings.Contains(r.Reason, etagOf(stale)) || !strings.Contains(r.Reason, e.newSHA) { |
| 296 | t.Fatalf("refusal = %+v, want one naming both hashes", r) |
| 297 | } |
| 298 | if r.Kind != RefusalNotProven || !r.Recovered || r.RestoredVersion != "old" || r.RecoveryError != "" { |
| 299 | t.Fatalf("refusal = %+v, want kind %q, recovered, restored to old", r, RefusalNotProven) |
| 300 | } |
| 301 | out = e.pass(t, e.store) |
| 302 | if e.live() != "old" { |
| 303 | t.Fatalf("live = %q on the second pass, want old", e.live()) |
| 304 | } |
| 305 | want := "refused: new's staged bytes do not match dev's proof" |
| 306 | if !strings.Contains(out, want) || !strings.Contains(out, "still on old") || !strings.Contains(out, "roll the exact bytes dev proved") { |
| 307 | t.Fatalf("second pass printed %q, want the not-proven gate line", out) |
| 308 | } |
| 309 | if strings.Contains(out, "--retry") || strings.Contains(out, "did not come up") { |
| 310 | t.Fatalf("second pass printed %q: a not-proven refusal is not a failed start", out) |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | // Final review triage (Task 4): the healthy history admits the manifest's |
| 315 | // sha256, but the staged file holds other bytes (a kept stale copy, as in |
| 316 | // TestProofBindingRefusesStagedBytesThatDiffer). bind must ask the history |
| 317 | // about the staged hash, not the manifest's, so the build is refused. |
| 318 | func TestProofHealthyManifestWithOtherStagedBytesIsRefused(t *testing.T) { |
| 319 | e := newProofEnv(t) |
| 320 | e.requireDev(t) |
| 321 | if err := RecordHealthy(e.bake, "bab", "console", HealthyBuild{SHA256: e.newSHA, Version: "new", How: "settle", At: e.clock}); err != nil { |
| 322 | t.Fatal(err) |
| 323 | } |
| 324 | stale := []byte("not the shipped bytes") |
| 325 | down := stageStaleCopyKept(t, e.ctx, e.store, e.opt, "bab", "console", "new", stale) |
| 326 | out := e.pass(t, down) |
| 327 | if e.live() != "old" || !strings.Contains(out, "not pointed live") { |
| 328 | t.Fatalf("live = %q, printed %q; want old and the binding refusal", e.live(), out) |
| 329 | } |
| 330 | r, _ := ReadRefusal(e.bake, proofHost) |
| 331 | if r == nil || r.Kind != RefusalNotProven || !strings.Contains(r.Reason, etagOf(stale)) || !strings.Contains(r.Reason, e.newSHA) { |
| 332 | t.Fatalf("refusal = %+v, want a not-proven record naming the staged and manifest hashes", r) |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | func TestProofMissIsCachedThirtySeconds(t *testing.T) { |
| 337 | e := newProofEnv(t) |
| 338 | e.requireDev(t) |
| 339 | cs := &proofReadCountingStore{Store: e.store} |
| 340 | e.pass(t, cs) |
| 341 | e.clock = e.clock.Add(10 * time.Second) |
| 342 | e.pass(t, cs) |
| 343 | if n := cs.proofGets(); n != 1 { |
| 344 | t.Fatalf("%d proof GETs inside 30s, want 1", n) |
| 345 | } |
| 346 | if e.gate.WaitingDue(proofHost) { |
| 347 | t.Fatal("due inside its 30s miss") |
| 348 | } |
| 349 | e.clock = e.clock.Add(21 * time.Second) |
| 350 | if !e.gate.WaitingDue(proofHost) { |
| 351 | t.Fatal("not due after 31s") |
| 352 | } |
| 353 | e.pass(t, cs) |
| 354 | if n := cs.proofGets(); n != 2 { |
| 355 | t.Fatalf("%d proof GETs after 31s, want 2", n) |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | // Review focus 1: a typo keeps the gate closed for a named-unit route. |
| 360 | func TestProofPolicyUnparseableKeepsGateClosed(t *testing.T) { |
| 361 | e := newProofEnv(t) |
| 362 | if err := e.reg.SetUnit(proofHost, "console.service"); err != nil { |
| 363 | t.Fatal(err) |
| 364 | } |
| 365 | e.writePolicy(t, `{"schema":1,"require":[`) |
| 366 | out := e.pass(t, e.store) |
| 367 | if e.live() != "old" { |
| 368 | t.Fatalf("live = %q, want old: an unparseable policy must not open the gate", e.live()) |
| 369 | } |
| 370 | if !strings.Contains(out, "does not parse") { |
| 371 | t.Fatalf("printed %q, want the policy error", out) |
| 372 | } |
| 373 | if w := e.gate.Waits(); len(w) != 1 || !strings.Contains(w[0].Reason, "does not parse") { |
| 374 | t.Fatalf("waits = %+v", w) |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | // The Task 2 carry: an error that is not a parse error (here, a policy path |
| 379 | // that is a directory and cannot be read) is closed too, never "no policy". |
| 380 | func TestProofPolicyUnreadableKeepsGateClosed(t *testing.T) { |
| 381 | e := newProofEnv(t) |
| 382 | if err := e.reg.SetUnit(proofHost, "console.service"); err != nil { |
| 383 | t.Fatal(err) |
| 384 | } |
| 385 | if err := os.Mkdir(e.policy, 0o755); err != nil { |
| 386 | t.Fatal(err) |
| 387 | } |
| 388 | e.pass(t, e.store) |
| 389 | if e.live() != "old" { |
| 390 | t.Fatalf("live = %q, want old: an unreadable policy must not open the gate", e.live()) |
| 391 | } |
| 392 | if w := e.gate.Waits(); len(w) != 1 || w[0].Host != proofHost { |
| 393 | t.Fatalf("waits = %+v, want the named-unit route waiting", w) |
| 394 | } |
| 395 | } |
| 396 | |
| 397 | // The agent's gate lives across passes: a policy edit is seen on the next |
| 398 | // pass without restarting anything. |
| 399 | func TestProofPolicyChangeSeenNextPass(t *testing.T) { |
| 400 | e := newProofEnv(t) |
| 401 | e.requireDev(t) |
| 402 | e.pass(t, e.store) |
| 403 | if e.live() != "old" { |
| 404 | t.Fatalf("live = %q, want old while required", e.live()) |
| 405 | } |
| 406 | e.writePolicy(t, `{"schema":1}`) |
| 407 | e.pass(t, e.store) |
| 408 | if e.live() != "new" { |
| 409 | t.Fatalf("live = %q, want new once the require entry is gone", e.live()) |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | // Ruling 5 (preflight 4.6): a host waiting on one head whose channel moves |
| 414 | // to a build that stops before the gate (bake hold, poisoned this pass, a |
| 415 | // refusal record) is no longer waiting. Otherwise WaitingDue stays true and |
| 416 | // the fast loop sweeps it every tick for the whole hold. |
| 417 | func TestProofWaitClearedWhenTheGateIsNotConsulted(t *testing.T) { |
| 418 | assertCleared := func(t *testing.T, e *proofEnv, out string) { |
| 419 | t.Helper() |
| 420 | if w := e.gate.Waits(); len(w) != 0 { |
| 421 | t.Fatalf("waits = %+v, printed %q; want none once the gate is not consulted", w, out) |
| 422 | } |
| 423 | if e.gate.WaitingDue(proofHost) { |
| 424 | t.Fatal("still due once the gate is not consulted") |
| 425 | } |
| 426 | } |
| 427 | waitOnNew := func(t *testing.T) *proofEnv { |
| 428 | t.Helper() |
| 429 | e := newProofEnv(t) |
| 430 | e.requireDev(t) |
| 431 | e.pass(t, e.store) |
| 432 | if w := e.gate.Waits(); len(w) != 1 || w[0].Version != "new" { |
| 433 | t.Fatalf("waits = %+v, want one on new", w) |
| 434 | } |
| 435 | e.clock = e.clock.Add(ProofMissTTL + time.Second) |
| 436 | return e |
| 437 | } |
| 438 | |
| 439 | t.Run("bake hold", func(t *testing.T) { |
| 440 | e := waitOnNew(t) |
| 441 | // beta has a 24h window, and a production app's windows apply. |
| 442 | if err := SetProduction(e.ctx, e.store, e.priv, "bab", "console", true, "test"); err != nil { |
| 443 | t.Fatal(err) |
| 444 | } |
| 445 | shipRelease(t, e.store, "bab", "console", "newer", adoptPlatform, []byte("newer build")) |
| 446 | if err := Promote(e.ctx, e.store, e.priv, "bab", "console", "beta", "newer"); err != nil { |
| 447 | t.Fatal(err) |
| 448 | } |
| 449 | addRoute(t, e.reg, proofHost, "console", "beta", false) |
| 450 | out := e.pass(t, e.store) |
| 451 | if !strings.Contains(out, "bake window") { |
| 452 | t.Fatalf("printed %q, want the bake hold", out) |
| 453 | } |
| 454 | assertCleared(t, e, out) |
| 455 | }) |
| 456 | |
| 457 | t.Run("poisoned", func(t *testing.T) { |
| 458 | e := waitOnNew(t) |
| 459 | // A second, hibernating route on the same channel goes after the |
| 460 | // awake one, so it meets the build the awake one poisoned. |
| 461 | const sleeper = "sleeper.example.com" |
| 462 | addRoute(t, e.reg, sleeper, "console", "canary/x", true) |
| 463 | if err := PointLive(e.opt, sleeper, "bab", "old", "console"); err != nil { |
| 464 | t.Fatal(err) |
| 465 | } |
| 466 | e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev")) |
| 467 | // The sleeper is the one whose wait must go: make it wait first. |
| 468 | e.gate.hold(registry.Route{Host: sleeper, App: "console"}, "new", e.newSHA, "dev", "no proof from dev", "old") |
| 469 | e.clock = e.clock.Add(ProofMissTTL + time.Second) |
| 470 | out := e.passWith(t, e.store, func(string) error { return errors.New("Job for console.service failed") }) |
| 471 | if !strings.Contains(out, "failed on another host this pass") { |
| 472 | t.Fatalf("printed %q, want the poisoned skip", out) |
| 473 | } |
| 474 | for _, w := range e.gate.Waits() { |
| 475 | if w.Host == sleeper { |
| 476 | t.Fatalf("waits = %+v, printed %q; the poisoned host is still waiting", e.gate.Waits(), out) |
| 477 | } |
| 478 | } |
| 479 | if e.gate.WaitingDue(sleeper) { |
| 480 | t.Fatal("poisoned host still due") |
| 481 | } |
| 482 | }) |
| 483 | |
| 484 | t.Run("refusal gate", func(t *testing.T) { |
| 485 | e := waitOnNew(t) |
| 486 | ptr, err := Channel(e.ctx, e.store, e.pub, "bab", "console", "canary/x") |
| 487 | if err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | if err := WriteRefusal(e.bake, Refusal{Host: proofHost, App: "console", Version: "new", |
| 491 | PointerPromotedAt: ptr.PromotedAt, RefusedAt: e.clock, Reason: "exited: status=2"}); err != nil { |
| 492 | t.Fatal(err) |
| 493 | } |
| 494 | out := e.pass(t, e.store) |
| 495 | if !strings.Contains(out, "refused: new") { |
| 496 | t.Fatalf("printed %q, want the refusal gate", out) |
| 497 | } |
| 498 | assertCleared(t, e, out) |
| 499 | }) |
| 500 | } |
| 501 | |
| 502 | // A host found live on its head (another process, root say, moved it) is |
| 503 | // not waiting for anything. |
| 504 | func TestProofWaitClearedOnceLiveOnTheHead(t *testing.T) { |
| 505 | e := newProofEnv(t) |
| 506 | e.requireDev(t) |
| 507 | e.pass(t, e.store) |
| 508 | if len(e.gate.Waits()) != 1 { |
| 509 | t.Fatalf("waits = %+v, want one", e.gate.Waits()) |
| 510 | } |
| 511 | if err := StageArtifact(e.opt, "bab", "new", "console", []byte("bad build")); err != nil { |
| 512 | t.Fatal(err) |
| 513 | } |
| 514 | if err := PointLive(e.opt, proofHost, "bab", "new", "console"); err != nil { |
| 515 | t.Fatal(err) |
| 516 | } |
| 517 | e.pass(t, e.store) |
| 518 | if w := e.gate.Waits(); len(w) != 0 || e.gate.WaitingDue(proofHost) { |
| 519 | t.Fatalf("waits = %+v once live on the head, want none", w) |
| 520 | } |
| 521 | } |
| 522 | |
| 523 | // A host that stops waiting forgets its cached miss too: when the channel |
| 524 | // comes back to the same build inside 30 s, the gate looks again and the |
| 525 | // host is listed as waiting, rather than held silently off the list. |
| 526 | func TestProofClearDropsTheCachedMiss(t *testing.T) { |
| 527 | e := newProofEnv(t) |
| 528 | e.requireDev(t) |
| 529 | cs := &proofReadCountingStore{Store: e.store} |
| 530 | e.pass(t, cs) |
| 531 | e.gate.clear(proofHost) |
| 532 | e.clock = e.clock.Add(time.Second) |
| 533 | e.pass(t, cs) |
| 534 | if n := cs.proofGets(); n != 2 { |
| 535 | t.Fatalf("%d proof GETs, want 2: a cleared host's miss is not cached", n) |
| 536 | } |
| 537 | if w := e.gate.Waits(); len(w) != 1 { |
| 538 | t.Fatalf("waits = %+v, want the host listed again", w) |
| 539 | } |
| 540 | } |
| 541 | |
| 542 | // Every gate method is nil-safe: no WithProofGate is today's behaviour. |
| 543 | func TestProofGateNilIsOff(t *testing.T) { |
| 544 | var g *ProofGate |
| 545 | g.BeginPass() |
| 546 | g.clear("x") |
| 547 | if g.Waits() != nil || g.WaitingDue("x") { |
| 548 | t.Fatal("a nil gate reports a wait") |
| 549 | } |
| 550 | if _, ok := g.admit(context.Background(), nil, "", registry.Route{Host: "x"}, &ChannelPointer{}, ""); !ok { |
| 551 | t.Fatal("a nil gate refused admission") |
| 552 | } |
| 553 | if !g.bind("", registry.Route{Host: "x"}, &ChannelPointer{}, proofVerdict{required: true}, "", time.Time{}) { |
| 554 | t.Fatal("a nil gate refused the staged bytes") |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | // bindingRefusal is TestProofBindingRefusesStagedBytesThatDiffer's first |
| 559 | // pass: a proven manifest, other bytes staged (kept, see stageStaleCopyKept), |
| 560 | // a RefusalNotProven written. |
| 561 | func bindingRefusal(t *testing.T) *proofEnv { |
| 562 | t.Helper() |
| 563 | e := newProofEnv(t) |
| 564 | e.requireDev(t) |
| 565 | down := stageStaleCopyKept(t, e.ctx, e.store, e.opt, "bab", "console", "new", []byte("not the shipped bytes")) |
| 566 | e.putAt(t, e.newSHA, e.signed(t, e.prover, "console", e.newSHA, "dev")) |
| 567 | e.pass(t, down) |
| 568 | if r, _ := ReadRefusal(e.bake, proofHost); r == nil || r.Kind != RefusalNotProven || e.live() != "old" { |
| 569 | t.Fatalf("refusal = %+v, live = %q; want a not-proven refusal and old", r, e.live()) |
| 570 | } |
| 571 | return e |
| 572 | } |
| 573 | |
| 574 | // Final review H3: removing the require entry, or the whole policy file, is |
| 575 | // the break glass, and it works after a binding refusal as well as after a |
| 576 | // missing-proof wait. The stored D3 record stops blocking and is cleared. |
| 577 | func TestProofBreakGlassOverridesABindingRefusal(t *testing.T) { |
| 578 | for name, lift := range map[string]func(t *testing.T, e *proofEnv){ |
| 579 | "require entry removed": func(t *testing.T, e *proofEnv) { e.writePolicy(t, `{"schema":1}`) }, |
| 580 | "policy file removed": func(t *testing.T, e *proofEnv) { |
| 581 | if err := os.Remove(e.policy); err != nil { |
| 582 | t.Fatal(err) |
| 583 | } |
| 584 | }, |
| 585 | } { |
| 586 | t.Run(name, func(t *testing.T) { |
| 587 | e := bindingRefusal(t) |
| 588 | lift(t, e) |
| 589 | out := e.pass(t, e.store) |
| 590 | if e.live() != "new" { |
| 591 | t.Fatalf("live = %q, printed %q; want new once the box no longer requires a proof", e.live(), out) |
| 592 | } |
| 593 | if r, _ := ReadRefusal(e.bake, proofHost); r != nil { |
| 594 | t.Fatalf("refusal = %+v, want it cleared", r) |
| 595 | } |
| 596 | if strings.Contains(out, "refused:") { |
| 597 | t.Fatalf("printed %q: the D3 refusal still gated the pass", out) |
| 598 | } |
| 599 | }) |
| 600 | } |
| 601 | } |
| 602 | |
| 603 | // A policy that will not load is closed: the D3 record keeps blocking, even |
| 604 | // on a route with no unit, which admit alone would let through. |
| 605 | func TestProofBrokenPolicyKeepsABindingRefusal(t *testing.T) { |
| 606 | e := bindingRefusal(t) |
| 607 | e.writePolicy(t, `{"schema":1}}`) |
| 608 | out := e.pass(t, e.store) |
| 609 | if e.live() != "old" || !strings.Contains(out, "refused: new's staged bytes do not match dev's proof") { |
| 610 | t.Fatalf("live = %q, printed %q; want old and the not-proven gate line", e.live(), out) |
| 611 | } |
| 612 | if r, _ := ReadRefusal(e.bake, proofHost); r == nil || r.Kind != RefusalNotProven { |
| 613 | t.Fatalf("refusal = %+v, want the not-proven record kept", r) |
| 614 | } |
| 615 | } |
| 616 | |
| 617 | // D1 and D2 records never depend on the policy: lifting the requirement |
| 618 | // leaves them blocking. |
| 619 | func TestProofBreakGlassLeavesOtherRefusals(t *testing.T) { |
| 620 | for _, kind := range []string{RefusalNotUp, RefusalWrongProgram} { |
| 621 | t.Run("kind "+kind, func(t *testing.T) { |
| 622 | e := bindingRefusal(t) |
| 623 | r, _ := ReadRefusal(e.bake, proofHost) |
| 624 | r.Kind = kind |
| 625 | if err := WriteRefusal(e.bake, *r); err != nil { |
| 626 | t.Fatal(err) |
| 627 | } |
| 628 | if err := os.Remove(e.policy); err != nil { |
| 629 | t.Fatal(err) |
| 630 | } |
| 631 | out := e.pass(t, e.store) |
| 632 | if e.live() != "old" || !strings.Contains(out, "refused: new") { |
| 633 | t.Fatalf("live = %q, printed %q; want old and the refusal gate", e.live(), out) |
| 634 | } |
| 635 | if got, _ := ReadRefusal(e.bake, proofHost); got == nil || got.Kind != kind { |
| 636 | t.Fatalf("refusal = %+v, want the %q record kept", got, kind) |
| 637 | } |
| 638 | }) |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | func TestProofGateRequires(t *testing.T) { |
| 643 | var nilGate *ProofGate |
| 644 | if req, perr := nilGate.Requires("console"); req || perr { |
| 645 | t.Fatalf("nil gate Requires = %v, %v; want false, false", req, perr) |
| 646 | } |
| 647 | e := newProofEnv(t) |
| 648 | e.gate.BeginPass() |
| 649 | if req, perr := e.gate.Requires("console"); req || perr { |
| 650 | t.Fatalf("no policy: Requires = %v, %v; want false, false", req, perr) |
| 651 | } |
| 652 | e.requireDev(t) |
| 653 | e.gate.BeginPass() |
| 654 | if req, perr := e.gate.Requires("console"); !req || perr { |
| 655 | t.Fatalf("listed: Requires = %v, %v; want true, false", req, perr) |
| 656 | } |
| 657 | if req, perr := e.gate.Requires("other"); req || perr { |
| 658 | t.Fatalf("unlisted: Requires = %v, %v; want false, false", req, perr) |
| 659 | } |
| 660 | e.writePolicy(t, `{"schema":1,"require":[`) |
| 661 | e.gate.BeginPass() |
| 662 | if req, perr := e.gate.Requires("other"); !req || !perr { |
| 663 | t.Fatalf("broken policy: Requires = %v, %v; want true, true", req, perr) |
| 664 | } |
| 665 | } |
| 666 | |