amadan
release: adoption enforces pipeline bake windows through the ratchet
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2 files changed,
+398
−32
internal/release/adopt.go+151 −32internal/release/adopt_test.go+247 −0
diff --git a/internal/release/adopt.go b/internal/release/adopt.go| index 46f6925ff..a86ea7060 100644 |
| --- a/internal/release/adopt.go |
| +++ b/internal/release/adopt.go |
| @@ -329,25 +329,130 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| pointers[k] = ptr |
| } |
| + // Resolve each app's pipeline once per pass — for every app with a |
| + // resolved NON-CANARY head, never one read per channel or per route. |
| + // canary/* lives outside the pipeline entirely (spec §2): no pipeline |
| + // can name it, its window is zero either way, so a canary-only |
| + // appearance has no verdict a read could change — which is also what |
| + // keeps the 2-3-second canary fast sweep at zero per-tick bucket reads |
| + // here, the same cost property the production-flag reads below hold. |
| + // |
| + // A read/verify ERROR is kept, and kept apart from verified-absent |
| + // (nil, nil): the errored app falls to the legacy path below — |
| + // DefaultHolds plus the production-flag machinery, the same fail-closed |
| + // posture an old binary that cannot parse pipeline.json at all takes |
| + // (spec §3's old-binary edge; its mitigation, a nonzero-bake pipeline |
| + // also setting production: true, is what makes legacy the closed |
| + // direction). The error is NEVER folded into the ratchet as a nil: nil |
| + // means verified-absent only (ObservePipeline's contract), and a |
| + // corrupt object must not masquerade as a deletion. |
| + type pipeState struct { |
| + p *Pipeline |
| + err error // non-nil: the object exists but would not read or verify |
| + } |
| + pipes := map[acctApp]pipeState{} |
| + for k, ptr := range pointers { |
| + if ptr == nil { |
| + continue |
| + } |
| + if r, ok := Rung(k.channel); ok && r == RungCanary { |
| + continue |
| + } |
| + ak := acctApp{k.acct, k.app} |
| + if _, done := pipes[ak]; done { |
| + continue |
| + } |
| + p, perr := GetPipeline(ctx, s, pub, ak.acct, ak.app) |
| + pipes[ak] = pipeState{p: p, err: perr} |
| + if perr != nil { |
| + // SAY SO, like the production flag below — same realistic |
| + // non-tampering trigger too: on a GetObject-without-ListBucket |
| + // fleet a GET of an ABSENT key answers 403, not 404, and every |
| + // app would land here every pass. |
| + fmt.Printf(" %s/%s: pipeline unreadable (%v) — legacy holds and the production flag apply\n", ak.acct, ak.app, perr) |
| + } |
| + } |
| + |
| + // The bake clock, loaded before any of this pass's reads are folded into |
| + // it. now is stamped once so every observation and verdict this pass |
| + // answers the same instant. |
| + now := time.Now() |
| + bake := LoadFirstSeen(FirstSeenPath(optDir)) |
| + if cfg.bakeDir != "" { |
| + bake = LoadFirstSeenMigrating(FirstSeenPath(cfg.bakeDir), FirstSeenPath(optDir)) |
| + } |
| + // bake.migrated forces this pass's Save even if nothing new is observed |
| + // below: otherwise a changed-gated Save could leave the new path absent |
| + // for days on a fleet that happens to see no new channel head, and the |
| + // legacy file would look safe to delete long before it actually is (see |
| + // LoadFirstSeenMigrating and BakeDir's doc comments, 2026-08-03 review). |
| + changed := bake.migrated |
| + |
| + // ORDER IS LOAD-BEARING: fold this pass's verified pipeline reads into |
| + // the ratchet memory HERE, before the per-app window maps below are |
| + // computed from it — a tightened window read this pass must be enforced |
| + // this pass, and a pending loosening whose old window has elapsed must |
| + // land before the verdicts that depend on it. Same same-pass rule as |
| + // ObserveProduction's fold further down. The err == nil guard is the |
| + // contract, not paranoia: nil reaches ObservePipeline only for a |
| + // verified-absent object (which never loosens anything, but does give |
| + // this app's pending promotions their chance to land); an errored read |
| + // never ticks the memory at all. |
| + for ak, ps := range pipes { |
| + if ps.err == nil && bake.ObservePipeline(ak.acct, ak.app, ps.p, now) { |
| + changed = true |
| + } |
| + } |
| + |
| + // One enforced-window map per app for this pass. A verified pipeline |
| + // resolves through the box's ratchet memory (HoldsForApp); everything |
| + // else — no pipeline, or a read this pass could not trust — keeps |
| + // DefaultHolds, today's behaviour byte-for-byte. canary/* is stitched |
| + // back in at zero: it lives outside the pipeline (spec §2), so |
| + // HoldsForApp's map cannot carry it, and HoldFor's fallback for a name |
| + // missing from the map is the 72h fail-closed default — which would |
| + // silently freeze every canary deploy of a pipelined app. Every OTHER |
| + // channel outside the pipeline does take that 72h (spec §3: a channel |
| + // with no pipeline entry keeps the fail-closed default), including rung |
| + // names like beta whose DefaultHolds entry no longer applies. |
| + appHolds := map[acctApp]map[string]time.Duration{} |
| + for ak, ps := range pipes { |
| + if ps.err == nil && ps.p != nil { |
| + h := bake.HoldsForApp(ak.acct, ak.app, ps.p) |
| + h["canary/*"] = 0 |
| + appHolds[ak] = h |
| + } |
| + } |
| + holdsFor := func(ak acctApp) map[string]time.Duration { |
| + if h, ok := appHolds[ak]; ok { |
| + return h |
| + } |
| + return DefaultHolds |
| + } |
| + |
| // Resolve each app's production flag once per pass — but ONLY for apps |
| - // with a HELD head resolved this pass (spec §6). That filter is a cost |
| - // property, not a nicety: any pass pays at most one read per (account, |
| - // app) with a held head, never one per channel or per route. For the |
| - // fast-converge sweep (PassFilter every 2-3 SECONDS) that usually means |
| - // zero flag reads — canary/* has HoldFor 0, and a fully-staged box's |
| - // sweep carries nothing else — but an UNSTAGED stable/beta route in the |
| + // with a HELD head resolved this pass (spec §6), held now meaning held |
| + // under the app's OWN window map above. That filter is a cost property, |
| + // not a nicety: any pass pays at most one read per (account, app) with a |
| + // held head, never one per channel or per route. For the fast-converge |
| + // sweep (PassFilter every 2-3 SECONDS) that usually means zero flag |
| + // reads — canary/* has a zero window on every path, and a fully-staged |
| + // box's sweep carries nothing else — but an UNSTAGED held route in the |
| // sweep's set does resolve its held channel and pays this read too. |
| // What keeps that bounded is the sweep's own membership, not this |
| // filter: an unstaged host backs off 2s→5min (internal/host's |
| // fastMembers), so a held, never-staging route costs one flag read per |
| - // backoff window at worst, not one per tick. |
| + // backoff window at worst, not one per tick. A PIPELINED app's held |
| + // head pays the read as well even though its verdict never consults the |
| + // flag (bakeIgnore below): the fold keeps the legacy latch fresh for |
| + // the day the pipeline is deleted and the app falls back to it. |
| type prodState struct { |
| flag *ProductionFlag |
| err error // non-nil: the object exists but would not read or verify |
| } |
| bakeable := map[acctApp]bool{} |
| for k, ptr := range pointers { |
| - if ptr != nil && HoldFor(DefaultHolds, k.channel) > 0 { |
| + if ptr != nil && HoldFor(holdsFor(acctApp{k.acct, k.app}), k.channel) > 0 { |
| bakeable[acctApp{k.acct, k.app}] = true |
| } |
| } |
| @@ -377,17 +482,6 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| // that head before, so the supply-chain window is measured from the box's |
| // own first sighting, never a feed timestamp (bake.go). The clock only |
| // ever ticks for pointers that verified above. |
| - now := time.Now() |
| - bake := LoadFirstSeen(FirstSeenPath(optDir)) |
| - if cfg.bakeDir != "" { |
| - bake = LoadFirstSeenMigrating(FirstSeenPath(cfg.bakeDir), FirstSeenPath(optDir)) |
| - } |
| - // bake.migrated forces this pass's Save even if nothing new is observed |
| - // below: otherwise a changed-gated Save could leave the new path absent |
| - // for days on a fleet that happens to see no new channel head, and the |
| - // legacy file would look safe to delete long before it actually is (see |
| - // LoadFirstSeenMigrating and BakeDir's doc comments, 2026-08-03 review). |
| - changed := bake.migrated |
| for k, ptr := range pointers { |
| if ptr != nil && bake.Observe(k.acct, k.app, k.channel, ptr.Version, now) { |
| changed = true |
| @@ -421,8 +515,11 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| // often none (canary/* has no hold), sometimes a few (an unstaged |
| // stable/beta member), never reliably ALL of the apps whose latches the |
| // clock file carries: its in-memory Production map is PARTIAL, not |
| - // repaired. Letting it write would overwrite the corrupt file with that |
| - // partial view AND clear the taint that is currently failing closed, |
| + // repaired. The pipeline ratchet's Holds map is partial for the same |
| + // reason (pipelines are read only for the narrowed set's own non-canary |
| + // apps), so this skip protects both memories at once. Letting it write |
| + // would overwrite the corrupt file with that partial view AND clear the |
| + // taint that is currently failing closed, |
| // dropping the latch of every app outside its set before anything |
| // re-observed those flags — and if a bucket flag had also been deleted, |
| // that latch would be gone for good. So a tainted clock on a narrowed |
| @@ -460,8 +557,22 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| // row can still carry an account NAME there, which resolves to ops. |
| bakeIgnore := func(rt registry.Route) bool { |
| ak := acctApp{RouteAccount(rt), rt.App} |
| + // A PIPELINED app never consults the flag: its bake policy is the |
| + // pipeline's alone (spec §3) — never ignore, and a zero-window |
| + // channel holds nothing anyway. The flag stays meaningful for |
| + // legacy apps, and for THIS app on a pass whose pipeline read |
| + // errored, which falls to exactly that legacy machinery. |
| + if ps := pipes[ak]; ps.err == nil && ps.p != nil { |
| + return false |
| + } |
| return !bake.ProductionBake(ak.acct, ak.app, prod[ak].err, bakeable[ak], DefaultHolds, now) |
| } |
| + // routeHolds resolves the same per-app window map for a route that the |
| + // bakeable filter above used — shared for the same reason bakeIgnore is: |
| + // the same question must not be answerable two ways for one app. |
| + routeHolds := func(rt registry.Route) map[string]time.Duration { |
| + return holdsFor(acctApp{RouteAccount(rt), rt.App}) |
| + } |
| // Awake hosts first, hibernating second: repoints have no health gate, so |
| // let an awake host discover (and poison) a broken build before any |
| @@ -476,7 +587,7 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| continue |
| } |
| ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| - adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, restart, now, cfg, bakeIgnore(rt)) |
| + adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, restart, now, cfg, routeHolds(rt), bakeIgnore(rt)) |
| } |
| for _, rt := range hibernating { |
| ptr := pointers[groupKey{RouteAccount(rt), rt.App, routeChannel(rt)}] |
| @@ -491,7 +602,7 @@ func PassFilter(ctx context.Context, reg *registry.Registry, s Store, pub ed2551 |
| // no-op on a stopped unit). The flag says a route MAY sleep, not that |
| // it is asleep now, and a warm exec child must not keep serving a |
| // binary its symlink no longer names. |
| - adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, cfg.restartHibernating, now, cfg, bakeIgnore(rt)) |
| + adoptHost(ctx, reg, s, optDir, bake, poisoned, rt, ptr, cfg.restartHibernating, now, cfg, routeHolds(rt), bakeIgnore(rt)) |
| } |
| return nil |
| } |
| @@ -581,12 +692,17 @@ func RouteAccount(rt registry.Route) string { |
| // string, and cfg.capsOps nil means the capability converge never runs at |
| // all. See NagExpiredCanaries and WithCapabilities for who sets each and why. |
| // |
| -// ignore is the app-level bake verdict from ProductionBake — the production |
| -// flag's doing, never a publisher field. True means the supply-chain window |
| -// does not CURRENTLY apply to this app on this box (not declared production, |
| -// or declared off and past the ratchet); the caller computes it once per |
| -// route from the flag it read this pass. |
| -func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir string, bake *FirstSeen, poisoned map[string]bool, rt registry.Route, ptr *ChannelPointer, restart func(host string) error, now time.Time, cfg passConfig, ignore bool) { |
| +// holds is the enforced per-channel window map the caller resolved for this |
| +// route's app: a verified pipeline's ratcheted HoldsForApp map (canary/* |
| +// stitched back in at zero), else DefaultHolds — never consulted app-blind. |
| +// |
| +// ignore is the app-level bake verdict from ProductionBake — the LEGACY |
| +// production flag's doing, never a publisher field, and constant false for |
| +// a pipelined app, whose windows come from holds alone (spec §3). True |
| +// means the supply-chain window does not CURRENTLY apply to this app on |
| +// this box (not declared production, or declared off and past the ratchet); |
| +// the caller computes it once per route from the flag it read this pass. |
| +func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir string, bake *FirstSeen, poisoned map[string]bool, rt registry.Route, ptr *ChannelPointer, restart func(host string) error, now time.Time, cfg passConfig, holds map[string]time.Duration, ignore bool) { |
| if ptr == nil { |
| return |
| } |
| @@ -636,9 +752,12 @@ func adoptHost(ctx context.Context, reg *registry.Registry, s Store, optDir stri |
| return // up to date — stay quiet on the periodic tick |
| } |
| // Two independent waivers, both root-signed in origin: ignore is the |
| - // app's production verdict (§2/§4), ptr.FastTrack is this ONE pointer's |
| - // per-promotion waiver (§5). Either one alone releases the head. |
| - if hold := Hold(bake, RouteAccount(rt), rt.App, channel, ver, DefaultHolds, ignore || ptr.FastTrack, now); hold != "" { |
| + // app's legacy production verdict (§2/§4; constant false for a pipelined |
| + // app), ptr.FastTrack is this ONE pointer's per-promotion waiver (§5) — |
| + // deliberately unratcheted (pipelines spec §3 amendment: ratcheting it |
| + // would make it wait out the very window it exists to skip). Either one |
| + // alone releases the head. |
| + if hold := Hold(bake, RouteAccount(rt), rt.App, channel, ver, holds, ignore || ptr.FastTrack, now); hold != "" { |
| fmt.Printf(" %s: %s (%s) is %s\n", rt.Host, ver, channel, hold) |
| return |
| } |
diff --git a/internal/release/adopt_test.go b/internal/release/adopt_test.go| index 032852440..6209ef80e 100644 |
| --- a/internal/release/adopt_test.go |
| +++ b/internal/release/adopt_test.go |
| @@ -2743,3 +2743,250 @@ func TestPassProductionOffStillRatchets(t *testing.T) { |
| t.Fatalf("production memory = %+v, want a latched app with an off-sighting recorded", mem) |
| } |
| } |
| + |
| +// ---- pipeline windows on the adopt path (release pipelines v2 spec §3) ------ |
| + |
| +// TestPassEnforcesPipelineWindows: an app with pipeline {edge(0), |
| +// production(24h)}. Four behaviours in one lifecycle, in order: a release at |
| +// the head of "production" is HELD even though the app's production flag is |
| +// absent (pipelined apps never consult the flag); the same release at the |
| +// head of "edge" adopts immediately; the declared 24h — narrower than the |
| +// 72h a brand-new channel baselines at — ratchets in only after the OLD |
| +// window has elapsed on this box's own clock, and then governs; and a |
| +// FastTrack-re-signed pointer adopts immediately while held (spec §3 |
| +// amendment — ratcheting the waiver would make it wait out the very window |
| +// it exists to skip). |
| +func TestPassEnforcesPipelineWindows(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + // NO production flag is ever set in this test: a pipelined app's windows |
| + // come from the pipeline alone. (Legacy behaviour — no flag, no window — |
| + // would adopt the production head immediately.) |
| + if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ |
| + {Name: "edge"}, {Name: "production", Bake: "24h"}, |
| + }, "test"); err != nil { |
| + t.Fatalf("SetPipeline: %v", err) |
| + } |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin one")) |
| + for _, ch := range []string{"edge", "production"} { |
| + if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { |
| + t.Fatalf("Promote %s: %v", ch, err) |
| + } |
| + } |
| + addRoute(t, reg, "e.example.com", "myapp", "edge", false) |
| + addRoute(t, reg, "p.example.com", "myapp", "production", false) |
| + |
| + // Sub-step 1: the production head is held with no flag in sight. |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, "p.example.com"); v != "" { |
| + t.Fatalf("production live = %q, want held — a pipelined app's bake window must apply with no production flag at all", v) |
| + } |
| + // Sub-step 2: the zero-bake entry channel adopted in that same pass. |
| + if v, _ := LiveVersion(opt, "e.example.com"); v != "v1" { |
| + t.Fatalf("edge live = %q, want v1 — the pipeline's zero-bake channel adopts immediately", v) |
| + } |
| + |
| + // Sub-step 3: the ratchet lands. The pass above latched the 72h |
| + // fail-closed baseline and parked the declared 24h as pending. Backdate |
| + // the box's own clocks: the pending sighting past the old window (72h |
| + // jittered < 90h < 91h), and v1's first sighting past 24h jittered |
| + // (< 30h < 31h) but NOT past 72h — adoption then proves the enforced |
| + // window really is the ratcheted-in 24h, not an expired baseline. |
| + st := LoadFirstSeen(FirstSeenPath(opt)) |
| + key := channelClockKey("bab", "myapp", "production") |
| + mem := st.Holds[key] |
| + if mem == nil || mem.Window != 72*time.Hour || mem.Pending != 24*time.Hour || mem.PendingAt.IsZero() { |
| + t.Fatalf("hold memory = %+v, want the 72h baseline with 24h pending", mem) |
| + } |
| + mem.PendingAt = mem.PendingAt.Add(-91 * time.Hour) |
| + if seen, ok := st.Channels[key]["v1"]; !ok { |
| + t.Fatalf("no first sighting recorded for v1 at the production head") |
| + } else { |
| + st.Channels[key]["v1"] = seen.Add(-31 * time.Hour) |
| + } |
| + if err := st.Save(); err != nil { |
| + t.Fatalf("Save backdated clock: %v", err) |
| + } |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { |
| + t.Fatalf("production live = %q, want v1 — the ratcheted-in 24h window elapsed on the box's own clock", v) |
| + } |
| + // And the landing was PERSISTED by that pass: ObservePipeline's changed |
| + // flag must reach the Save path, or the promotion would re-run (and the |
| + // window claim silently regress) every pass until some other change |
| + // happened to force a write. |
| + if mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key]; mem == nil || mem.Window != 24*time.Hour || !mem.PendingAt.IsZero() { |
| + t.Fatalf("persisted hold memory = %+v, want the landed 24h window", mem) |
| + } |
| + |
| + // Sub-step 4: fast-track. v2 at the production head is inside the (now |
| + // 24h) window; the root-re-signed waiver releases it, unchanged by the |
| + // pipeline. |
| + shipRelease(t, s, "bab", "myapp", "v2", adoptPlatform, []byte("bin two")) |
| + if err := Promote(ctx, s, priv, "bab", "myapp", "production", "v2"); err != nil { |
| + t.Fatalf("Promote v2: %v", err) |
| + } |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, "p.example.com"); v != "v1" { |
| + t.Fatalf("production live = %q, want still v1 — v2 is inside the window until the waiver", v) |
| + } |
| + if _, err := FastTrack(ctx, s, priv, "bab", "myapp", "production", "test"); err != nil { |
| + t.Fatalf("FastTrack: %v", err) |
| + } |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, "p.example.com"); v != "v2" { |
| + t.Fatalf("production live = %q, want v2 — FastTrack must keep bypassing a pipelined hold", v) |
| + } |
| +} |
| + |
| +// TestPassPipelinedAppCanaryKeepsNoWindow: canary/* lives OUTSIDE the |
| +// pipeline (spec §2), so a pipelined app's canary routes must keep their |
| +// zero window. The trap is structural: HoldsForApp's map carries only the |
| +// pipeline's channels, and HoldFor's fallback for a name missing from the |
| +// map is the 72h fail-closed default — without canary/* stitched back in at |
| +// zero, declaring a pipeline would freeze every canary deploy of the app. |
| +func TestPassPipelinedAppCanaryKeepsNoWindow(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + if _, err := SetPipeline(ctx, s, priv, "bab", "myapp", []PipelineChannel{ |
| + {Name: "edge"}, {Name: "production", Bake: "24h"}, |
| + }, "test"); err != nil { |
| + t.Fatalf("SetPipeline: %v", err) |
| + } |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) |
| + for _, ch := range []string{"canary/x", "production"} { |
| + if err := Promote(ctx, s, priv, "bab", "myapp", ch, "v1"); err != nil { |
| + t.Fatalf("Promote %s: %v", ch, err) |
| + } |
| + } |
| + addRoute(t, reg, "c.example.com", "myapp", "canary/x", false) |
| + addRoute(t, reg, "p.example.com", "myapp", "production", false) |
| + |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, "c.example.com"); v != "v1" { |
| + t.Fatalf("canary live = %q, want v1 — canary/* is outside the pipeline and must keep its zero window", v) |
| + } |
| + // Not vacuous: the held production head in the SAME pass proves the |
| + // pipelined holds map (not DefaultHolds-with-no-flag, which would have |
| + // adopted it) is what governed — including the canary route above. |
| + if v, _ := LiveVersion(opt, "p.example.com"); v != "" { |
| + t.Fatalf("production live = %q, want held — without the hold this test proves nothing about the map the canary resolved through", v) |
| + } |
| +} |
| + |
| +// TestPassCorruptPipelineFailsTowardLegacyHolds: a pipeline object that |
| +// EXISTS but will not read or verify is not a pipeline and not an absence — |
| +// the app falls back to the legacy path (DefaultHolds + the production |
| +// flag), the same fail-closed posture an old binary that cannot parse |
| +// pipeline.json takes, whose mitigation is that enabling a nonzero-bake |
| +// pipeline also sets production: true (spec §3). And the errored read must |
| +// NEVER reach the ratchet memory as a nil: nil means verified-absent, and a |
| +// nil fold would land pending promotions — a corrupt object masquerading as |
| +// a deletion. |
| +func TestPassCorruptPipelineFailsTowardLegacyHolds(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, s, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) |
| + if err := Promote(ctx, s, priv, "bab", "myapp", "production", "v1"); err != nil { |
| + t.Fatalf("Promote: %v", err) |
| + } |
| + // The flag the console sets alongside any nonzero-bake pipeline. |
| + if err := SetProduction(ctx, s, priv, "bab", "myapp", true, "test"); err != nil { |
| + t.Fatalf("SetProduction: %v", err) |
| + } |
| + if _, err := s.Put(ctx, PipelineKey("bab", "myapp"), []byte("not a wrapper"), ""); err != nil { |
| + t.Fatalf("corrupt the pipeline object: %v", err) |
| + } |
| + addRoute(t, reg, "p.example.com", "myapp", "production", false) |
| + |
| + // Pre-seed ratchet memory with a pending loosening whose old window has |
| + // long elapsed, and a v1 sighting past the pending 24h but inside the |
| + // enforced 72h. If the errored read were wrongly folded as nil, the |
| + // promotion would land and the memory would read 24h below. |
| + now := time.Now() |
| + st := LoadFirstSeen(FirstSeenPath(opt)) |
| + key := channelClockKey("bab", "myapp", "production") |
| + st.Holds[key] = &holdMemory{ |
| + Window: 72 * time.Hour, SetAt: now.Add(-100 * time.Hour), |
| + Pending: 24 * time.Hour, PendingAt: now.Add(-91 * time.Hour), |
| + } |
| + st.Channels[key] = map[string]time.Time{"v1": now.Add(-31 * time.Hour)} |
| + if err := st.Save(); err != nil { |
| + t.Fatalf("Save pre-seeded clock: %v", err) |
| + } |
| + |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if v, _ := LiveVersion(opt, "p.example.com"); v != "" { |
| + t.Fatalf("live = %q, want held — an unreadable pipeline must fall to legacy holds with the flag applying, never adopt free", v) |
| + } |
| + mem := LoadFirstSeen(FirstSeenPath(opt)).Holds[key] |
| + if mem == nil || mem.Window != 72*time.Hour || mem.PendingAt.IsZero() { |
| + t.Fatalf("hold memory after the errored read = %+v, want untouched (72h enforced, promotion still pending) — an errored read must never tick the ratchet", mem) |
| + } |
| +} |
| + |
| +// The 2-3 SECOND canary fast loop must never pay a pipeline bucket read |
| +// either: canary/* is outside every pipeline, so no canary-only pass has a |
| +// verdict the object could change — the mirror of |
| +// TestCanaryFastLoopPaysNoProductionRead. The hourly pass pays exactly one |
| +// read per (account, app) with a non-canary head — not one per channel, not |
| +// one per route. |
| +func TestCanaryFastLoopPaysNoPipelineRead(t *testing.T) { |
| + dir := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + priv, pub := testKeys(t) |
| + opt := t.TempDir() |
| + reg := testRegistry(t) |
| + |
| + shipRelease(t, dir, "bab", "myapp", "v1", adoptPlatform, []byte("bin")) |
| + for _, ch := range []string{"canary/x", "canary/y", "edge", "stable"} { |
| + if err := Promote(ctx, dir, priv, "bab", "myapp", ch, "v1"); err != nil { |
| + t.Fatalf("Promote %s: %v", ch, err) |
| + } |
| + } |
| + addRoute(t, reg, "cx.example.com", "myapp", "canary/x", false) |
| + addRoute(t, reg, "cy.example.com", "myapp", "canary/y", false) |
| + addRoute(t, reg, "e.example.com", "myapp", "edge", false) |
| + addRoute(t, reg, "s.example.com", "myapp", "stable", false) |
| + |
| + s := &prodReadStore{Store: dir, key: PipelineKey("bab", "myapp")} |
| + if err := PassFilter(ctx, reg, s, pub, opt, func(string) error { return nil }, isCanaryTestRoute); err != nil { |
| + t.Fatalf("PassFilter: %v", err) |
| + } |
| + if s.n != 0 { |
| + t.Fatalf("canary-only pass read the pipeline %d times, want 0 — canary/* is outside every pipeline", s.n) |
| + } |
| + |
| + // Not vacuous: the full pass, with an edge and a stable head resolved, |
| + // reads it — once for the whole app. |
| + if err := Pass(ctx, reg, s, pub, opt, func(string) error { return nil }); err != nil { |
| + t.Fatalf("Pass: %v", err) |
| + } |
| + if s.n != 1 { |
| + t.Fatalf("full pass read the pipeline %d times, want exactly 1 per (account, app)", s.n) |
| + } |
| +} |