carlos / platform Public

release: bake ratchet memory — pipeline windows loosen only after the old window runs

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev ad40a593cd07ec80a2bc17c1ce186cd860fc90c3 parent e993646
2 files changed, +253 −0
  • internal/release/bake.go +177 −0
  • internal/release/bake_test.go +76 −0
diff --git a/internal/release/bake.go b/internal/release/bake.go
index 8b8d0c311..e871c7f26 100644
--- a/internal/release/bake.go
+++ b/internal/release/bake.go
@@ -108,6 +108,27 @@ type FirstSeen struct {
// early, and never lets a deleted true downgrade anything either.
Production map[string]*prodMemory `json:"production,omitempty"`
+ // Holds is the pipeline bake-ratchet memory (release pipelines v2 spec
+ // §3), keyed by channelClockKey(acct, app, channel) — the same key
+ // shape and the same account-scoping argument as Channels (an app name
+ // never contains clockSep or "/", so the two shapes cannot collide).
+ // Each entry is the ENFORCED window for that channel, which only ever
+ // tightens instantly and only ever loosens after the OLD window has
+ // elapsed on this box's own clock — a declared window is a policy
+ // change an attacker with bucket write (but no key) could otherwise use
+ // to shrink a channel's bake to zero and ship through it immediately.
+ //
+ // ADDITIVE, but only one pass deep, same as Production above: an OLD
+ // binary's Save re-marshals a struct with no Holds field, which DELETES
+ // every app's ratchet memory from the shared file. That is not silently
+ // unsafe: the next new-binary pass re-latches every channel from
+ // HoldFor(DefaultHolds, name) — the fail-closed baseline — and any
+ // off-ratchet in flight simply restarts its clock from that pass's
+ // sighting. A restarted clock only ever DELAYS a loosening; it can
+ // never let one through early, so a mixed-version rollout costs at most
+ // one pass of this memory in the closed direction, same as Production.
+ Holds map[string]*holdMemory `json:"holds,omitempty"`
+
path string // where Save persists; set by LoadFirstSeen
// migrated marks a clock LoadFirstSeenMigrating built from its
@@ -195,6 +216,9 @@ func LoadFirstSeen(path string) *FirstSeen {
if st.Production == nil {
st.Production = map[string]*prodMemory{}
}
+ if st.Holds == nil {
+ st.Holds = map[string]*holdMemory{}
+ }
if st.Jitter < 0 || st.Jitter >= 0.25 {
st.Jitter = 0
}
@@ -475,3 +499,156 @@ func (st *FirstSeen) ProductionBake(acct, app string, flagErr error, hasBakeable
eligible := mem.OffFirstSeen.Add(time.Duration(float64(d) * (1 + st.Jitter)))
return now.Before(eligible)
}
+
+// ---- pipeline bake ratchet ---------------------------------------------------
+
+// holdMemory is one channel's bake-window ratchet memory: the box's own
+// record of what window is actually enforced, versus what the pipeline
+// object currently declares. It exists for the same reason prodMemory
+// exists — a bucket write alone (no signing key) must never be able to
+// shrink a channel's supply-chain window and have that apply immediately.
+type holdMemory struct {
+ // Window is the ENFORCED window right now.
+ Window time.Duration `json:"window"`
+ // SetAt is the replay floor: the newest Pipeline.SetAt this box has
+ // folded into this channel's memory. Anything at or before it is
+ // either the same object re-read or an older replay — neither changes
+ // policy, though a pending promotion may still land on its own clock.
+ SetAt time.Time `json:"set_at"`
+ // Pending is a declared window narrower than Window, waiting for the
+ // OLD Window to run its full course (from PendingAt) before it takes
+ // effect. Zero (with PendingAt zero) means nothing is pending.
+ Pending time.Duration `json:"pending,omitempty"`
+ // PendingAt is THIS BOX's own sighting of the narrower declaration —
+ // never Pipeline.SetAt, same argument as OffFirstSeen above: an
+ // attacker controlling the feed controls every timestamp in it, but
+ // not what the box already saw. A re-read that leaves Pending
+ // unchanged must not push this forward, or the clock would never
+ // elapse against a fleet that polls faster than the window itself.
+ PendingAt time.Time `json:"pending_at,omitzero"`
+}
+
+// holdKeyPrefix is channelClockKey's key shape without the channel segment,
+// for enumerating an app's remembered channels when no Pipeline is at hand
+// (ObservePipeline's p == nil case). Mirrors channelClockKey's two shapes
+// exactly, including the ops-account bare form.
+func holdKeyPrefix(acct, app string) string {
+ if acct == "" || acct == OpsAccount {
+ return app + "/"
+ }
+ return acct + clockSep + app + clockSep
+}
+
+// checkHoldPromotion applies mem's pending promotion if the OLD window has
+// now elapsed on this box's own clock (mem.PendingAt), jittered the same
+// way EligibleAt jitters a hold. Reports whether it promoted. nil and
+// no-pending are both no-ops so callers can call it unconditionally.
+func (f *FirstSeen) checkHoldPromotion(mem *holdMemory, now time.Time) bool {
+ if mem == nil || mem.PendingAt.IsZero() {
+ return false
+ }
+ elapsed := time.Duration(float64(mem.Window) * (1 + f.Jitter))
+ if now.Before(mem.PendingAt.Add(elapsed)) {
+ return false
+ }
+ mem.Window = mem.Pending
+ mem.Pending = 0
+ mem.PendingAt = time.Time{}
+ return true
+}
+
+// ObservePipeline folds one verified pipeline read into this box's
+// per-channel bake-window ratchet memory (release pipelines v2 spec §3).
+// Returns whether anything changed; the caller persists via Save.
+//
+// A brand-new channel baselines at HoldFor(DefaultHolds, name) — the
+// fail-closed default — never its own declared window, so a narrower
+// window a brand-new channel declares ratchets in like any other tightening
+// move rather than applying on first sight. A declared window that is no
+// narrower than what's enforced tightens immediately; a narrower one goes
+// pending and only takes effect once the OLD window has run its course,
+// timed from this box's OWN sighting of the narrower declaration
+// (PendingAt), never the pipeline's SetAt — the same argument as the
+// production-flag ratchet and the channel clock itself: an attacker with
+// bucket write but no signing key controls every timestamp in the object,
+// but not what this box already saw. Pending promotion is checked on every
+// call, including replays and p == nil, so a fleet that only polls this
+// path occasionally still lands the promotion the pass after it elapses.
+//
+// p == nil (pipeline deleted, or absent this pass) never loosens anything
+// from absence — existing memory persists exactly as ObserveProduction
+// leaves Production alone on a missing flag — but pending promotions for
+// this app's already-remembered channels still get their chance to land.
+//
+// A channel whose BakeWindow() fails to parse is skipped: old memory is
+// kept (fail closed) and this never prevents any other channel in the same
+// pipeline from being observed.
+func (f *FirstSeen) ObservePipeline(acct, app string, p *Pipeline, now time.Time) bool {
+ if f.Holds == nil {
+ f.Holds = map[string]*holdMemory{}
+ }
+ changed := false
+ if p == nil {
+ prefix := holdKeyPrefix(acct, app)
+ for key, mem := range f.Holds {
+ if strings.HasPrefix(key, prefix) && f.checkHoldPromotion(mem, now) {
+ changed = true
+ }
+ }
+ return changed
+ }
+ for _, ch := range p.Channels {
+ key := channelClockKey(acct, app, ch.Name)
+ mem := f.Holds[key]
+ if w, err := ch.BakeWindow(); err == nil {
+ firstSighting := mem == nil
+ if firstSighting {
+ mem = &holdMemory{Window: HoldFor(DefaultHolds, ch.Name)}
+ f.Holds[key] = mem
+ changed = true
+ }
+ // The replay floor: equal or older SetAt is either the same
+ // object re-read or a replay — no policy value change, though
+ // the pending-promotion check below still runs.
+ if firstSighting || p.SetAt.After(mem.SetAt) {
+ mem.SetAt = p.SetAt
+ changed = true
+ if w >= mem.Window {
+ mem.Window = w
+ mem.Pending = 0
+ mem.PendingAt = time.Time{}
+ } else if mem.Pending != w {
+ mem.Pending = w
+ mem.PendingAt = now
+ }
+ }
+ }
+ if f.checkHoldPromotion(mem, now) {
+ changed = true
+ }
+ }
+ return changed
+}
+
+// HoldsForApp returns the ENFORCED per-channel bake window map for a
+// pipelined app — one entry per channel this box has actually latched
+// memory for via ObservePipeline (a channel p declares but this box has
+// never observed simply has no entry yet). nil when p == nil, so the
+// caller falls back to DefaultHolds entirely, the pre-pipeline behaviour
+// for an app with no pipeline at all. Channels absent from the returned
+// map fall to HoldFor's 72h fail-closed default when the caller passes
+// this map to HoldFor — which lands on exactly the same baseline
+// ObservePipeline would have latched for that channel on its first
+// sighting, so the two paths agree.
+func (f *FirstSeen) HoldsForApp(acct, app string, p *Pipeline) map[string]time.Duration {
+ if p == nil {
+ return nil
+ }
+ out := map[string]time.Duration{}
+ for _, ch := range p.Channels {
+ if mem := f.Holds[channelClockKey(acct, app, ch.Name)]; mem != nil {
+ out[ch.Name] = mem.Window
+ }
+ }
+ return out
+}
diff --git a/internal/release/bake_test.go b/internal/release/bake_test.go
index 7465509cb..9ef5e203a 100644
--- a/internal/release/bake_test.go
+++ b/internal/release/bake_test.go
@@ -607,3 +607,79 @@ func TestCorruptClockFailsTowardBakingNotAwayFromIt(t *testing.T) {
t.Fatal("an unreadable (directory-at-path) clock file did not fail toward baking")
}
}
+
+func TestHoldMemoryRatchet(t *testing.T) {
+ f := &FirstSeen{Jitter: 0}
+ now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)
+ pipe := func(bake string, setAt time.Time) *Pipeline {
+ return &Pipeline{Schema: 1, Channels: []PipelineChannel{
+ {Name: "edge"}, {Name: "production", Bake: bake}}, SetAt: setAt}
+ }
+ // First sighting: production baselines at the fail-closed 72h, NOT its declared 24h.
+ f.ObservePipeline("bae", "web", pipe("24h", now), now)
+ holds := f.HoldsForApp("bae", "web", pipe("24h", now))
+ if holds["production"] != 72*time.Hour {
+ t.Fatalf("first-sighting window = %v; want fail-closed 72h baseline", holds["production"])
+ }
+ if holds["edge"] != 0 {
+ t.Fatalf("edge window = %v; want 0 (legacy baseline)", holds["edge"])
+ }
+ // After the old window elapses on the box's own clock, the pending 24h lands.
+ later := now.Add(72*time.Hour + time.Second)
+ f.ObservePipeline("bae", "web", pipe("24h", now), later)
+ if h := f.HoldsForApp("bae", "web", pipe("24h", now)); h["production"] != 24*time.Hour {
+ t.Fatalf("post-ratchet window = %v; want 24h", h["production"])
+ }
+ // Tightening is immediate.
+ f.ObservePipeline("bae", "web", pipe("96h", now.Add(80*time.Hour)), later.Add(time.Minute))
+ if h := f.HoldsForApp("bae", "web", nil); h != nil {
+ t.Fatalf("HoldsForApp(nil pipeline) = %v; want nil", h)
+ }
+ if h := f.HoldsForApp("bae", "web", pipe("96h", now)); h["production"] != 96*time.Hour {
+ t.Fatalf("tighten = %v; want 96h immediately", h["production"])
+ }
+}
+
+func TestHoldMemoryReplayFloorAndReReadKeepsClock(t *testing.T) {
+ f := &FirstSeen{Jitter: 0}
+ now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)
+ p1 := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "1h"}}, SetAt: now}
+ f.ObservePipeline("bae", "web", p1, now)
+ // Re-reading the same object an hour later must NOT restart the pending clock…
+ f.ObservePipeline("bae", "web", p1, now.Add(time.Hour))
+ // …so the loosening lands 72h after the FIRST sighting, not the second.
+ f.ObservePipeline("bae", "web", p1, now.Add(72*time.Hour+time.Second))
+ if h := f.HoldsForApp("bae", "web", p1); h["stable"] != time.Hour {
+ t.Fatalf("window = %v; want 1h landed 72h after first sighting", h["stable"])
+ }
+ // A replayed OLDER SetAt can never loosen further.
+ old := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "0s"}}, SetAt: now.Add(-time.Hour)}
+ f.ObservePipeline("bae", "web", old, now.Add(73*time.Hour))
+ if h := f.HoldsForApp("bae", "web", p1); h["stable"] != time.Hour {
+ t.Fatalf("replayed old object loosened the window to %v", h["stable"])
+ }
+}
+
+func TestHoldMemorySurvivesSaveLoadAndOldBinaryDrop(t *testing.T) {
+ // Additive-field discipline: marshal a FirstSeen with Holds, unmarshal,
+ // memory intact; unmarshal a PRE-CHANGE JSON (no holds key) into the new
+ // struct — no error, nil map, and ObservePipeline re-latches from scratch
+ // (the closed direction: a restarted clock only ever delays loosening).
+ f := &FirstSeen{Jitter: 0, Holds: map[string]*holdMemory{
+ channelClockKey("bae", "web", "production"): {Window: 24 * time.Hour, SetAt: time.Now().UTC()}}}
+ b, err := json.Marshal(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var back FirstSeen
+ if err := json.Unmarshal(b, &back); err != nil {
+ t.Fatal(err)
+ }
+ if back.Holds[channelClockKey("bae", "web", "production")].Window != 24*time.Hour {
+ t.Fatal("holds memory lost across marshal round trip")
+ }
+ var legacy FirstSeen
+ if err := json.Unmarshal([]byte(`{"jitter":0,"channels":{}}`), &legacy); err != nil || legacy.Holds != nil {
+ t.Fatalf("legacy clock parse = %v, holds %v", err, legacy.Holds)
+ }
+}