| 1 | // Bake windows — ported from Eleven bake.go, intact. |
| 2 | // |
| 3 | // The supply-chain bake window: beta and stable subscribers don't install a |
| 4 | // release the moment it appears — the box waits until the release has sat at |
| 5 | // the channel head for a while first, so a compromised release that gets |
| 6 | // caught and pulled back never reaches them. The clock is THIS BOX's own |
| 7 | // observation ("when did I first see this version at this channel head"), |
| 8 | // never a manifest timestamp: an attacker holding the signing key controls |
| 9 | // every timestamp in the feed and would simply backdate one, but cannot |
| 10 | // backdate what the box already saw. |
| 11 | // |
| 12 | // canary/* has no window — it IS the operator's own deploy path (`ship && |
| 13 | // promote`). beta's 24h absorbs the caught-within-a-day mass of real |
| 14 | // supply-chain incidents; stable's 72h additionally spans a weekend, so a |
| 15 | // Friday-night malicious publish is seen by a human on Monday with margin to |
| 16 | // pull it back. edge has no window either — merged to main is picked up as it |
| 17 | // lands (pipelines spec §3). |
| 18 | // |
| 19 | // There is deliberately NO publisher-side bypass ("critical fix, skip the |
| 20 | // hold") — that flag is the first thing an attacker sets. The bypass is |
| 21 | // local and explicit (the ignore parameter of Hold), which needs root on the |
| 22 | // box. |
| 23 | package release |
| 24 | |
| 25 | import ( |
| 26 | "encoding/json" |
| 27 | "errors" |
| 28 | "fmt" |
| 29 | "math/rand" |
| 30 | "os" |
| 31 | "path/filepath" |
| 32 | "strings" |
| 33 | "time" |
| 34 | |
| 35 | "github.com/carlosframework/platform/internal/safefile" |
| 36 | ) |
| 37 | |
| 38 | // DefaultHolds is the default bake window per channel; per-deployment |
| 39 | // overrides come from host config as a replacement map passed by the caller. |
| 40 | var DefaultHolds = map[string]time.Duration{ |
| 41 | "canary/*": 0, |
| 42 | "edge": 0, |
| 43 | "beta": 24 * time.Hour, |
| 44 | "stable": 72 * time.Hour, |
| 45 | } |
| 46 | |
| 47 | // HoldFor resolves a channel's bake window: an exact entry wins, then a |
| 48 | // "<head>/*" wildcard for two-segment channels (canary/<branch>), and an |
| 49 | // unknown channel gets the longest standard window — fail closed, never |
| 50 | // zero, for a channel nobody configured. |
| 51 | func HoldFor(holds map[string]time.Duration, channel string) time.Duration { |
| 52 | if d, ok := holds[channel]; ok { |
| 53 | return d |
| 54 | } |
| 55 | if i := strings.Index(channel, "/"); i >= 0 { |
| 56 | if d, ok := holds[channel[:i]+"/*"]; ok { |
| 57 | return d |
| 58 | } |
| 59 | } |
| 60 | return 72 * time.Hour |
| 61 | } |
| 62 | |
| 63 | // FirstSeenPath is the LEGACY bake clock location, under the box's opt dir. |
| 64 | // It is read-only for every production caller now: BakeDir moved the |
| 65 | // writable clock to the data dir (issue #13 — the sandboxed edge unit |
| 66 | // cannot write under optDir, and must not be able to), and |
| 67 | // LoadFirstSeenMigrating reaches back to this path only to carry an |
| 68 | // existing box's observations across that move. Nothing in production |
| 69 | // writes here anymore; a box still on the legacy path only reads it once, |
| 70 | // on the pass that migrates it. |
| 71 | func FirstSeenPath(optDir string) string { |
| 72 | return filepath.Join(optDir, "channel-first-seen.json") |
| 73 | } |
| 74 | |
| 75 | // firstSeenKeep bounds each channel's history. Past heads are kept (not just |
| 76 | // the current one) so a pull-back to a previously-seen version is immediately |
| 77 | // eligible — evacuating a bad release must never itself sit in a bake window. |
| 78 | const firstSeenKeep = 12 |
| 79 | |
| 80 | // FirstSeen is the bake clock: for each account/app/channel, when this box |
| 81 | // first observed each version at its head. |
| 82 | type FirstSeen struct { |
| 83 | // Jitter desynchronises a fleet of boxes: each waits hold*(1+Jitter), so |
| 84 | // boxes don't all install in the same hour and the earliest become |
| 85 | // canaries for the rest. Minted once per box, in [0, 0.25). |
| 86 | Jitter float64 `json:"jitter"` |
| 87 | // Channels is keyed "<account>/<app>/<channel>" -> version -> first-seen |
| 88 | // time. Pre-accounts clocks have bare "<app>/<channel>" keys; see |
| 89 | // channelClockKey. |
| 90 | Channels map[string]map[string]time.Time `json:"channels"` |
| 91 | |
| 92 | // Production is the per-app production-flag memory (2026-08-04 console |
| 93 | // bake controls spec §4), keyed with clockSep — the same SEPARATOR |
| 94 | // argument as channelClockKey (a channel segment may contain "/"; an |
| 95 | // account or app name never contains clockSep). prodKey folds "" and |
| 96 | // OpsAccount to the same key — there is no legacy bare shape to |
| 97 | // preserve here, this memory is new, so callers can pass whatever |
| 98 | // RouteAccount(rt) (adopt.go) returns and land on the right memory |
| 99 | // either way. |
| 100 | // |
| 101 | // ADDITIVE, but only one pass deep: an OLD binary that Saves this file |
| 102 | // re-marshals the struct IT knows about, which has no Production field — |
| 103 | // its write DELETES every app's production memory from the shared file. |
| 104 | // That is not silently unsafe: the latch re-latches from the bucket flag |
| 105 | // the very next pass a new binary reads it (true stays true), and a lost |
| 106 | // ratchet clock simply restarts from that pass's sighting (an |
| 107 | // in-progress off-ratchet gets pushed out, never skipped) — both are the |
| 108 | // closed direction on their own. A mixed-version rollout only ever costs |
| 109 | // up to one pass of this memory; it never lets an off-flag through |
| 110 | // early, and never lets a deleted true downgrade anything either. |
| 111 | Production map[string]*prodMemory `json:"production,omitempty"` |
| 112 | |
| 113 | // Holds is the pipeline bake-ratchet memory (release pipelines v2 spec |
| 114 | // §3), keyed by channelClockKey(acct, app, channel) — the same key |
| 115 | // shape and the same account-scoping argument as Channels (an app name |
| 116 | // never contains clockSep or "/", so the two shapes cannot collide). |
| 117 | // Each entry is the ENFORCED window for that channel, which only ever |
| 118 | // tightens instantly and only ever loosens after the OLD window has |
| 119 | // elapsed on this box's own clock — a declared window is a policy |
| 120 | // change an attacker with bucket write (but no key) could otherwise use |
| 121 | // to shrink a channel's bake to zero and ship through it immediately. |
| 122 | // |
| 123 | // ADDITIVE, but only one pass deep, same as Production above: an OLD |
| 124 | // binary's Save re-marshals a struct with no Holds field, which DELETES |
| 125 | // every app's ratchet memory from the shared file. For the WINDOW that |
| 126 | // is not silently unsafe: the next new-binary pass re-latches every |
| 127 | // channel from HoldFor(DefaultHolds, name) — the fail-closed baseline — |
| 128 | // and any off-ratchet in flight simply restarts its clock from that |
| 129 | // pass's sighting. A restarted clock only ever DELAYS a loosening; it |
| 130 | // can never let one through early. |
| 131 | // |
| 132 | // That closed-direction argument covers Window/Pending/PendingAt and NOT |
| 133 | // holdMemory.DeletedAt, which fails OPEN when dropped (a deleted-pipeline |
| 134 | // app loses the only evidence it was ever pipelined, and the flagless |
| 135 | // legacy verdict adopts). See that field's own comment — including why |
| 136 | // today's exposure is a rollback within this branch rather than a fleet |
| 137 | // property. |
| 138 | Holds map[string]*holdMemory `json:"holds,omitempty"` |
| 139 | |
| 140 | path string // where Save persists; set by LoadFirstSeen |
| 141 | save func() error |
| 142 | |
| 143 | // migrated marks a clock LoadFirstSeenMigrating built from its |
| 144 | // legacy-fallback branch (the new path didn't exist yet). PassFilter |
| 145 | // checks this to force an immediate Save even when Observe reports no |
| 146 | // change this pass — see LoadFirstSeenMigrating's doc comment for why |
| 147 | // a changed-gated Save alone isn't enough to make the migration durable. |
| 148 | migrated bool |
| 149 | |
| 150 | // tainted marks a clock whose file read succeeded but did NOT parse as |
| 151 | // JSON — distinct from a missing file, which is just a fresh, empty |
| 152 | // clock. Never serialized (unexported); see LoadFirstSeen and |
| 153 | // ProductionBake for why the two failure modes must be told apart. |
| 154 | tainted bool |
| 155 | } |
| 156 | |
| 157 | // channelClockKey identifies one channel head's clock. The ACCOUNT is part of |
| 158 | // it because app names are unique per account, not globally: without it, one |
| 159 | // account's 72h stable observation would satisfy another account's |
| 160 | // identically-named app on the same box, silently skipping the supply-chain |
| 161 | // window that is the entire point of this file. |
| 162 | // |
| 163 | // The ops account keeps the pre-accounts key shape ("<app>/<channel>"), so |
| 164 | // every existing box's observations carry across an upgrade. Restarting them |
| 165 | // would freeze the whole fleet for 72h on the day this ships, which is a |
| 166 | // worse failure than the one it would be protecting against. That shape is |
| 167 | // injective on its own: an app name can never contain "/", so the first |
| 168 | // segment is always the app and the rest is always the channel. |
| 169 | // |
| 170 | // Everyone else is joined with clockSep, NOT "/" — channels may contain a |
| 171 | // "/" (canary/<branch>), so "/" between segments would make ops app "kass" on |
| 172 | // channel "canary/x" collide with account "kass", app "canary", channel "x", |
| 173 | // and one account's observation would satisfy another's window. clockSep |
| 174 | // cannot appear in a sqid, an app name or a channel segment, so the split is |
| 175 | // unambiguous — and because it can never appear in an ops key either, the two |
| 176 | // shapes cannot collide with each other. |
| 177 | func channelClockKey(acct, app, channel string) string { |
| 178 | if acct == "" || acct == OpsAccount { |
| 179 | return app + "/" + channel |
| 180 | } |
| 181 | return acct + clockSep + app + clockSep + channel |
| 182 | } |
| 183 | |
| 184 | // clockSep separates the bake clock's key segments. "|" is outside every |
| 185 | // character class the platform allows in a sqid (lowercase letters), an app |
| 186 | // name (appRe) or a channel segment (channelSegRe). |
| 187 | const clockSep = "|" |
| 188 | |
| 189 | // LoadFirstSeen reads the bake clock from path. For the CHANNEL clock, a |
| 190 | // missing OR corrupt file is simply an empty clock — every head then bakes |
| 191 | // from "now", which fails closed (a wiped clock delays updates; it never |
| 192 | // lets one through early either way). |
| 193 | // |
| 194 | // For the PRODUCTION memory (§4), those two failure modes must be told |
| 195 | // apart. A file that reads but does not PARSE marks the clock tainted, and |
| 196 | // ProductionBake fails toward baking for every held head on a tainted |
| 197 | // clock — the same closed direction, reached differently, because an |
| 198 | // unmarshal error can leave Production nil or half-written rather than |
| 199 | // usably empty. A file that is simply ABSENT, though, is indistinguishable |
| 200 | // from a fresh box: the same read that lets the channel clock safely |
| 201 | // restart "from now" also silently loses the production latch and the |
| 202 | // off-ratchet's replay floor, because there is nothing on disk to say a |
| 203 | // latch ever existed. That gap is accepted: the latch defends against |
| 204 | // BUCKET-side tampering (someone with bucket write but no key deleting the |
| 205 | // flag object) — a box's own /data is a different trust boundary. Whatever |
| 206 | // already has write access there already holds adopt authority, i.e. |
| 207 | // already controls what this box installs; deleting its bake-clock file |
| 208 | // buys it nothing a corrupt bucket object would. |
| 209 | func LoadFirstSeen(path string) *FirstSeen { |
| 210 | st := &FirstSeen{path: path} |
| 211 | if raw, err := safefile.ReadFile(path); err == nil { |
| 212 | if err := json.Unmarshal(raw, st); err != nil { |
| 213 | st.tainted = true |
| 214 | } |
| 215 | } else if !errors.Is(err, os.ErrNotExist) { |
| 216 | // The file EXISTS but this process could not read it (EACCES, |
| 217 | // EISDIR, an I/O error) — a different failure than "no file", and |
| 218 | // one this box cannot tell apart from a bucket object it can't |
| 219 | // trust either. Same taint, same fail-toward-baking direction. |
| 220 | st.tainted = true |
| 221 | } |
| 222 | if st.Channels == nil { |
| 223 | st.Channels = map[string]map[string]time.Time{} |
| 224 | } |
| 225 | if st.Production == nil { |
| 226 | st.Production = map[string]*prodMemory{} |
| 227 | } |
| 228 | if st.Holds == nil { |
| 229 | st.Holds = map[string]*holdMemory{} |
| 230 | } |
| 231 | if st.Jitter < 0 || st.Jitter >= 0.25 { |
| 232 | st.Jitter = 0 |
| 233 | } |
| 234 | return st |
| 235 | } |
| 236 | |
| 237 | // LoadFirstSeenMigrating reads the bake clock from path, reaching back to |
| 238 | // legacyPath for its CONTENT when path does not exist yet — the one-time |
| 239 | // migration off /opt/carlos (issue #13: the sandboxed edge cannot write the |
| 240 | // version store, and must not be able to; the clock belongs in the data |
| 241 | // dir). Save always writes path. The legacy file is never written or |
| 242 | // deleted: an older binary still running against it keeps a working clock, |
| 243 | // and this binary simply stops advancing it. |
| 244 | // |
| 245 | // The returned clock is marked migrated, which its caller (PassFilter) uses |
| 246 | // to force that first Save even when nothing new was observed this pass — |
| 247 | // Save is otherwise changed-gated (bake.go/adopt.go), and a box whose fleet |
| 248 | // happens to see no new channel head could otherwise run for days with path |
| 249 | // still absent, still re-reading legacyPath every pass. That window matters |
| 250 | // because it invites deleting the legacy file as soon as this binary ships, |
| 251 | // which — before the migrated Save lands — would silently wipe every stable |
| 252 | // channel's observations and freeze the fleet on a fresh 72h window. Forcing |
| 253 | // the first Save makes migration complete (path fully populated, |
| 254 | // legacyPath now provably disposable) after exactly one pass. |
| 255 | func LoadFirstSeenMigrating(path, legacyPath string) *FirstSeen { |
| 256 | if _, err := os.Stat(path); os.IsNotExist(err) { |
| 257 | st := LoadFirstSeen(legacyPath) |
| 258 | st.path = path |
| 259 | st.migrated = true |
| 260 | return st |
| 261 | } |
| 262 | return LoadFirstSeen(path) |
| 263 | } |
| 264 | |
| 265 | // Observe records a channel head this box hasn't seen at that head before. |
| 266 | // Returns whether anything changed (the caller persists via Save). Never |
| 267 | // call it on a pointer that failed verification — the clock only ever ticks |
| 268 | // for signed feeds. An already-known version's timestamp is NEVER touched: |
| 269 | // that preserved first sighting is exactly what makes a pointer moved BACK |
| 270 | // to a previously-seen version (rollback, evacuation) immediately eligible. |
| 271 | func (st *FirstSeen) Observe(acct, app, channel, version string, now time.Time) bool { |
| 272 | changed := false |
| 273 | if st.Jitter == 0 && len(st.Channels) == 0 { |
| 274 | // A fresh box mints its jitter once, alongside its first observation. |
| 275 | st.Jitter = rand.Float64() * 0.25 |
| 276 | changed = true |
| 277 | } |
| 278 | key := channelClockKey(acct, app, channel) |
| 279 | seen := st.Channels[key] |
| 280 | if seen == nil { |
| 281 | seen = map[string]time.Time{} |
| 282 | st.Channels[key] = seen |
| 283 | } |
| 284 | if _, ok := seen[version]; !ok { |
| 285 | seen[version] = now.UTC().Truncate(time.Second) |
| 286 | changed = true |
| 287 | } |
| 288 | // Prune oldest entries beyond the cap — the just-recorded head is the |
| 289 | // newest, so it always survives. |
| 290 | for len(seen) > firstSeenKeep { |
| 291 | oldest, at := "", time.Time{} |
| 292 | for v, t := range seen { |
| 293 | if oldest == "" || t.Before(at) { |
| 294 | oldest, at = v, t |
| 295 | } |
| 296 | } |
| 297 | delete(seen, oldest) |
| 298 | } |
| 299 | return changed |
| 300 | } |
| 301 | |
| 302 | // Save persists the clock back to the path it was loaded from, via a |
| 303 | // temp-file-in-the-same-dir + rename — the same co-written-file pattern |
| 304 | // internal/appconfig/sync.go and syncexec.go already use for /data files |
| 305 | // more than one uid writes (2026-08-03 review, issue #13): carlos-edge.service |
| 306 | // runs as `carlos`, the root adopt one-shot runs as root, and BakeDir makes |
| 307 | // both write this same path. An in-place O_TRUNC write needs write |
| 308 | // permission on the EXISTING file, so whichever uid creates it first (0644, |
| 309 | // owned by that uid) locks the other one out — EACCES with nothing to |
| 310 | // repair the ownership (bootstrap only chowns /data once, at bootstrap). |
| 311 | // rename(2), by contrast, only requires write permission on the containing |
| 312 | // directory, which both uids have on /data, so either side can always |
| 313 | // replace the file — and a crash mid-write leaves the old content intact |
| 314 | // rather than a torn, half-written file. |
| 315 | // |
| 316 | // The temp name is minted fresh per attempt (os.CreateTemp's "channel- |
| 317 | // first-seen-*.json"), never a fixed "<path>.tmp": two uids can each crash |
| 318 | // between write and rename, and a fixed name would leave cross-uid residue — |
| 319 | // a root-owned 0644 leftover that the carlos-uid loop then fails to |
| 320 | // O_TRUNC-rewrite forever (EACCES, no repair path short of root's next |
| 321 | // pass), aborting its whole Pass every tick until then. A unique name per |
| 322 | // attempt means a crash never leaves a temp file another uid's next Save |
| 323 | // must contend with. CreateTemp also mints its file 0600, which this |
| 324 | // explicitly widens to 0644 with Chmod BEFORE the rename: a 0600 root:root |
| 325 | // clock is unreadable by the carlos-uid loop, and LoadFirstSeen turns that |
| 326 | // read failure into a silently EMPTY clock — not an error, just every |
| 327 | // channel head baking from "now" again, freezing stable adoption for a |
| 328 | // fresh 72h. Any failure path removes the temp file rather than leaving it |
| 329 | // behind for the next Save to trip over. |
| 330 | func (st *FirstSeen) Save() error { |
| 331 | if st.save != nil { |
| 332 | return st.save() |
| 333 | } |
| 334 | raw, err := json.MarshalIndent(st, "", " ") |
| 335 | if err != nil { |
| 336 | return err |
| 337 | } |
| 338 | raw = append(raw, '\n') |
| 339 | return safefile.WriteFile(st.path, raw, 0o644) |
| 340 | } |
| 341 | |
| 342 | // EligibleAt is the moment a version first seen at a channel head may be |
| 343 | // installed: firstSeen + hold*(1+jitter). ok=false when the version has no |
| 344 | // record at this channel's head — brand new, so not eligible (callers that |
| 345 | // Observe first make that unreachable; a status read that doesn't tick the |
| 346 | // clock can legitimately see it). |
| 347 | func (st *FirstSeen) EligibleAt(acct, app, channel, version string, holds map[string]time.Duration) (time.Time, bool) { |
| 348 | seen, ok := st.Channels[channelClockKey(acct, app, channel)][version] |
| 349 | if !ok { |
| 350 | return time.Time{}, false |
| 351 | } |
| 352 | d := HoldFor(holds, channel) |
| 353 | if d == 0 { |
| 354 | return seen, true |
| 355 | } |
| 356 | return seen.Add(time.Duration(float64(d) * (1 + st.Jitter))), true |
| 357 | } |
| 358 | |
| 359 | // Hold returns a non-empty human reason while a channel head must not yet be |
| 360 | // installed on this box, "" once it may be. ignore is the local, root-only |
| 361 | // override — deliberately never a publisher-side flag. |
| 362 | func Hold(st *FirstSeen, acct, app, channel, version string, holds map[string]time.Duration, ignore bool, now time.Time) string { |
| 363 | if ignore { |
| 364 | return "" |
| 365 | } |
| 366 | at, ok := st.EligibleAt(acct, app, channel, version, holds) |
| 367 | if !ok { |
| 368 | // Fail closed: no record means no clock has run, so nothing installs. |
| 369 | return "in its supply-chain bake window (clock starts this pass)" |
| 370 | } |
| 371 | if now.Before(at) { |
| 372 | return fmt.Sprintf("in its supply-chain bake window until %s", at.UTC().Format("2006-01-02 15:04 MST")) |
| 373 | } |
| 374 | return "" |
| 375 | } |
| 376 | |
| 377 | // ---- production memory ------------------------------------------------------ |
| 378 | |
| 379 | // prodMemory is one app's production-flag memory on this box. It exists to |
| 380 | // hold two properties no bucket read can: DELETION IS NOT A DOWNGRADE (once |
| 381 | // a verified production: true is seen, absence never clears it — anyone |
| 382 | // with bucket write but no key can delete an object), and A STALE FLAG |
| 383 | // CANNOT BE REPLAYED (an off from a past era, re-written by someone with |
| 384 | // bucket access alone, must not roll back a newer on). |
| 385 | type prodMemory struct { |
| 386 | // WasProduction latches on the first verified production: true. |
| 387 | WasProduction bool `json:"was_production,omitempty"` |
| 388 | // NewestSetAt is the replay floor: the newest set_at this box has |
| 389 | // honored. Anything at or below it is old news. |
| 390 | NewestSetAt time.Time `json:"newest_set_at,omitzero"` |
| 391 | // OffSetAt is the discriminator: non-zero means the newest verified word |
| 392 | // for this app is off (only ever read via IsZero — its VALUE isn't |
| 393 | // otherwise consulted; NewestSetAt already carries the replay floor). |
| 394 | // OffFirstSeen is when THIS BOX first saw that off-flag edition — the |
| 395 | // ratchet clock, the box's own observation, never the flag's own |
| 396 | // timestamp (same argument as the bake clock itself). |
| 397 | OffSetAt time.Time `json:"off_set_at,omitzero"` |
| 398 | OffFirstSeen time.Time `json:"off_first_seen,omitzero"` |
| 399 | } |
| 400 | |
| 401 | // prodKey identifies one app's production memory. Joined with clockSep for |
| 402 | // the same reason channelClockKey is (an app or account name can never |
| 403 | // contain it, unlike a channel segment), and it folds "" and OpsAccount to |
| 404 | // the same key — unlike channelClockKey there's no legacy bare shape to |
| 405 | // keep separate, this memory is new, so RouteAccount(rt)'s either form |
| 406 | // always resolves to the one record. |
| 407 | func prodKey(acct, app string) string { |
| 408 | if acct == "" || acct == OpsAccount { |
| 409 | acct = OpsAccount |
| 410 | } |
| 411 | return acct + clockSep + app |
| 412 | } |
| 413 | |
| 414 | // ObserveProduction folds one VERIFIED flag read into this box's memory. |
| 415 | // nil (no object in the bucket) changes nothing — absence is not evidence. |
| 416 | // Returns whether anything changed; the caller persists via Save. Never |
| 417 | // call it with a flag that failed verification: like Observe, the memory |
| 418 | // only ever ticks for signed documents. |
| 419 | func (st *FirstSeen) ObserveProduction(acct, app string, flag *ProductionFlag, now time.Time) bool { |
| 420 | if flag == nil { |
| 421 | return false |
| 422 | } |
| 423 | key := prodKey(acct, app) |
| 424 | mem := st.Production[key] |
| 425 | if mem == nil { |
| 426 | mem = &prodMemory{} |
| 427 | } |
| 428 | // The replay floor. Equal set_at is the object already recorded — the |
| 429 | // every-pass re-read — so nothing changes; older is a replay and is |
| 430 | // ignored outright. |
| 431 | if !flag.SetAt.After(mem.NewestSetAt) { |
| 432 | return false |
| 433 | } |
| 434 | mem.NewestSetAt = flag.SetAt |
| 435 | if flag.Production { |
| 436 | mem.WasProduction = true |
| 437 | mem.OffSetAt, mem.OffFirstSeen = time.Time{}, time.Time{} |
| 438 | } else { |
| 439 | mem.OffSetAt = flag.SetAt |
| 440 | mem.OffFirstSeen = now.UTC().Truncate(time.Second) |
| 441 | } |
| 442 | st.Production[key] = mem |
| 443 | return true |
| 444 | } |
| 445 | |
| 446 | // ProductionBake is the adopt pass's one question: does the supply-chain |
| 447 | // window apply to this app on this box right now? Call ObserveProduction |
| 448 | // first — this reads memory, plus flagErr, which is non-nil when a flag |
| 449 | // object EXISTS but would not read or verify: that fails toward baking (an |
| 450 | // attacker who can corrupt but not sign must not shorten anything). |
| 451 | // hasBakeableHead is the spec §4 carve-out: with no held channel head |
| 452 | // resolved this pass, an off-flag needs no ratchet — there is nothing at a |
| 453 | // baked head to protect. |
| 454 | func (st *FirstSeen) ProductionBake(acct, app string, flagErr error, hasBakeableHead bool, holds map[string]time.Duration, now time.Time) bool { |
| 455 | if st.tainted { |
| 456 | // This box's own clock file existed but couldn't be read or didn't |
| 457 | // parse — untrustworthy in a way absence never is. Fail toward |
| 458 | // baking for every held head, same direction as flagErr, for the |
| 459 | // lifetime of THIS load: tainted is set once, in LoadFirstSeen, and |
| 460 | // nothing clears it in memory afterward. Save rewrites the file |
| 461 | // from this struct's current (now-valid, in-memory) content, so the |
| 462 | // NEXT LoadFirstSeen of that path starts clean. |
| 463 | return true |
| 464 | } |
| 465 | if flagErr != nil { |
| 466 | return true |
| 467 | } |
| 468 | mem := st.Production[prodKey(acct, app)] |
| 469 | if mem == nil { |
| 470 | return false // never declared production — the default |
| 471 | } |
| 472 | if mem.OffSetAt.IsZero() { |
| 473 | // The newest verified word is production: true — or the flag has |
| 474 | // vanished since, which never downgrades (WasProduction latches). |
| 475 | return mem.WasProduction |
| 476 | } |
| 477 | // The newest verified word is production: false. |
| 478 | if !mem.WasProduction { |
| 479 | return false // off, and never was on: nothing to ratchet |
| 480 | } |
| 481 | if !hasBakeableHead { |
| 482 | return false // the carve-out |
| 483 | } |
| 484 | // The ratchet: off is honored only after the stable window has elapsed |
| 485 | // on this box's own sighting of the off-flag, jittered like any hold — |
| 486 | // the flag defends itself with the mechanism it controls. |
| 487 | d := HoldFor(holds, RungStable) |
| 488 | eligible := mem.OffFirstSeen.Add(time.Duration(float64(d) * (1 + st.Jitter))) |
| 489 | return now.Before(eligible) |
| 490 | } |
| 491 | |
| 492 | // ---- pipeline bake ratchet --------------------------------------------------- |
| 493 | |
| 494 | // holdMemory is one channel's bake-window ratchet memory: the box's own |
| 495 | // record of what window is actually enforced, versus what the pipeline |
| 496 | // object currently declares. It exists for the same reason prodMemory |
| 497 | // exists — a bucket write alone (no signing key) must never be able to |
| 498 | // shrink a channel's supply-chain window and have that apply immediately. |
| 499 | // |
| 500 | // DELETION is covered by the same ratchet, not by prodMemory's |
| 501 | // deletion-is-not-a-downgrade latch: a deleted pipeline means the app is |
| 502 | // legacy again, so each remembered channel treats its LEGACY default |
| 503 | // (HoldFor(DefaultHolds, name)) as the newly-declared window and ratchets |
| 504 | // toward it through the ordinary pending machinery (ObservePipeline's |
| 505 | // p == nil branch) — the old window is enforced for one full cycle first, |
| 506 | // never collapsed instantly, and never pinned forever. |
| 507 | type holdMemory struct { |
| 508 | // Window is the ENFORCED window right now. |
| 509 | Window time.Duration `json:"window"` |
| 510 | // SetAt is the replay floor: the newest Pipeline.SetAt this box has |
| 511 | // folded into this channel's memory. Anything at or before it is |
| 512 | // either the same object re-read or an older replay — neither changes |
| 513 | // policy, though a pending promotion may still land on its own clock. |
| 514 | SetAt time.Time `json:"set_at"` |
| 515 | // Pending is a declared window narrower than Window, waiting for the |
| 516 | // OLD Window to run its full course (from PendingAt) before it takes |
| 517 | // effect. Zero (with PendingAt zero) means nothing is pending. |
| 518 | Pending time.Duration `json:"pending,omitempty"` |
| 519 | // PendingAt is THIS BOX's own sighting of the narrower declaration — |
| 520 | // never Pipeline.SetAt, same argument as OffFirstSeen above: an |
| 521 | // attacker controlling the feed controls every timestamp in it, but |
| 522 | // not what the box already saw. A re-read that leaves Pending |
| 523 | // unchanged must not push this forward, or the clock would never |
| 524 | // elapse against a fleet that polls faster than the window itself. |
| 525 | PendingAt time.Time `json:"pending_at,omitzero"` |
| 526 | // DeletedAt is this box's own first sighting of the pipeline's ABSENCE |
| 527 | // for this channel — additive, and stamped only on the p == nil path. |
| 528 | // |
| 529 | // It exists because the pending machinery cannot carry the deletion |
| 530 | // ratchet on its own for every channel name. Deletion re-declares the |
| 531 | // name's LEGACY default, and for an app-defined name that default is |
| 532 | // HoldFor's 72h fail-closed fallback — LONGER than the window such a |
| 533 | // channel usually enforces, so it lands instantly (a longer bake always |
| 534 | // does) and stamps no Pending at all. The window is then perfectly safe |
| 535 | // and the memory looks, to any reader, exactly like a legacy app's: |
| 536 | // nothing pending, window == legacy. Without this stamp the adopt pass |
| 537 | // would have no evidence that this app was pipelined ninety seconds ago, |
| 538 | // would take the flagless legacy VERDICT, and one keyless DELETE would |
| 539 | // adopt a build minutes old — collapsing the very protection the ratchet |
| 540 | // exists to give (spec §3: "deletion never collapses a window |
| 541 | // instantly"). Re-observing absence must not restart it, exactly as |
| 542 | // PendingAt must not. |
| 543 | // |
| 544 | // It means "absence has been seen since the last VERIFIED read", which is |
| 545 | // a fact about the app, not about one channel: a verified pipeline clears |
| 546 | // it for every remembered channel under the app's prefix, including the |
| 547 | // ones that object does not name. Clearing only the named channels was |
| 548 | // poisonable — see the sweep in ObservePipeline's p != nil path. |
| 549 | // |
| 550 | // ADDITIVE, and one pass deep like every other field here — but NOT in |
| 551 | // the same direction, which is worth saying plainly. Holds's own caveat |
| 552 | // argues that a dropped ratchet memory restarts closed; that argument |
| 553 | // does not extend to this field. An old binary's Save drops DeletedAt |
| 554 | // along with the rest of Holds, and a dropped stamp does not delay a |
| 555 | // loosening — it removes the only evidence the adopt pass has that a |
| 556 | // deleted-pipeline app was ever pipelined, so floored comes up empty and |
| 557 | // the flagless legacy verdict adopts. That is FAIL OPEN, for the window |
| 558 | // between an old binary's Save and the next verified read (which, for a |
| 559 | // deleted pipeline, never comes). |
| 560 | // |
| 561 | // The exposure is bounded by deployment reality, not by the design: |
| 562 | // nothing in the fleet carries a Holds map at all yet — pipelines v2 has |
| 563 | // not shipped — so the only binary that could drop it is a rollback |
| 564 | // WITHIN this branch. If that stops being true before this ships, the |
| 565 | // mitigation is the same one spec §3 gives for the old-binary edge (the |
| 566 | // production: true co-write), and it wants saying in the spec rather |
| 567 | // than assumed here. |
| 568 | DeletedAt time.Time `json:"deleted_at,omitzero"` |
| 569 | } |
| 570 | |
| 571 | // liveRatchet reports whether this box is still running a pipeline ratchet |
| 572 | // for the channel — the adopt pass's evidence that the app IS pipelined as |
| 573 | // far as this box knows, independent of whether the memory happens to RAISE |
| 574 | // the window above legacy (an app-defined name's legacy resolution is 72h, |
| 575 | // so an ordinary 24h production channel raises nothing at all). |
| 576 | // |
| 577 | // Three shapes, any of which is live: |
| 578 | // - the enforced window differs from the name's legacy resolution, in |
| 579 | // EITHER direction: something other than the legacy ladder set it. |
| 580 | // - a pending loosening is waiting out its old window. |
| 581 | // - a deletion is still running its cycle: the window now ENFORCED has not |
| 582 | // yet elapsed on this box's own clock since the absence was first seen. |
| 583 | // |
| 584 | // Measuring that last cycle against the CURRENTLY enforced window is exact, |
| 585 | // not an approximation, because the two deletion shapes divide cleanly: |
| 586 | // |
| 587 | // - the legacy default is LONGER (every app-defined name, whose legacy |
| 588 | // resolution is HoldFor's 72h fallback): it lands immediately, stamps no |
| 589 | // Pending, and the enforced window is now that longer one — so the cycle |
| 590 | // served here is at least the window it replaced. |
| 591 | // - the legacy default is SHORTER (edge, 0): it goes PENDING instead, and |
| 592 | // the pending machinery runs the FULL old window before landing. By the |
| 593 | // time the enforced window drops to the shorter value, the cycle this |
| 594 | // clause would ask for has already been served in full — which is why a |
| 595 | // landed deletion on a zero-window channel is over the moment it lands, |
| 596 | // and why that is not a hole. |
| 597 | // |
| 598 | // The first two conditions cover the whole of the pending case anyway |
| 599 | // (Window != legacy, then PendingAt non-zero), so nothing is resting on this |
| 600 | // clause while a pending is in flight. |
| 601 | // |
| 602 | // name is the channel's own name, jitter and now the pass's (FirstSeen.Jitter |
| 603 | // and its single stamped instant), so every hold on the box answers the same |
| 604 | // clock. |
| 605 | func (mem *holdMemory) liveRatchet(name string, jitter float64, now time.Time) bool { |
| 606 | if mem == nil { |
| 607 | return false |
| 608 | } |
| 609 | if mem.Window != HoldFor(DefaultHolds, name) || !mem.PendingAt.IsZero() { |
| 610 | return true |
| 611 | } |
| 612 | if mem.DeletedAt.IsZero() { |
| 613 | return false |
| 614 | } |
| 615 | return now.Before(mem.DeletedAt.Add(time.Duration(float64(mem.Window) * (1 + jitter)))) |
| 616 | } |
| 617 | |
| 618 | // holdKeyPrefix is channelClockKey's key shape without the channel segment, |
| 619 | // for enumerating an app's remembered channels when no Pipeline is at hand |
| 620 | // (ObservePipeline's p == nil case). Mirrors channelClockKey's two shapes |
| 621 | // exactly, including the ops-account bare form. |
| 622 | func holdKeyPrefix(acct, app string) string { |
| 623 | if acct == "" || acct == OpsAccount { |
| 624 | return app + "/" |
| 625 | } |
| 626 | return acct + clockSep + app + clockSep |
| 627 | } |
| 628 | |
| 629 | // checkHoldPromotion applies mem's pending promotion if the OLD window has |
| 630 | // now elapsed on this box's own clock (mem.PendingAt), jittered the same |
| 631 | // way EligibleAt jitters a hold. Reports whether it promoted. nil and |
| 632 | // no-pending are both no-ops so callers can call it unconditionally. |
| 633 | func (f *FirstSeen) checkHoldPromotion(mem *holdMemory, now time.Time) bool { |
| 634 | if mem == nil || mem.PendingAt.IsZero() { |
| 635 | return false |
| 636 | } |
| 637 | elapsed := time.Duration(float64(mem.Window) * (1 + f.Jitter)) |
| 638 | if now.Before(mem.PendingAt.Add(elapsed)) { |
| 639 | return false |
| 640 | } |
| 641 | mem.Window = mem.Pending |
| 642 | mem.Pending = 0 |
| 643 | mem.PendingAt = time.Time{} |
| 644 | return true |
| 645 | } |
| 646 | |
| 647 | // ObservePipeline folds one verified pipeline read into this box's |
| 648 | // per-channel bake-window ratchet memory (release pipelines v2 spec §3). |
| 649 | // Returns whether anything changed; the caller persists via Save. |
| 650 | // |
| 651 | // A brand-new channel baselines at HoldFor(DefaultHolds, name) — the |
| 652 | // fail-closed default — never its own declared window, so a narrower |
| 653 | // window a brand-new channel declares ratchets in like any other tightening |
| 654 | // move rather than applying on first sight. A declared window that is no |
| 655 | // narrower than what's enforced tightens immediately; a narrower one goes |
| 656 | // pending and only takes effect once the OLD window has run its course, |
| 657 | // timed from this box's OWN sighting of the narrower declaration |
| 658 | // (PendingAt), never the pipeline's SetAt — the same argument as the |
| 659 | // production-flag ratchet and the channel clock itself: an attacker with |
| 660 | // bucket write but no signing key controls every timestamp in the object, |
| 661 | // but not what this box already saw. Pending promotion is checked on every |
| 662 | // call, including replays and p == nil, so a fleet that only polls this |
| 663 | // path occasionally still lands the promotion the pass after it elapses. |
| 664 | // |
| 665 | // p == nil (pipeline deleted, or absent this pass) is a policy declaration |
| 666 | // of the LEGACY defaults, ratcheted like any other: for every channel this |
| 667 | // app has memory of, HoldFor(DefaultHolds, name) goes through the same |
| 668 | // compare a declared window gets — no narrower than what's enforced applies |
| 669 | // immediately, narrower goes pending for one old-window cycle, and |
| 670 | // re-observed absence never restarts that clock. Deletion therefore never |
| 671 | // collapses a ratcheted window instantly (a bucket write alone can delete |
| 672 | // the object) and never pins an app to a dead pipeline's windows forever. |
| 673 | // An app with no memory at all — every legacy app, every pass — is a |
| 674 | // strict no-op here. Pending promotions still get their chance to land. |
| 675 | // |
| 676 | // A channel whose BakeWindow() fails to parse is skipped: old memory is |
| 677 | // kept (fail closed) and this never prevents any other channel in the same |
| 678 | // pipeline from being observed. |
| 679 | // |
| 680 | // Never call this with a pipeline that failed verification — like Observe |
| 681 | // and ObserveProduction, memory only ever ticks for a signed object. p == |
| 682 | // nil must mean genuinely, verifiably absent (the object deleted); a read |
| 683 | // or verify FAILURE is a different thing and must fail toward the longest |
| 684 | // window instead, never be handed to this function as nil — passing a |
| 685 | // corrupt read as nil would let it cheaply masquerade as deletion and |
| 686 | // inherit the calm "never loosens from absence" path above instead of |
| 687 | // failing closed. (Enforcing that at the call site is Task 5's job; this |
| 688 | // function's contract is documented here so that call site gets it right.) |
| 689 | func (f *FirstSeen) ObservePipeline(acct, app string, p *Pipeline, now time.Time) bool { |
| 690 | if f.Holds == nil { |
| 691 | f.Holds = map[string]*holdMemory{} |
| 692 | } |
| 693 | changed := false |
| 694 | if p == nil { |
| 695 | prefix := holdKeyPrefix(acct, app) |
| 696 | for key, mem := range f.Holds { |
| 697 | // mem == nil: {"holds":{"x":null}} parses cleanly into a nil |
| 698 | // entry — skip it, never dereference (same guard as the adopt |
| 699 | // pass's floor loops). |
| 700 | if mem == nil || !strings.HasPrefix(key, prefix) { |
| 701 | continue |
| 702 | } |
| 703 | name := key[len(prefix):] |
| 704 | if name == "" { |
| 705 | continue |
| 706 | } |
| 707 | // The legacy default is the newly-declared window, through the |
| 708 | // SAME compare as a declared pipeline. There is NO SetAt replay |
| 709 | // floor here: absence carries no SetAt to compare — and mem.SetAt |
| 710 | // is deliberately left standing, so a replayed pipeline object |
| 711 | // older than the one that was deleted still cannot tick the |
| 712 | // memory later. |
| 713 | // Stamp the absence itself, once. The window machinery below |
| 714 | // cannot express "this app was pipelined until just now" for a |
| 715 | // channel whose legacy default is LONGER than what it was |
| 716 | // enforcing (every app-defined name, whose legacy resolution is |
| 717 | // the 72h fallback): that lands instantly and leaves a memory |
| 718 | // indistinguishable from a legacy app's. DeletedAt is what the |
| 719 | // adopt pass's verdict rides on for one cycle afterwards — see |
| 720 | // holdMemory.DeletedAt and liveRatchet. Never restarted by |
| 721 | // re-observing absence, same rule as PendingAt. |
| 722 | if mem.DeletedAt.IsZero() { |
| 723 | mem.DeletedAt = now.UTC().Truncate(time.Second) |
| 724 | changed = true |
| 725 | } |
| 726 | w := HoldFor(DefaultHolds, name) |
| 727 | if w >= mem.Window { |
| 728 | if mem.Window != w || !mem.PendingAt.IsZero() { |
| 729 | mem.Window = w |
| 730 | mem.Pending = 0 |
| 731 | mem.PendingAt = time.Time{} |
| 732 | changed = true |
| 733 | } |
| 734 | } else if mem.PendingAt.IsZero() || mem.Pending != w { |
| 735 | // Only a NEW pending value stamps the clock: re-observation |
| 736 | // of absence must not restart it — same rule as a re-read. |
| 737 | mem.Pending = w |
| 738 | mem.PendingAt = now |
| 739 | changed = true |
| 740 | } |
| 741 | if f.checkHoldPromotion(mem, now) { |
| 742 | changed = true |
| 743 | } |
| 744 | } |
| 745 | return changed |
| 746 | } |
| 747 | // A verified pipeline ends every deletion cycle this app is carrying — |
| 748 | // EVERY remembered channel under its prefix, not only the ones this |
| 749 | // object happens to name. The stamp means "absence has been seen since |
| 750 | // the last verified read", which is a fact about the APP, so it is |
| 751 | // cleared the same way the p == nil branch stamps it: by prefix. |
| 752 | // |
| 753 | // Per-channel clearing was poisonable (PR #258 re-review). There is no |
| 754 | // object-level replay floor — only a per-channel SetAt one — so a |
| 755 | // HISTORICAL signed pipeline naming a SUBSET of the app's channels can be |
| 756 | // PUT back: delete (every channel stamped), replay an old edge-only |
| 757 | // object (edge cleared, production's stamp left standing while the app |
| 758 | // reads as verifiably pipelined), dwell past production's cycle (the |
| 759 | // stale stamp expires unnoticed), delete again (already non-zero, so |
| 760 | // never restamped). The app then carries no live ratchet at all and its |
| 761 | // held heads adopt with no bake — a keyless bucket write reaching exactly |
| 762 | // the outcome the ratchet exists to prevent. |
| 763 | if prefix := holdKeyPrefix(acct, app); len(f.Holds) > 0 { |
| 764 | for key, mem := range f.Holds { |
| 765 | if mem == nil || !strings.HasPrefix(key, prefix) || mem.DeletedAt.IsZero() { |
| 766 | continue |
| 767 | } |
| 768 | mem.DeletedAt = time.Time{} |
| 769 | changed = true |
| 770 | } |
| 771 | } |
| 772 | for _, ch := range p.Channels { |
| 773 | key := channelClockKey(acct, app, ch.Name) |
| 774 | mem := f.Holds[key] |
| 775 | if w, err := ch.BakeWindow(); err == nil { |
| 776 | firstSighting := mem == nil |
| 777 | if firstSighting { |
| 778 | mem = &holdMemory{Window: HoldFor(DefaultHolds, ch.Name)} |
| 779 | f.Holds[key] = mem |
| 780 | changed = true |
| 781 | } |
| 782 | // The replay floor: equal or older SetAt is either the same |
| 783 | // object re-read or a replay — no policy value change, though |
| 784 | // the pending-promotion check below still runs. |
| 785 | if firstSighting || p.SetAt.After(mem.SetAt) { |
| 786 | mem.SetAt = p.SetAt |
| 787 | changed = true |
| 788 | if w >= mem.Window { |
| 789 | mem.Window = w |
| 790 | mem.Pending = 0 |
| 791 | mem.PendingAt = time.Time{} |
| 792 | } else if mem.PendingAt.IsZero() || mem.Pending != w { |
| 793 | // PendingAt.IsZero() must stand on its own here, not |
| 794 | // just mem.Pending != w: a declared window of exactly |
| 795 | // ZERO is a legal value, and Pending's own zero value |
| 796 | // is indistinguishable from "nothing pending" — without |
| 797 | // this, a fresh channel whose first-ever declared bake |
| 798 | // is 0 would never stamp PendingAt (mem.Pending, freshly |
| 799 | // zeroed, already equals w=0), so checkHoldPromotion's |
| 800 | // PendingAt.IsZero() guard would short-circuit forever |
| 801 | // and the channel would sit pinned at its fail-closed |
| 802 | // baseline for the life of the box. |
| 803 | mem.Pending = w |
| 804 | mem.PendingAt = now |
| 805 | } |
| 806 | } |
| 807 | // OUTSIDE the replay floor, on every verified observation: a |
| 808 | // pending NARROWER than the currently-declared window can only |
| 809 | // have been stamped by something that is no longer the live |
| 810 | // policy — absence (the deletion ratchet), whose pipeline has |
| 811 | // since been restored byte-identically (same SetAt, so the |
| 812 | // policy block above is skipped; a same-object pending always |
| 813 | // equals its declared w). Left standing, it would land the |
| 814 | // legacy default one old-window cycle later and STAY, enforcing |
| 815 | // less than a live verified pipeline declares on every later |
| 816 | // pass. Cancel it — the closed direction: cancelling a |
| 817 | // loosening can only delay a shortening, never hasten one. |
| 818 | if mem != nil && !mem.PendingAt.IsZero() && mem.Pending < w { |
| 819 | mem.Pending, mem.PendingAt = 0, time.Time{} |
| 820 | changed = true |
| 821 | } |
| 822 | } |
| 823 | if f.checkHoldPromotion(mem, now) { |
| 824 | changed = true |
| 825 | } |
| 826 | } |
| 827 | return changed |
| 828 | } |
| 829 | |
| 830 | // HoldsForApp returns the ENFORCED per-channel bake window map for a |
| 831 | // pipelined app: one entry per channel p declares, using this box's |
| 832 | // ratchet memory where it has any and HoldFor(DefaultHolds, name) — the |
| 833 | // exact same fail-closed baseline ObservePipeline would latch on that |
| 834 | // channel's first sighting — for a channel not yet observed. The two |
| 835 | // paths therefore always agree, including for a rung name like "edge" |
| 836 | // whose baseline is 0: synthesizing it here (rather than leaving the |
| 837 | // channel absent from the map) matters precisely because HoldFor's OWN |
| 838 | // unknown-channel fallback is a hardcoded 72h, which would silently |
| 839 | // contradict a 0 baseline if this function ever left it out. nil when p |
| 840 | // == nil, so the caller falls back to DefaultHolds entirely — the |
| 841 | // pre-pipeline behaviour for an app with no pipeline at all. |
| 842 | // |
| 843 | // Never call this with a pipeline that failed verification, same |
| 844 | // prohibition as ObservePipeline: p == nil must mean genuinely absent, not |
| 845 | // "read or verify failed" — a verify failure must fail toward the longest |
| 846 | // window at the call site instead (Task 5), never reach here as nil. |
| 847 | func (f *FirstSeen) HoldsForApp(acct, app string, p *Pipeline) map[string]time.Duration { |
| 848 | if p == nil { |
| 849 | return nil |
| 850 | } |
| 851 | out := map[string]time.Duration{} |
| 852 | for _, ch := range p.Channels { |
| 853 | w := HoldFor(DefaultHolds, ch.Name) |
| 854 | if mem := f.Holds[channelClockKey(acct, app, ch.Name)]; mem != nil { |
| 855 | w = mem.Window |
| 856 | } |
| 857 | out[ch.Name] = w |
| 858 | } |
| 859 | return out |
| 860 | } |
| 861 | |
amadan