| 1 | package release |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "testing" |
| 11 | "time" |
| 12 | ) |
| 13 | |
| 14 | // Jitter is in [0, 0.25), so a hold of H is eligible somewhere in |
| 15 | // [firstSeen+H, firstSeen+1.25H). 91h is safely past stable's worst case |
| 16 | // (72h * 1.25 = 90h); 71h is safely inside the best case. |
| 17 | const ( |
| 18 | insideStableHold = 71 * time.Hour |
| 19 | pastStableHold = 91 * time.Hour |
| 20 | ) |
| 21 | |
| 22 | func TestBakeHoldBlocksNewStableUntilHeld(t *testing.T) { |
| 23 | path := filepath.Join(t.TempDir(), "first-seen.json") |
| 24 | st := LoadFirstSeen(path) |
| 25 | t0 := time.Now() |
| 26 | if !st.Observe("bab", "myapp", "stable", "v1", t0) { |
| 27 | t.Fatalf("first Observe reported no change") |
| 28 | } |
| 29 | if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0); hold == "" { |
| 30 | t.Fatalf("brand-new stable head not held") |
| 31 | } |
| 32 | if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0.Add(insideStableHold)); hold == "" { |
| 33 | t.Fatalf("stable head released before its 72h window") |
| 34 | } |
| 35 | if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, t0.Add(pastStableHold)); hold != "" { |
| 36 | t.Fatalf("stable head still held past its window: %s", hold) |
| 37 | } |
| 38 | // A version with NO first-seen record fails closed. |
| 39 | if hold := Hold(st, "bab", "myapp", "stable", "vGhost", DefaultHolds, false, t0.Add(pastStableHold)); hold == "" { |
| 40 | t.Fatalf("unseen version not held (must fail closed)") |
| 41 | } |
| 42 | // The local root-only override releases immediately. |
| 43 | if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, true, t0); hold != "" { |
| 44 | t.Fatalf("ignore=true still held: %s", hold) |
| 45 | } |
| 46 | // canary/* has no window: eligible the moment it is first seen. |
| 47 | st.Observe("bab", "myapp", "canary/x", "v1", t0) |
| 48 | if hold := Hold(st, "bab", "myapp", "canary/x", "v1", DefaultHolds, false, t0); hold != "" { |
| 49 | t.Fatalf("canary held: %s", hold) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | // TestRollbackNeverBakes is THE key property: a pointer moved BACK to a |
| 54 | // previously-seen version applies immediately — its first-seen record |
| 55 | // predates the hold (that's how it got installed the first time), and |
| 56 | // Observe never resets an existing timestamp. Evacuating a bad release must |
| 57 | // never itself sit in a bake window. |
| 58 | func TestRollbackNeverBakes(t *testing.T) { |
| 59 | s := NewDirStore(t.TempDir()) |
| 60 | ctx := context.Background() |
| 61 | priv, pub := testKeys(t) |
| 62 | shipRelease(t, s, "bab", "myapp", "v1", "linux-amd64", []byte("one")) |
| 63 | shipRelease(t, s, "bab", "myapp", "v2", "linux-amd64", []byte("two")) |
| 64 | |
| 65 | st := LoadFirstSeen(filepath.Join(t.TempDir(), "first-seen.json")) |
| 66 | t0 := time.Now() |
| 67 | |
| 68 | // v1 promoted to stable; the box first sees it at t0, bakes, installs. |
| 69 | if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v1"); err != nil { |
| 70 | t.Fatalf("Promote v1: %v", err) |
| 71 | } |
| 72 | ptr, err := Channel(ctx, s, pub, "bab", "myapp", "stable") |
| 73 | if err != nil { |
| 74 | t.Fatalf("Channel: %v", err) |
| 75 | } |
| 76 | st.Observe("bab", "myapp", "stable", ptr.Version, t0) |
| 77 | |
| 78 | // Time passes; v1's window elapses; v2 is promoted and first seen NOW. |
| 79 | t1 := t0.Add(pastStableHold) |
| 80 | if err := Promote(ctx, s, priv, "bab", "myapp", "stable", "v2"); err != nil { |
| 81 | t.Fatalf("Promote v2: %v", err) |
| 82 | } |
| 83 | ptr, _ = Channel(ctx, s, pub, "bab", "myapp", "stable") |
| 84 | st.Observe("bab", "myapp", "stable", ptr.Version, t1) |
| 85 | if hold := Hold(st, "bab", "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold == "" { |
| 86 | t.Fatalf("fresh v2 not held") |
| 87 | } |
| 88 | |
| 89 | // v2 turns out bad — rollback. The pointer is back at v1, and v1 applies |
| 90 | // IMMEDIATELY: its first-seen record is t0, long past the hold. Observe |
| 91 | // must not reset it. |
| 92 | if err := Rollback(ctx, s, priv, "bab", "myapp", "stable"); err != nil { |
| 93 | t.Fatalf("Rollback: %v", err) |
| 94 | } |
| 95 | ptr, err = Channel(ctx, s, pub, "bab", "myapp", "stable") |
| 96 | if err != nil { |
| 97 | t.Fatalf("Channel after rollback: %v", err) |
| 98 | } |
| 99 | if ptr.Version != "v1" { |
| 100 | t.Fatalf("rollback pointer = %s, want v1", ptr.Version) |
| 101 | } |
| 102 | st.Observe("bab", "myapp", "stable", ptr.Version, t1) // re-seen at head: no reset |
| 103 | if hold := Hold(st, "bab", "myapp", "stable", ptr.Version, DefaultHolds, false, t1); hold != "" { |
| 104 | t.Fatalf("rolled-back v1 held: %s (evacuation must never bake)", hold) |
| 105 | } |
| 106 | // And v2 stays held — the rollback didn't launder it. |
| 107 | if hold := Hold(st, "bab", "myapp", "stable", "v2", DefaultHolds, false, t1); hold == "" { |
| 108 | t.Fatalf("bad v2 no longer held after rollback") |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | func TestLoadFirstSeenCorruptFailsClosed(t *testing.T) { |
| 113 | path := filepath.Join(t.TempDir(), "first-seen.json") |
| 114 | if err := os.WriteFile(path, []byte("{ this is not json"), 0o644); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | st := LoadFirstSeen(path) |
| 118 | // A corrupt file is an empty clock: everything bakes from now — a wiped |
| 119 | // clock delays updates, it never lets one through early. |
| 120 | if hold := Hold(st, "bab", "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { |
| 121 | t.Fatalf("corrupt clock did not fail closed") |
| 122 | } |
| 123 | // Missing file: same. |
| 124 | st2 := LoadFirstSeen(filepath.Join(t.TempDir(), "absent.json")) |
| 125 | if hold := Hold(st2, "bab", "myapp", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { |
| 126 | t.Fatalf("missing clock did not fail closed") |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestFirstSeenSaveLoadRoundTrip(t *testing.T) { |
| 131 | path := filepath.Join(t.TempDir(), "first-seen.json") |
| 132 | st := LoadFirstSeen(path) |
| 133 | t0 := time.Now() |
| 134 | st.Observe("bab", "myapp", "stable", "v1", t0) |
| 135 | if st.Jitter < 0 || st.Jitter >= 0.25 { |
| 136 | t.Fatalf("minted jitter %v outside [0, 0.25)", st.Jitter) |
| 137 | } |
| 138 | if err := st.Save(); err != nil { |
| 139 | t.Fatalf("Save: %v", err) |
| 140 | } |
| 141 | st2 := LoadFirstSeen(path) |
| 142 | if st2.Jitter != st.Jitter { |
| 143 | t.Fatalf("jitter not persisted: %v != %v", st2.Jitter, st.Jitter) |
| 144 | } |
| 145 | at, ok := st2.EligibleAt("bab", "myapp", "stable", "v1", DefaultHolds) |
| 146 | if !ok { |
| 147 | t.Fatalf("first-seen record lost on reload") |
| 148 | } |
| 149 | want := t0.UTC().Truncate(time.Second).Add(time.Duration(float64(72*time.Hour) * (1 + st.Jitter))) |
| 150 | if !at.Equal(want) { |
| 151 | t.Fatalf("EligibleAt = %v, want %v", at, want) |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | func TestHoldForWildcardAndUnknown(t *testing.T) { |
| 156 | if d := HoldFor(DefaultHolds, "canary/any-branch"); d != 0 { |
| 157 | t.Fatalf("canary/* = %v, want 0", d) |
| 158 | } |
| 159 | if d := HoldFor(DefaultHolds, "beta"); d != 24*time.Hour { |
| 160 | t.Fatalf("beta = %v, want 24h", d) |
| 161 | } |
| 162 | if d := HoldFor(DefaultHolds, "stable"); d != 72*time.Hour { |
| 163 | t.Fatalf("stable = %v, want 72h", d) |
| 164 | } |
| 165 | // An unknown channel fails closed to the longest standard window. |
| 166 | if d := HoldFor(DefaultHolds, "mystery"); d != 72*time.Hour { |
| 167 | t.Fatalf("unknown channel = %v, want 72h (fail closed)", d) |
| 168 | } |
| 169 | // A per-deployment override map replaces the defaults wholesale. |
| 170 | custom := map[string]time.Duration{"stable": time.Hour} |
| 171 | if d := HoldFor(custom, "stable"); d != time.Hour { |
| 172 | t.Fatalf("override stable = %v, want 1h", d) |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | // The bake clock is per (account, app, channel, version). Keyed on the app |
| 177 | // name alone, one account's 72h stable observation would satisfy ANOTHER |
| 178 | // account's identically-named app on the same box — its supply-chain window |
| 179 | // silently skipped, which is the one thing this clock exists to prevent. |
| 180 | func TestBakeClockIsPerAccount(t *testing.T) { |
| 181 | st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) |
| 182 | long := time.Now().Add(-100 * time.Hour) |
| 183 | |
| 184 | // Account bab saw v1 at stable 100h ago; account kqmv has never seen it. |
| 185 | st.Observe("bab", "blog", "stable", "v1", long) |
| 186 | if hold := Hold(st, "bab", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold != "" { |
| 187 | t.Fatalf("the account that saw it 100h ago is held: %s", hold) |
| 188 | } |
| 189 | if hold := Hold(st, "kqmv", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { |
| 190 | t.Fatal("another account's identically-named app skipped its bake window " + |
| 191 | "on the strength of the first account's clock") |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | // The clock key must be injective. Channels may contain "/", so joining |
| 196 | // segments with "/" would make ops app "kass" + channel "canary/x" collide |
| 197 | // with account "kass" + app "canary" + channel "x" — one account's |
| 198 | // observation silently satisfying another's window, which is the same bug |
| 199 | // TestBakeClockIsPerAccount closes, reached by a different route. |
| 200 | func TestBakeClockKeyIsInjective(t *testing.T) { |
| 201 | st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) |
| 202 | long := time.Now().Add(-100 * time.Hour) |
| 203 | |
| 204 | // ops app "kass" on channel "canary/x". |
| 205 | st.Observe(OpsAccount, "kass", "canary/x", "v1", long) |
| 206 | // Account "kass", app "canary", channel "x" — three different segments |
| 207 | // that concatenate the same way, and must NOT resolve to that record. |
| 208 | if _, ok := st.EligibleAt("kass", "canary", "x", "v1", DefaultHolds); ok { |
| 209 | t.Fatal("the clock key collides across (account, app, channel) boundaries") |
| 210 | } |
| 211 | if hold := Hold(st, "kass", "canary", "x", "v1", DefaultHolds, false, time.Now()); hold == "" { |
| 212 | t.Fatal("a colliding key let another account skip its bake window") |
| 213 | } |
| 214 | // The record it was actually written under still resolves. |
| 215 | if hold := Hold(st, OpsAccount, "kass", "canary/x", "v1", DefaultHolds, false, time.Now()); hold != "" { |
| 216 | t.Fatalf("the real observation stopped resolving: %s", hold) |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | // A clock written before accounts existed has bare "<app>/<channel>" keys. |
| 221 | // Reading those as the ops account keeps every existing box's observations — |
| 222 | // otherwise the whole fleet's windows restart on upgrade, and nothing |
| 223 | // installs for 72h. |
| 224 | func TestBakeClockReadsPreAccountRecordsAsOps(t *testing.T) { |
| 225 | path := filepath.Join(t.TempDir(), "clock.json") |
| 226 | long := time.Now().Add(-100 * time.Hour).UTC().Truncate(time.Second) |
| 227 | legacy := map[string]any{ |
| 228 | "jitter": 0.1, |
| 229 | "channels": map[string]any{ |
| 230 | "blog/stable": map[string]any{"v1": long}, |
| 231 | }, |
| 232 | } |
| 233 | raw, err := json.MarshalIndent(legacy, "", " ") |
| 234 | if err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | if err := os.WriteFile(path, raw, 0o644); err != nil { |
| 238 | t.Fatal(err) |
| 239 | } |
| 240 | st := LoadFirstSeen(path) |
| 241 | if hold := Hold(st, OpsAccount, "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold != "" { |
| 242 | t.Fatalf("a pre-accounts observation was lost on upgrade: %s", hold) |
| 243 | } |
| 244 | // A non-ops account gets no free ride from it. |
| 245 | if hold := Hold(st, "kqmv", "blog", "stable", "v1", DefaultHolds, false, time.Now()); hold == "" { |
| 246 | t.Fatal("a pre-accounts record satisfied a different account's window") |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | func TestLoadFirstSeenMigrating(t *testing.T) { |
| 251 | now := time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) |
| 252 | |
| 253 | t.Run("prefers the new path when it exists", func(t *testing.T) { |
| 254 | dir := t.TempDir() |
| 255 | newPath := filepath.Join(dir, "new.json") |
| 256 | oldPath := filepath.Join(dir, "old.json") |
| 257 | stNew := LoadFirstSeen(newPath) |
| 258 | stNew.Observe("acct", "app", "stable", "v-new", now) |
| 259 | if err := stNew.Save(); err != nil { |
| 260 | t.Fatal(err) |
| 261 | } |
| 262 | stOld := LoadFirstSeen(oldPath) |
| 263 | stOld.Observe("acct", "app", "stable", "v-old", now) |
| 264 | if err := stOld.Save(); err != nil { |
| 265 | t.Fatal(err) |
| 266 | } |
| 267 | st := LoadFirstSeenMigrating(newPath, oldPath) |
| 268 | if _, ok := st.EligibleAt("acct", "app", "stable", "v-new", nil); !ok { |
| 269 | t.Fatal("new path's observation missing") |
| 270 | } |
| 271 | if _, ok := st.EligibleAt("acct", "app", "stable", "v-old", nil); ok { |
| 272 | t.Fatal("legacy content leaked in despite the new path existing") |
| 273 | } |
| 274 | }) |
| 275 | |
| 276 | t.Run("falls back to legacy content, saves to the new path, leaves legacy untouched", func(t *testing.T) { |
| 277 | dir := t.TempDir() |
| 278 | newPath := filepath.Join(dir, "new.json") |
| 279 | oldPath := filepath.Join(dir, "old.json") |
| 280 | stOld := LoadFirstSeen(oldPath) |
| 281 | stOld.Observe("acct", "app", "stable", "v1", now) |
| 282 | if err := stOld.Save(); err != nil { |
| 283 | t.Fatal(err) |
| 284 | } |
| 285 | legacyRaw, err := os.ReadFile(oldPath) |
| 286 | if err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | st := LoadFirstSeenMigrating(newPath, oldPath) |
| 290 | if _, ok := st.EligibleAt("acct", "app", "stable", "v1", nil); !ok { |
| 291 | t.Fatal("legacy observation not carried over") |
| 292 | } |
| 293 | if err := st.Save(); err != nil { |
| 294 | t.Fatal(err) |
| 295 | } |
| 296 | if _, err := os.Stat(newPath); err != nil { |
| 297 | t.Fatalf("Save did not write the new path: %v", err) |
| 298 | } |
| 299 | afterRaw, err := os.ReadFile(oldPath) |
| 300 | if err != nil || !bytes.Equal(legacyRaw, afterRaw) { |
| 301 | t.Fatalf("legacy file must be byte-identical after migration (err %v)", err) |
| 302 | } |
| 303 | }) |
| 304 | |
| 305 | t.Run("both missing is a fresh clock at the new path", func(t *testing.T) { |
| 306 | dir := t.TempDir() |
| 307 | newPath := filepath.Join(dir, "new.json") |
| 308 | st := LoadFirstSeenMigrating(newPath, filepath.Join(dir, "absent.json")) |
| 309 | st.Observe("acct", "app", "stable", "v1", now) |
| 310 | if err := st.Save(); err != nil { |
| 311 | t.Fatal(err) |
| 312 | } |
| 313 | if _, err := os.Stat(newPath); err != nil { |
| 314 | t.Fatalf("Save must create the new path: %v", err) |
| 315 | } |
| 316 | }) |
| 317 | } |
| 318 | |
| 319 | // TestFirstSeenSaveReplacesFileItCannotWriteInPlace stands in for the real |
| 320 | // production race (2026-08-03 review, CRITICAL): carlos-edge.service writes |
| 321 | // this path as user `carlos`, the root adopt one-shot writes it as root — |
| 322 | // whichever creates the file first locks the other uid out, because an |
| 323 | // in-place O_TRUNC write needs write permission on the FILE, and a 0644 file |
| 324 | // owned by the other uid has none for anyone else. A file this process |
| 325 | // cannot open for writing (0o444, no write bit for anyone, including its own |
| 326 | // owner) reproduces the same "cannot write in place" shape without needing a |
| 327 | // second uid: rename(2) only requires write permission on the containing |
| 328 | // directory, never on the file being replaced, so a Save implemented as |
| 329 | // temp-file+rename must still succeed here where a truncating write cannot. |
| 330 | func TestFirstSeenSaveReplacesFileItCannotWriteInPlace(t *testing.T) { |
| 331 | path := filepath.Join(t.TempDir(), "clock.json") |
| 332 | if err := os.WriteFile(path, []byte("{}"), 0o444); err != nil { |
| 333 | t.Fatal(err) |
| 334 | } |
| 335 | st := LoadFirstSeen(path) |
| 336 | st.Observe("acct", "app", "stable", "v1", time.Now()) |
| 337 | if err := st.Save(); err != nil { |
| 338 | t.Fatalf("Save must replace via rename, not an in-place write: %v", err) |
| 339 | } |
| 340 | // The temp name is minted fresh per attempt (CreateTemp's |
| 341 | // ".carlos-write-*"), not a fixed "<path>.tmp" — glob for the |
| 342 | // pattern rather than stat-ing one fixed name. |
| 343 | if matches, err := filepath.Glob(filepath.Join(filepath.Dir(path), ".carlos-write-*")); err != nil { |
| 344 | t.Fatalf("glob for stray temp files: %v", err) |
| 345 | } else if len(matches) != 0 { |
| 346 | t.Fatalf("Save left a temp file behind: %v", matches) |
| 347 | } |
| 348 | raw, err := os.ReadFile(path) |
| 349 | if err != nil { |
| 350 | t.Fatal(err) |
| 351 | } |
| 352 | var reloaded FirstSeen |
| 353 | if err := json.Unmarshal(raw, &reloaded); err != nil { |
| 354 | t.Fatalf("Save produced invalid/partial content: %v", err) |
| 355 | } |
| 356 | if _, ok := reloaded.Channels[channelClockKey("acct", "app", "stable")]["v1"]; !ok { |
| 357 | t.Fatal("observation missing after Save") |
| 358 | } |
| 359 | } |
| 360 | |
| 361 | // The pipelines spec defines edge as "picked up as it lands — no human in |
| 362 | // the loop, by definition". Without its own entry, HoldFor fail-closes edge |
| 363 | // to 72h — longer than beta, and a spec violation (2026-08-04 console bake |
| 364 | // controls spec, §1/§8). |
| 365 | func TestHoldForEdgeIsZero(t *testing.T) { |
| 366 | if d := HoldFor(DefaultHolds, "edge"); d != 0 { |
| 367 | t.Fatalf("edge hold = %v, want 0", d) |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | // The box-side production memory (2026-08-04 console bake controls spec §4). |
| 372 | // Scenario table, because the security content IS the state machine: the |
| 373 | // default is off; on is instant; off ratchets on the box's own clock; a |
| 374 | // DELETED flag never downgrades a box that saw true; a stale replayed off |
| 375 | // cannot roll back a newer on. |
| 376 | func TestProductionMemoryRatchet(t *testing.T) { |
| 377 | base := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC) |
| 378 | flag := func(prod bool, setAt time.Time) *ProductionFlag { |
| 379 | return &ProductionFlag{Schema: 1, Account: "bab", App: "kass", |
| 380 | Production: prod, SetAt: setAt} |
| 381 | } |
| 382 | newClock := func() *FirstSeen { |
| 383 | st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) |
| 384 | st.Jitter = 0 // determinism: eligibility is exactly firstSeen + hold |
| 385 | return st |
| 386 | } |
| 387 | |
| 388 | t.Run("default is off", func(t *testing.T) { |
| 389 | st := newClock() |
| 390 | if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 391 | t.Fatal("an app with no flag ever seen bakes") |
| 392 | } |
| 393 | }) |
| 394 | |
| 395 | t.Run("unverifiable flag fails toward baking", func(t *testing.T) { |
| 396 | st := newClock() |
| 397 | if !st.ProductionBake("bab", "kass", fmt.Errorf("sig"), true, DefaultHolds, base) { |
| 398 | t.Fatal("a corrupt flag read as the default instead of production") |
| 399 | } |
| 400 | }) |
| 401 | |
| 402 | t.Run("on is instant, deletion is not a downgrade", func(t *testing.T) { |
| 403 | st := newClock() |
| 404 | if !st.ObserveProduction("bab", "kass", flag(true, base), base) { |
| 405 | t.Fatal("first observation did not change the clock") |
| 406 | } |
| 407 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 408 | t.Fatal("production: true did not apply immediately") |
| 409 | } |
| 410 | // The object vanishes from the bucket (flag == nil every later pass): |
| 411 | // the box holds its verdict. |
| 412 | st.ObserveProduction("bab", "kass", nil, base.Add(time.Hour)) |
| 413 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour)) { |
| 414 | t.Fatal("deleting the flag object downgraded a box that saw true") |
| 415 | } |
| 416 | }) |
| 417 | |
| 418 | t.Run("off ratchets for the stable window", func(t *testing.T) { |
| 419 | st := newClock() |
| 420 | st.ObserveProduction("bab", "kass", flag(true, base), base) |
| 421 | off := flag(false, base.Add(time.Hour)) |
| 422 | st.ObserveProduction("bab", "kass", off, base.Add(2*time.Hour)) // box sees it an hour late |
| 423 | hold := HoldFor(DefaultHolds, RungStable) |
| 424 | // Still baking just inside the window, measured from the box's OWN |
| 425 | // sighting (base+2h), not the flag's set_at (base+1h). |
| 426 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold-time.Minute)) { |
| 427 | t.Fatal("off was honored before the ratchet elapsed") |
| 428 | } |
| 429 | if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold+time.Minute)) { |
| 430 | t.Fatal("off was not honored after the ratchet elapsed") |
| 431 | } |
| 432 | }) |
| 433 | |
| 434 | t.Run("no bakeable head skips the ratchet", func(t *testing.T) { |
| 435 | st := newClock() |
| 436 | st.ObserveProduction("bab", "kass", flag(true, base), base) |
| 437 | st.ObserveProduction("bab", "kass", flag(false, base.Add(time.Hour)), base.Add(time.Hour)) |
| 438 | if st.ProductionBake("bab", "kass", nil, false, DefaultHolds, base.Add(time.Hour)) { |
| 439 | t.Fatal("carve-out: nothing at a held head, yet off still ratcheted") |
| 440 | } |
| 441 | }) |
| 442 | |
| 443 | t.Run("a stale off cannot be replayed over a newer on", func(t *testing.T) { |
| 444 | st := newClock() |
| 445 | staleOff := flag(false, base) |
| 446 | st.ObserveProduction("bab", "kass", staleOff, base) |
| 447 | st.ObserveProduction("bab", "kass", flag(true, base.Add(time.Hour)), base.Add(time.Hour)) |
| 448 | // Attacker re-writes the old signed off object; the box refuses to |
| 449 | // regress below the newest set_at it has honored. |
| 450 | if st.ObserveProduction("bab", "kass", staleOff, base.Add(2*time.Hour)) { |
| 451 | t.Fatal("a replayed stale flag changed the clock") |
| 452 | } |
| 453 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(100*time.Hour)) { |
| 454 | t.Fatal("a replayed stale off eventually turned the bake off") |
| 455 | } |
| 456 | }) |
| 457 | |
| 458 | t.Run("memory survives a save/load round trip", func(t *testing.T) { |
| 459 | st := newClock() |
| 460 | st.ObserveProduction("bab", "kass", flag(true, base), base) |
| 461 | if err := st.Save(); err != nil { |
| 462 | t.Fatal(err) |
| 463 | } |
| 464 | st2 := LoadFirstSeen(st.path) |
| 465 | if !st2.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 466 | t.Fatal("production memory did not survive persistence") |
| 467 | } |
| 468 | }) |
| 469 | |
| 470 | // Task-4-review fix 2: prodKey must fold the ops account the same way |
| 471 | // channelClockKey does, or an app addressed as "" (the pre-accounts |
| 472 | // bare shape some callers still use) and the same app addressed as |
| 473 | // OpsAccount silently keep SEPARATE production memories. |
| 474 | t.Run("prodKey folds the ops account like channelClockKey", func(t *testing.T) { |
| 475 | st := newClock() |
| 476 | st.ObserveProduction(OpsAccount, "kass", flag(true, base), base) |
| 477 | if !st.ProductionBake("", "kass", nil, true, DefaultHolds, base) { |
| 478 | t.Fatal(`observed under OpsAccount, not found evaluating under ""`) |
| 479 | } |
| 480 | |
| 481 | st2 := newClock() |
| 482 | st2.ObserveProduction("", "kass", flag(true, base), base) |
| 483 | if !st2.ProductionBake(OpsAccount, "kass", nil, true, DefaultHolds, base) { |
| 484 | t.Fatal(`observed under "", not found evaluating under OpsAccount`) |
| 485 | } |
| 486 | // No free ride for an unrelated account: "bab"/"kass" (the ops |
| 487 | // account) must not satisfy "zzz"/"kass". |
| 488 | if st2.ProductionBake("zzz", "kass", nil, true, DefaultHolds, base) { |
| 489 | t.Fatal("an unrelated account satisfied the ops account's production memory") |
| 490 | } |
| 491 | }) |
| 492 | |
| 493 | // Task-4-review fix 3a: re-reading the SAME off object (the every-pass |
| 494 | // re-read, not a fresh sighting) must not change anything — in |
| 495 | // particular must not push the ratchet clock forward from the box's |
| 496 | // FIRST sighting to whatever pass happens to re-read it. |
| 497 | t.Run("equal set_at re-read does not reset the ratchet clock", func(t *testing.T) { |
| 498 | st := newClock() |
| 499 | st.ObserveProduction("bab", "kass", flag(true, base), base) |
| 500 | off := flag(false, base.Add(time.Hour)) |
| 501 | st.ObserveProduction("bab", "kass", off, base.Add(2*time.Hour)) // first sighting |
| 502 | if st.ObserveProduction("bab", "kass", off, base.Add(3*time.Hour)) { |
| 503 | t.Fatal("re-reading the identical off object reported a change") |
| 504 | } |
| 505 | hold := HoldFor(DefaultHolds, RungStable) |
| 506 | // Still measured from the FIRST sighting (base+2h), not the re-read |
| 507 | // (base+3h) — a re-read one hour later must not buy the flag one |
| 508 | // more hour of ratchet. |
| 509 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold-time.Minute)) { |
| 510 | t.Fatal("off honored before the ratchet from its first sighting elapsed") |
| 511 | } |
| 512 | if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(2*time.Hour).Add(hold+time.Minute)) { |
| 513 | t.Fatal("re-reading the same object pushed the ratchet clock forward") |
| 514 | } |
| 515 | }) |
| 516 | |
| 517 | // Task-4-review fix 3b: an in-progress off-ratchet (not just a bare |
| 518 | // "on" latch) must itself survive Save/LoadFirstSeen. |
| 519 | t.Run("off-state ratchet survives a save/load round trip", func(t *testing.T) { |
| 520 | st := newClock() |
| 521 | st.ObserveProduction("bab", "kass", flag(true, base), base) |
| 522 | st.ObserveProduction("bab", "kass", flag(false, base.Add(time.Hour)), base.Add(time.Hour)) |
| 523 | if err := st.Save(); err != nil { |
| 524 | t.Fatal(err) |
| 525 | } |
| 526 | st2 := LoadFirstSeen(st.path) |
| 527 | hold := HoldFor(DefaultHolds, RungStable) |
| 528 | if !st2.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(hold-time.Minute)) { |
| 529 | t.Fatal("off-state ratchet lost on reload: should still be baking mid-window") |
| 530 | } |
| 531 | if st2.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(hold+time.Minute)) { |
| 532 | t.Fatal("off-state ratchet lost on reload: should be off past the window") |
| 533 | } |
| 534 | }) |
| 535 | |
| 536 | // Task-4-review fix 3c: the ratchet is jittered exactly like any other |
| 537 | // hold (HoldFor's formula), not a bare, un-jittered 72h. |
| 538 | t.Run("the ratchet is jittered like any other hold", func(t *testing.T) { |
| 539 | st := LoadFirstSeen(filepath.Join(t.TempDir(), "clock.json")) |
| 540 | st.Jitter = 0.2 |
| 541 | st.ObserveProduction("bab", "kass", flag(true, base), base) |
| 542 | st.ObserveProduction("bab", "kass", flag(false, base.Add(time.Hour)), base.Add(time.Hour)) |
| 543 | hold := HoldFor(DefaultHolds, RungStable) // 72h, unjittered |
| 544 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(hold).Add(time.Minute)) { |
| 545 | t.Fatal("jitter not applied: off honored right at the un-jittered window") |
| 546 | } |
| 547 | jittered := time.Duration(float64(hold) * 1.2) |
| 548 | if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(time.Hour).Add(jittered).Add(time.Minute)) { |
| 549 | t.Fatal("jitter not applied: off still honored past the jittered window") |
| 550 | } |
| 551 | }) |
| 552 | |
| 553 | // Task-4-review fix 3d: an app declared off and NEVER on has nothing to |
| 554 | // ratchet — it must never bake, immediately and forever. |
| 555 | t.Run("an app declared off and never on never bakes", func(t *testing.T) { |
| 556 | st := newClock() |
| 557 | st.ObserveProduction("bab", "kass", flag(false, base), base) |
| 558 | if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 559 | t.Fatal("an app declared off, never on, baked immediately") |
| 560 | } |
| 561 | if st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base.Add(1000*time.Hour)) { |
| 562 | t.Fatal("an app declared off, never on, baked eventually") |
| 563 | } |
| 564 | }) |
| 565 | } |
| 566 | |
| 567 | // Task-4-review fix 1 (Critical): LoadFirstSeen must distinguish a file that |
| 568 | // read but did not PARSE from a file that is simply absent. Before this fix |
| 569 | // both collapsed to the same "empty clock" result, which for the CHANNEL |
| 570 | // clock is fine (fails closed either way — everything re-bakes from now), |
| 571 | // but for the PRODUCTION memory silently failed OPEN: a corrupt clock file |
| 572 | // (mem == nil for every app) made ProductionBake return false, instantly |
| 573 | // un-baking every production app on the box. A corrupt file must instead |
| 574 | // fail TOWARD baking (st.tainted), while an absent file — a genuinely fresh |
| 575 | // box — still gets the ordinary false default. |
| 576 | func TestCorruptClockFailsTowardBakingNotAwayFromIt(t *testing.T) { |
| 577 | base := time.Date(2026, 8, 4, 12, 0, 0, 0, time.UTC) |
| 578 | |
| 579 | path := filepath.Join(t.TempDir(), "clock.json") |
| 580 | if err := os.WriteFile(path, []byte("{ this is not json"), 0o644); err != nil { |
| 581 | t.Fatal(err) |
| 582 | } |
| 583 | st := LoadFirstSeen(path) |
| 584 | if !st.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 585 | t.Fatal("a corrupt clock file did not fail toward baking") |
| 586 | } |
| 587 | |
| 588 | // A genuinely fresh box (no file at all) is a different case entirely — |
| 589 | // still the ordinary false default, not tainted. |
| 590 | fresh := LoadFirstSeen(filepath.Join(t.TempDir(), "absent.json")) |
| 591 | if fresh.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 592 | t.Fatal("a fresh box with no clock file baked by default") |
| 593 | } |
| 594 | |
| 595 | // A file that EXISTS but can't be read at all (not "did not parse") is |
| 596 | // a different os.ReadFile failure mode than os.ErrNotExist — a |
| 597 | // directory sitting at the clock path is the easiest way to reproduce |
| 598 | // it portably (EISDIR), and review found the first cut of this fix only |
| 599 | // tainted the "read OK, parse failed" branch, leaving this one to |
| 600 | // silently fail open. |
| 601 | dirPath := filepath.Join(t.TempDir(), "clock.json") |
| 602 | if err := os.Mkdir(dirPath, 0o755); err != nil { |
| 603 | t.Fatal(err) |
| 604 | } |
| 605 | unreadable := LoadFirstSeen(dirPath) |
| 606 | if !unreadable.ProductionBake("bab", "kass", nil, true, DefaultHolds, base) { |
| 607 | t.Fatal("an unreadable (directory-at-path) clock file did not fail toward baking") |
| 608 | } |
| 609 | } |
| 610 | |
| 611 | func TestHoldMemoryRatchet(t *testing.T) { |
| 612 | f := &FirstSeen{Jitter: 0} |
| 613 | now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) |
| 614 | pipe := func(bake string, setAt time.Time) *Pipeline { |
| 615 | return &Pipeline{Schema: 1, Channels: []PipelineChannel{ |
| 616 | {Name: "edge"}, {Name: "production", Bake: bake}}, SetAt: setAt} |
| 617 | } |
| 618 | // First sighting: production baselines at the fail-closed 72h, NOT its declared 24h. |
| 619 | f.ObservePipeline("bae", "web", pipe("24h", now), now) |
| 620 | holds := f.HoldsForApp("bae", "web", pipe("24h", now)) |
| 621 | if holds["production"] != 72*time.Hour { |
| 622 | t.Fatalf("first-sighting window = %v; want fail-closed 72h baseline", holds["production"]) |
| 623 | } |
| 624 | if holds["edge"] != 0 { |
| 625 | t.Fatalf("edge window = %v; want 0 (legacy baseline)", holds["edge"]) |
| 626 | } |
| 627 | // After the old window elapses on the box's own clock, the pending 24h lands. |
| 628 | later := now.Add(72*time.Hour + time.Second) |
| 629 | f.ObservePipeline("bae", "web", pipe("24h", now), later) |
| 630 | if h := f.HoldsForApp("bae", "web", pipe("24h", now)); h["production"] != 24*time.Hour { |
| 631 | t.Fatalf("post-ratchet window = %v; want 24h", h["production"]) |
| 632 | } |
| 633 | // Tightening is immediate. |
| 634 | f.ObservePipeline("bae", "web", pipe("96h", now.Add(80*time.Hour)), later.Add(time.Minute)) |
| 635 | if h := f.HoldsForApp("bae", "web", nil); h != nil { |
| 636 | t.Fatalf("HoldsForApp(nil pipeline) = %v; want nil", h) |
| 637 | } |
| 638 | if h := f.HoldsForApp("bae", "web", pipe("96h", now)); h["production"] != 96*time.Hour { |
| 639 | t.Fatalf("tighten = %v; want 96h immediately", h["production"]) |
| 640 | } |
| 641 | } |
| 642 | |
| 643 | func TestHoldMemoryReplayFloorAndReReadKeepsClock(t *testing.T) { |
| 644 | f := &FirstSeen{Jitter: 0} |
| 645 | now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) |
| 646 | p1 := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "1h"}}, SetAt: now} |
| 647 | f.ObservePipeline("bae", "web", p1, now) |
| 648 | // Re-reading the same object an hour later must NOT restart the pending clock… |
| 649 | f.ObservePipeline("bae", "web", p1, now.Add(time.Hour)) |
| 650 | // …so the loosening lands 72h after the FIRST sighting, not the second. |
| 651 | f.ObservePipeline("bae", "web", p1, now.Add(72*time.Hour+time.Second)) |
| 652 | if h := f.HoldsForApp("bae", "web", p1); h["stable"] != time.Hour { |
| 653 | t.Fatalf("window = %v; want 1h landed 72h after first sighting", h["stable"]) |
| 654 | } |
| 655 | // A replayed OLDER SetAt can never loosen further. |
| 656 | old := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "0s"}}, SetAt: now.Add(-time.Hour)} |
| 657 | f.ObservePipeline("bae", "web", old, now.Add(73*time.Hour)) |
| 658 | if h := f.HoldsForApp("bae", "web", p1); h["stable"] != time.Hour { |
| 659 | t.Fatalf("replayed old object loosened the window to %v", h["stable"]) |
| 660 | } |
| 661 | } |
| 662 | |
| 663 | func TestHoldMemorySurvivesSaveLoadAndOldBinaryDrop(t *testing.T) { |
| 664 | // Additive-field discipline: marshal a FirstSeen with Holds, unmarshal, |
| 665 | // memory intact; unmarshal a PRE-CHANGE JSON (no holds key) into the new |
| 666 | // struct — no error, nil map, and ObservePipeline re-latches from scratch |
| 667 | // (the closed direction: a restarted clock only ever delays loosening). |
| 668 | f := &FirstSeen{Jitter: 0, Holds: map[string]*holdMemory{ |
| 669 | channelClockKey("bae", "web", "production"): {Window: 24 * time.Hour, SetAt: time.Now().UTC()}}} |
| 670 | b, err := json.Marshal(f) |
| 671 | if err != nil { |
| 672 | t.Fatal(err) |
| 673 | } |
| 674 | var back FirstSeen |
| 675 | if err := json.Unmarshal(b, &back); err != nil { |
| 676 | t.Fatal(err) |
| 677 | } |
| 678 | if back.Holds[channelClockKey("bae", "web", "production")].Window != 24*time.Hour { |
| 679 | t.Fatal("holds memory lost across marshal round trip") |
| 680 | } |
| 681 | var legacy FirstSeen |
| 682 | if err := json.Unmarshal([]byte(`{"jitter":0,"channels":{}}`), &legacy); err != nil || legacy.Holds != nil { |
| 683 | t.Fatalf("legacy clock parse = %v, holds %v", err, legacy.Holds) |
| 684 | } |
| 685 | } |
| 686 | |
| 687 | // TestHoldMemoryZeroBakeRatchetsIn is the review fix for the critical bug: |
| 688 | // Pending's zero value is indistinguishable from "nothing pending", and a |
| 689 | // declared bake of exactly ZERO is a legal value — without PendingAt as |
| 690 | // its own sentinel, a brand-new channel with no bake declared would latch |
| 691 | // its 72h fail-closed baseline and NEVER ratchet down, because |
| 692 | // mem.Pending (freshly zeroed) already equals w=0 on the very first call. |
| 693 | func TestHoldMemoryZeroBakeRatchetsIn(t *testing.T) { |
| 694 | f := &FirstSeen{Jitter: 0} |
| 695 | now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) |
| 696 | pipe := func(setAt time.Time) *Pipeline { |
| 697 | return &Pipeline{Schema: 1, Channels: []PipelineChannel{ |
| 698 | {Name: "dev"}, {Name: "live"}}, SetAt: setAt} |
| 699 | } |
| 700 | f.ObservePipeline("bae", "web", pipe(now), now) |
| 701 | if h := f.HoldsForApp("bae", "web", pipe(now)); h["dev"] != 72*time.Hour || h["live"] != 72*time.Hour { |
| 702 | t.Fatalf("first-sighting windows = %v; want fail-closed 72h baseline on both channels", h) |
| 703 | } |
| 704 | later := now.Add(72*time.Hour + time.Second) |
| 705 | if !f.ObservePipeline("bae", "web", pipe(now), later) { |
| 706 | t.Fatal("re-observe past the elapsed baseline did not report changed") |
| 707 | } |
| 708 | h := f.HoldsForApp("bae", "web", pipe(now)) |
| 709 | if h["dev"] != 0 || h["live"] != 0 { |
| 710 | t.Fatalf("post-ratchet windows = %v; want 0 (declared zero bake) on both channels", h) |
| 711 | } |
| 712 | } |
| 713 | |
| 714 | // TestHoldsForAppBeforeObserveReturnsBaselines is the review fix for |
| 715 | // HoldsForApp's fallback-agreement claim: before any ObservePipeline call, |
| 716 | // every declared channel must still report the exact baseline |
| 717 | // ObservePipeline would have latched on first sighting — including a rung |
| 718 | // name like "edge" whose baseline is 0, which HoldFor's own unknown-channel |
| 719 | // fallback (72h) would silently get wrong if left out of the map. |
| 720 | func TestHoldsForAppBeforeObserveReturnsBaselines(t *testing.T) { |
| 721 | f := &FirstSeen{Jitter: 0} |
| 722 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{ |
| 723 | {Name: "edge"}, {Name: "custom"}}, SetAt: time.Now()} |
| 724 | h := f.HoldsForApp("bae", "web", p) |
| 725 | if h["edge"] != 0 { |
| 726 | t.Fatalf("edge baseline = %v; want 0 (rung default) before any observation", h["edge"]) |
| 727 | } |
| 728 | if h["custom"] != 72*time.Hour { |
| 729 | t.Fatalf("custom baseline = %v; want 72h fail-closed before any observation", h["custom"]) |
| 730 | } |
| 731 | } |
| 732 | |
| 733 | // TestObservePipelineChangedContract pins the `changed` return value — |
| 734 | // PassFilter's sole trigger for a changed-gated Save — across the cases |
| 735 | // that matter: a genuine first sighting, a replay that changes nothing, a |
| 736 | // pending promotion actually landing, and p == nil with nothing to |
| 737 | // promote. |
| 738 | func TestObservePipelineChangedContract(t *testing.T) { |
| 739 | now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) |
| 740 | |
| 741 | t.Run("first sighting reports changed", func(t *testing.T) { |
| 742 | f := &FirstSeen{Jitter: 0} |
| 743 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "96h"}}, SetAt: now} |
| 744 | if !f.ObservePipeline("bae", "web", p, now) { |
| 745 | t.Fatal("first sighting did not report changed") |
| 746 | } |
| 747 | }) |
| 748 | |
| 749 | t.Run("replay of the same object with nothing pending reports no change", func(t *testing.T) { |
| 750 | f := &FirstSeen{Jitter: 0} |
| 751 | // 96h >= the 72h baseline: tightens immediately, no pending created. |
| 752 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "96h"}}, SetAt: now} |
| 753 | f.ObservePipeline("bae", "web", p, now) |
| 754 | if f.ObservePipeline("bae", "web", p, now.Add(time.Hour)) { |
| 755 | t.Fatal("replay of the identical object (nothing pending) reported changed") |
| 756 | } |
| 757 | }) |
| 758 | |
| 759 | t.Run("pending promotion firing reports changed", func(t *testing.T) { |
| 760 | f := &FirstSeen{Jitter: 0} |
| 761 | // 1h < the 72h baseline: goes pending at the first sighting. |
| 762 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "1h"}}, SetAt: now} |
| 763 | f.ObservePipeline("bae", "web", p, now) |
| 764 | if !f.ObservePipeline("bae", "web", p, now.Add(72*time.Hour+time.Second)) { |
| 765 | t.Fatal("pending promotion elapsing did not report changed") |
| 766 | } |
| 767 | }) |
| 768 | |
| 769 | t.Run("p == nil stamps the deletion once, then reports no change", func(t *testing.T) { |
| 770 | f := &FirstSeen{Jitter: 0} |
| 771 | // 72h == stable's legacy default: tightens to exactly the baseline, |
| 772 | // nothing pending — so absence has no WINDOW to ratchet down. The |
| 773 | // FIRST absence is still a change: it stamps DeletedAt, which is the |
| 774 | // box's own record that this app was pipelined until just now, and |
| 775 | // which the adopt pass's bake verdict rides on for one cycle (PR #258 |
| 776 | // review, C1 — for an app-defined channel name the legacy default is |
| 777 | // LONGER than the enforced window, so deletion lands instantly and |
| 778 | // this stamp is the only evidence left). A stamp that is not |
| 779 | // persisted is no evidence at all, so it must report changed. |
| 780 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "stable", Bake: "72h"}}, SetAt: now} |
| 781 | f.ObservePipeline("bae", "web", p, now) |
| 782 | if !f.ObservePipeline("bae", "web", nil, now.Add(time.Hour)) { |
| 783 | t.Fatal("the first p == nil did not report changed — the deletion stamp would never be saved") |
| 784 | } |
| 785 | // Every pass after it is the steady state a fleet actually sits in, |
| 786 | // and must not force a Save: re-observed absence changes nothing. |
| 787 | if f.ObservePipeline("bae", "web", nil, now.Add(2*time.Hour)) { |
| 788 | t.Fatal("re-observed absence reported changed — the stamp must not be restamped") |
| 789 | } |
| 790 | }) |
| 791 | |
| 792 | t.Run("p == nil with no memory at all reports no change", func(t *testing.T) { |
| 793 | // Every legacy app on every pass lands here — the byte-identical |
| 794 | // no-behaviour-change case the deletion ratchet must not disturb. |
| 795 | f := &FirstSeen{Jitter: 0} |
| 796 | if f.ObservePipeline("bae", "web", nil, now) { |
| 797 | t.Fatal("p == nil with no memory reported changed") |
| 798 | } |
| 799 | if len(f.Holds) != 0 { |
| 800 | t.Fatalf("p == nil minted memory out of nothing: %v", f.Holds) |
| 801 | } |
| 802 | }) |
| 803 | } |
| 804 | |
| 805 | // TestHoldMemoryDeletionRatchetsDownToLegacy: deleting pipeline.json is a |
| 806 | // bucket write, no signing key required — so absence must be treated as a |
| 807 | // POLICY DECLARATION of the legacy defaults, ratcheted like any other: a |
| 808 | // channel whose legacy default is narrower than what's enforced goes pending |
| 809 | // for one old-window cycle; deletion never collapses a window instantly, and |
| 810 | // never sticks forever. Re-observed absence must not restart the clock, and |
| 811 | // absence carries no SetAt, so the remembered replay floor stays untouched. |
| 812 | func TestHoldMemoryDeletionRatchetsDownToLegacy(t *testing.T) { |
| 813 | f := &FirstSeen{Jitter: 0} |
| 814 | now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) |
| 815 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "24h"}}, SetAt: now} |
| 816 | f.ObservePipeline("bae", "web", p, now) |
| 817 | key := channelClockKey("bae", "web", "edge") |
| 818 | if f.Holds[key].Window != 24*time.Hour { |
| 819 | t.Fatalf("window = %v, want 24h tightened immediately", f.Holds[key].Window) |
| 820 | } |
| 821 | setAt := f.Holds[key].SetAt |
| 822 | |
| 823 | // The deletion: edge's legacy default (0) is narrower than the enforced |
| 824 | // 24h, so it goes PENDING — the 24h stays enforced for one full cycle. |
| 825 | del := now.Add(time.Hour) |
| 826 | if !f.ObservePipeline("bae", "web", nil, del) { |
| 827 | t.Fatal("first observation of absence did not report changed — the pending clock would never persist") |
| 828 | } |
| 829 | mem := f.Holds[key] |
| 830 | if mem.Window != 24*time.Hour { |
| 831 | t.Fatalf("window = %v the pass after deletion, want the ratcheted 24h still enforced", mem.Window) |
| 832 | } |
| 833 | if mem.Pending != 0 || !mem.PendingAt.Equal(del) { |
| 834 | t.Fatalf("pending = %v at %v, want the legacy 0 pending from the deletion sighting %v", mem.Pending, mem.PendingAt, del) |
| 835 | } |
| 836 | if !mem.SetAt.Equal(setAt) { |
| 837 | t.Fatalf("SetAt = %v after absence, want %v untouched — absence carries no SetAt", mem.SetAt, setAt) |
| 838 | } |
| 839 | |
| 840 | // Re-observation of absence must NOT restart the clock — same rule as a |
| 841 | // re-read of the same object. |
| 842 | if f.ObservePipeline("bae", "web", nil, del.Add(time.Hour)) { |
| 843 | t.Fatal("re-observation of absence reported changed") |
| 844 | } |
| 845 | if !f.Holds[key].PendingAt.Equal(del) { |
| 846 | t.Fatalf("re-observation restarted the pending clock: %v, want %v", f.Holds[key].PendingAt, del) |
| 847 | } |
| 848 | |
| 849 | // After the OLD window elapses on the box's own clock, legacy lands. |
| 850 | if !f.ObservePipeline("bae", "web", nil, del.Add(24*time.Hour+time.Second)) { |
| 851 | t.Fatal("the pending legacy window elapsing did not report changed") |
| 852 | } |
| 853 | if f.Holds[key].Window != 0 { |
| 854 | t.Fatalf("window = %v after the old window elapsed, want edge's legacy 0", f.Holds[key].Window) |
| 855 | } |
| 856 | // Quiescent thereafter — nothing left to change, so the changed-gated |
| 857 | // Save is not forced every pass forever. |
| 858 | if f.ObservePipeline("bae", "web", nil, del.Add(48*time.Hour)) { |
| 859 | t.Fatal("absence at the legacy windows reported changed") |
| 860 | } |
| 861 | |
| 862 | // And a later replayed pipeline OLDER than the remembered SetAt still |
| 863 | // cannot tick the memory — deletion did not soften the replay floor. |
| 864 | old := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "1h"}}, SetAt: setAt.Add(-time.Hour)} |
| 865 | f.ObservePipeline("bae", "web", old, del.Add(49*time.Hour)) |
| 866 | if f.Holds[key].Window != 0 || !f.Holds[key].PendingAt.IsZero() { |
| 867 | t.Fatalf("replayed older pipeline ticked the memory after deletion: %+v", f.Holds[key]) |
| 868 | } |
| 869 | } |
| 870 | |
| 871 | // TestHoldMemoryDeletionTightensBelowLegacyImmediately: a channel whose |
| 872 | // enforced window sits BELOW its legacy default (a fully ratcheted-in short |
| 873 | // bake) tightens back to the legacy default the moment the pipeline is |
| 874 | // deleted — widening is always immediate, exactly as for a declared window. |
| 875 | func TestHoldMemoryDeletionTightensBelowLegacyImmediately(t *testing.T) { |
| 876 | f := &FirstSeen{Jitter: 0} |
| 877 | now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) |
| 878 | key := channelClockKey("bae", "web", "stable") |
| 879 | f.Holds = map[string]*holdMemory{key: {Window: time.Hour, SetAt: now}} |
| 880 | if !f.ObservePipeline("bae", "web", nil, now.Add(time.Minute)) { |
| 881 | t.Fatal("deletion tightening did not report changed") |
| 882 | } |
| 883 | mem := f.Holds[key] |
| 884 | if mem.Window != 72*time.Hour { |
| 885 | t.Fatalf("window = %v after deletion, want stable's fail-closed 72h immediately", mem.Window) |
| 886 | } |
| 887 | if mem.Pending != 0 || !mem.PendingAt.IsZero() { |
| 888 | t.Fatalf("pending = %v at %v after a tighten, want cleared", mem.Pending, mem.PendingAt) |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | // TestLiveRatchetIsWiderThanARaisedWindow pins the distinction PR #258's |
| 893 | // review turned on (C1): "is this box still running a pipeline ratchet for |
| 894 | // this channel" is NOT "does the memory raise the window above legacy". An |
| 895 | // app-defined name resolves to HoldFor's 72h fallback, so an ordinary |
| 896 | // production channel enforcing 24h raises nothing while being fully |
| 897 | // pipelined — and after a deletion its window lands on that same 72h |
| 898 | // instantly, leaving a memory that looks exactly like a legacy app's. The |
| 899 | // DeletedAt stamp is what carries the verdict across that gap, and it runs |
| 900 | // out. |
| 901 | func TestLiveRatchetIsWiderThanARaisedWindow(t *testing.T) { |
| 902 | now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) |
| 903 | legacyProduction := 72 * time.Hour // HoldFor's unknown-name fallback |
| 904 | |
| 905 | // Ratcheted DOWN below legacy: raises nothing, still live. |
| 906 | mem := &holdMemory{Window: 24 * time.Hour, SetAt: now} |
| 907 | if !mem.liveRatchet("production", 0, now) { |
| 908 | t.Error("a window ratcheted BELOW legacy did not read as a live ratchet — this is the C1 collapse") |
| 909 | } |
| 910 | // Raised above legacy: live, the case that always worked. |
| 911 | if !(&holdMemory{Window: 24 * time.Hour}).liveRatchet("edge", 0, now) { |
| 912 | t.Error("a window raised above legacy did not read as a live ratchet") |
| 913 | } |
| 914 | // Exactly legacy with a pending loosening: live. |
| 915 | if !(&holdMemory{Window: legacyProduction, Pending: time.Hour, PendingAt: now}).liveRatchet("production", 0, now) { |
| 916 | t.Error("a pending loosening did not read as a live ratchet") |
| 917 | } |
| 918 | // Exactly legacy, nothing pending, never deleted: a legacy app's memory. |
| 919 | if (&holdMemory{Window: legacyProduction, SetAt: now}).liveRatchet("production", 0, now) { |
| 920 | t.Error("a memory sitting at legacy with nothing in flight read as a live ratchet") |
| 921 | } |
| 922 | // Deleted, cycle still running / run out. |
| 923 | deleted := &holdMemory{Window: legacyProduction, SetAt: now, DeletedAt: now} |
| 924 | if !deleted.liveRatchet("production", 0, now.Add(71*time.Hour)) { |
| 925 | t.Error("a deletion inside its own cycle did not read as a live ratchet") |
| 926 | } |
| 927 | if deleted.liveRatchet("production", 0, now.Add(73*time.Hour)) { |
| 928 | t.Error("a deletion whose cycle has run out still read as live — the app must return to legacy") |
| 929 | } |
| 930 | // nil is never live (the {"holds":{"x":null}} shape). |
| 931 | if (*holdMemory)(nil).liveRatchet("production", 0, now) { |
| 932 | t.Error("a nil memory read as a live ratchet") |
| 933 | } |
| 934 | } |
| 935 | |
| 936 | // TestHoldMemoryDeletionStampClearedByAVerifiedPipeline: the stamp says "this |
| 937 | // app was pipelined until just now". A verified pipeline naming the channel |
| 938 | // says it IS pipelined, on the object's own word, so the stamp has no job |
| 939 | // left — and must not go on ticking a cycle that would later expire under a |
| 940 | // live pipeline. |
| 941 | func TestHoldMemoryDeletionStampClearedByAVerifiedPipeline(t *testing.T) { |
| 942 | f := &FirstSeen{Jitter: 0} |
| 943 | now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) |
| 944 | key := channelClockKey("bae", "web", "production") |
| 945 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "production", Bake: "24h"}}, SetAt: now} |
| 946 | f.ObservePipeline("bae", "web", p, now) |
| 947 | f.ObservePipeline("bae", "web", nil, now.Add(time.Hour)) |
| 948 | if f.Holds[key].DeletedAt.IsZero() { |
| 949 | t.Fatal("absence did not stamp DeletedAt") |
| 950 | } |
| 951 | stamp := f.Holds[key].DeletedAt |
| 952 | // Re-observed absence must not restamp it. |
| 953 | f.ObservePipeline("bae", "web", nil, now.Add(2*time.Hour)) |
| 954 | if !f.Holds[key].DeletedAt.Equal(stamp) { |
| 955 | t.Fatalf("re-observed absence restamped DeletedAt: %v, want %v", f.Holds[key].DeletedAt, stamp) |
| 956 | } |
| 957 | // The object comes back — same SetAt, the byte-identical restore. |
| 958 | if !f.ObservePipeline("bae", "web", p, now.Add(3*time.Hour)) { |
| 959 | t.Fatal("a restored pipeline that clears the deletion stamp did not report changed") |
| 960 | } |
| 961 | if !f.Holds[key].DeletedAt.IsZero() { |
| 962 | t.Fatalf("DeletedAt = %v after the pipeline was restored, want cleared", f.Holds[key].DeletedAt) |
| 963 | } |
| 964 | } |
| 965 | |
| 966 | // TestHoldMemoryRestoredPipelineCancelsAbsencePending: the re-review probe. |
| 967 | // Delete the pipeline (absence stamps the legacy default pending), then |
| 968 | // restore the object BYTE-IDENTICALLY — same SetAt: a bucket-versioning |
| 969 | // restore, an undo, a replay. The replay floor rightly skips the policy |
| 970 | // block, but the absence-stamped pending must not survive a live verified |
| 971 | // pipeline declaring more: unlanded it is cancelled, and one old-window |
| 972 | // cycle later the enforced window is still the declared 24h, never the |
| 973 | // legacy 0 the deletion briefly put in flight. |
| 974 | func TestHoldMemoryRestoredPipelineCancelsAbsencePending(t *testing.T) { |
| 975 | f := &FirstSeen{Jitter: 0} |
| 976 | now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) |
| 977 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "24h"}}, SetAt: now} |
| 978 | f.ObservePipeline("bae", "web", p, now) |
| 979 | key := channelClockKey("bae", "web", "edge") |
| 980 | |
| 981 | // The deletion: legacy 0 goes pending under the enforced 24h. |
| 982 | del := now.Add(time.Hour) |
| 983 | f.ObservePipeline("bae", "web", nil, del) |
| 984 | if mem := f.Holds[key]; mem.Pending != 0 || !mem.PendingAt.Equal(del) { |
| 985 | t.Fatalf("absence did not stamp the legacy pending: %+v", mem) |
| 986 | } |
| 987 | |
| 988 | // The restore, byte-identical (same SetAt). The pending is no longer |
| 989 | // the live policy — it must be cancelled, and that is a change. |
| 990 | if !f.ObservePipeline("bae", "web", p, del.Add(time.Hour)) { |
| 991 | t.Fatal("cancelling the absence pending did not report changed — it would never persist") |
| 992 | } |
| 993 | if mem := f.Holds[key]; mem.Window != 24*time.Hour || mem.Pending != 0 || !mem.PendingAt.IsZero() { |
| 994 | t.Fatalf("memory after the restore = %+v, want 24h enforced with nothing pending", mem) |
| 995 | } |
| 996 | |
| 997 | // Past the old window, still observing the restored pipeline: the |
| 998 | // declared 24h holds — the legacy 0 never lands. |
| 999 | if f.ObservePipeline("bae", "web", p, del.Add(25*time.Hour)) { |
| 1000 | t.Fatal("a quiescent restored pipeline reported changed") |
| 1001 | } |
| 1002 | if mem := f.Holds[key]; mem.Window != 24*time.Hour || !mem.PendingAt.IsZero() { |
| 1003 | t.Fatalf("memory past the old window = %+v, want the declared 24h still enforced", mem) |
| 1004 | } |
| 1005 | } |
| 1006 | |
| 1007 | // TestObservePipelineNullHoldsEntryDoesNotPanic: {"holds":{"x":null}} parses |
| 1008 | // cleanly into a nil *holdMemory. Every loop over Holds must skip it, never |
| 1009 | // dereference it — a panic here takes down the adopt pass fleet-wide. |
| 1010 | func TestObservePipelineNullHoldsEntryDoesNotPanic(t *testing.T) { |
| 1011 | path := filepath.Join(t.TempDir(), "clock.json") |
| 1012 | blob := `{"jitter":0,"channels":{},"holds":{"` + channelClockKey("bae", "web", "edge") + `":null}}` |
| 1013 | if err := os.WriteFile(path, []byte(blob), 0o644); err != nil { |
| 1014 | t.Fatal(err) |
| 1015 | } |
| 1016 | f := LoadFirstSeen(path) |
| 1017 | if f.tainted { |
| 1018 | t.Fatal("a null holds entry is legal JSON, not a tainted clock") |
| 1019 | } |
| 1020 | now := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) |
| 1021 | f.ObservePipeline("bae", "web", nil, now) // must not panic |
| 1022 | p := &Pipeline{Schema: 1, Channels: []PipelineChannel{{Name: "edge", Bake: "24h"}}, SetAt: now} |
| 1023 | f.ObservePipeline("bae", "web", p, now) // must not panic; re-latches over the null |
| 1024 | if f.Holds[channelClockKey("bae", "web", "edge")].Window != 24*time.Hour { |
| 1025 | t.Fatal("observation over a null entry did not re-latch the memory") |
| 1026 | } |
| 1027 | } |
| 1028 | |
amadan