| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "errors" |
| 6 | "math" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/activator" |
| 13 | ) |
| 14 | |
| 15 | // The crash-epoch hard limit (zero-downtime flag readiness round 2, Task 1; |
| 16 | // Paul's ruling 2026-10-12): a stale signed retry grant never clears a later |
| 17 | // failure. A restriction's generation is per (account, app), moves on every |
| 18 | // change of the app's restrictions, is published only once it is on disk, |
| 19 | // never repeats across a restart, and counts in a book identity that comes |
| 20 | // from crypto/rand and is rotated when the rule is adopted. |
| 21 | |
| 22 | // diskBook is a book saved under a data dir of its own, its owed save |
| 23 | // landed, and the dir. |
| 24 | func diskBook(t *testing.T) (*FailureBook, string) { |
| 25 | t.Helper() |
| 26 | dd := t.TempDir() |
| 27 | b, err := OpenFailureBook(dd) |
| 28 | if err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | if err := b.Flush(); err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | return b, dd |
| 35 | } |
| 36 | |
| 37 | // published is the generation status publishes for build's restriction of |
| 38 | // host in bab/blog. |
| 39 | func published(b *FailureBook, build, host string) uint64 { |
| 40 | return b.PublishedEpoch("bab", "blog", build, host) |
| 41 | } |
| 42 | |
| 43 | // grantAt applies a signed host retry of build on host naming b's identity |
| 44 | // at epoch. |
| 45 | func grantAt(b *FailureBook, build, host string, epoch uint64) (rec, mark bool) { |
| 46 | rec, mark, _, _ = b.RetryHostAt("bab", "blog", build, host, b.ID(), epoch) |
| 47 | return rec, mark |
| 48 | } |
| 49 | |
| 50 | func mustRecord(t *testing.T, b *FailureBook, r FailureRecord) RecordResult { |
| 51 | t.Helper() |
| 52 | res, err := b.Record(r) |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | return res |
| 57 | } |
| 58 | |
| 59 | func mustMark(t *testing.T, b *FailureBook, host string, h HoldRan) { |
| 60 | t.Helper() |
| 61 | if err := b.MarkHoldRan(host, h); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | // Plan 8: a book whose save fails publishes nothing, and a grant for the |
| 67 | // generation published before the change clears nothing. Once the owed save |
| 68 | // lands the new generation is published and a grant at it clears. |
| 69 | func TestAFailingDiskPublishesNoEpoch(t *testing.T) { |
| 70 | b, dd := diskBook(t) |
| 71 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1", Mode: string(activator.ModeHold), HeldDown: []string{"h1"}}) |
| 72 | e1 := published(b, "v2", "h1") |
| 73 | if e1 == 0 { |
| 74 | t.Fatal("a saved restriction published no generation") |
| 75 | } |
| 76 | readOnly(t, dd) |
| 77 | if _, err := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "h2"}); err == nil { |
| 78 | t.Fatal("the save of v3's record did not fail: the case needs a failing disk") |
| 79 | } |
| 80 | if e := published(b, "v2", "h1"); e != 0 { |
| 81 | t.Fatalf("published %d while the book's save is owed, want 0", e) |
| 82 | } |
| 83 | if rec, mark, _, err := b.RetryHostAt("bab", "blog", "v2", "h1", b.ID(), e1); rec || mark { |
| 84 | t.Fatalf("a grant from before the unsaved change cleared record %v, marker %v (%v)", rec, mark, err) |
| 85 | } |
| 86 | if !b.HeldDown("bab", "blog", "v2", "h1") { |
| 87 | t.Fatal("h1 is no longer held down") |
| 88 | } |
| 89 | writable(t, dd) |
| 90 | if err := b.SaveIfDirty(); err != nil { |
| 91 | t.Fatalf("the owed save did not land: %v", err) |
| 92 | } |
| 93 | e2 := published(b, "v2", "h1") |
| 94 | if e2 <= e1 { |
| 95 | t.Fatalf("published %d after the save landed, want past %d", e2, e1) |
| 96 | } |
| 97 | if rec, _ := grantAt(b, "v2", "h1", e2); !rec { |
| 98 | t.Fatal("a grant at the published generation cleared nothing") |
| 99 | } |
| 100 | again, err := OpenFailureBook(dd) |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | if again.HeldDown("bab", "blog", "v2", "h1") { |
| 105 | t.Fatal("the retry was not saved") |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | // Plan 1: every removal of a restriction of the app moves the app's |
| 110 | // generation, so a grant signed before it clears nothing. |
| 111 | func TestEveryRemovalMovesTheGeneration(t *testing.T) { |
| 112 | const host = "h1" |
| 113 | // v2's record (from a sibling's failure) holds h1, and v2's marker |
| 114 | // stands on h1. |
| 115 | setup := func(t *testing.T) *FailureBook { |
| 116 | b, _ := diskBook(t) |
| 117 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: promotedAt, At: t0}) |
| 118 | mustMark(t, b, host, HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a1", At: t0}) |
| 119 | return b |
| 120 | } |
| 121 | for name, change := range map[string]func(t *testing.T, b *FailureBook){ |
| 122 | "the marker replaced by another build's": func(t *testing.T, b *FailureBook) { |
| 123 | mustMark(t, b, host, HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a2", At: t0}) |
| 124 | }, |
| 125 | "the record cleared by a pointer": func(t *testing.T, b *FailureBook) { |
| 126 | if ok, err := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(time.Minute)); !ok || err != nil { |
| 127 | t.Fatalf("ClearOnPointer = %v, %v", ok, err) |
| 128 | } |
| 129 | }, |
| 130 | "the record pruned": func(t *testing.T, b *FailureBook) { |
| 131 | if err := b.Prune(func(string, string, string) bool { return false }); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | if _, ok := b.Lookup("bab", "blog", "v2"); ok { |
| 135 | t.Fatal("the record was not pruned") |
| 136 | } |
| 137 | }, |
| 138 | "the marker cleared by a good write": func(t *testing.T, b *FailureBook) { |
| 139 | if ok, err := b.ClearHoldRanBefore(host, b.Snapshot()); !ok || err != nil { |
| 140 | t.Fatalf("ClearHoldRanBefore = %v, %v", ok, err) |
| 141 | } |
| 142 | }, |
| 143 | "the marker cleared by a rollback": func(t *testing.T, b *FailureBook) { |
| 144 | if ok, err := b.ClearHoldRanOnPointer(host, "bab", "blog", t0.Add(time.Minute)); !ok || err != nil { |
| 145 | t.Fatalf("ClearHoldRanOnPointer = %v, %v", ok, err) |
| 146 | } |
| 147 | }, |
| 148 | "the marker cleared": func(t *testing.T, b *FailureBook) { |
| 149 | if ok, err := b.ClearHoldRan(host); !ok || err != nil { |
| 150 | t.Fatalf("ClearHoldRan = %v, %v", ok, err) |
| 151 | } |
| 152 | }, |
| 153 | "the record merged with a newer pointer": func(t *testing.T, b *FailureBook) { |
| 154 | if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib2", PromotedAt: promotedAt.Add(time.Minute)}); res != Merged { |
| 155 | t.Fatalf("Record = %v, want merged", res) |
| 156 | } |
| 157 | }, |
| 158 | } { |
| 159 | t.Run(name, func(t *testing.T) { |
| 160 | b := setup(t) |
| 161 | movesAcross(t, b, "blog", "v2", host, func() { change(t, b) }) |
| 162 | }) |
| 163 | } |
| 164 | // Fix round 1, finding 1a: a grant that clears only a marker (no record |
| 165 | // of its build holds the host) moves the app's generation, so a grant at |
| 166 | // the value published before it for another build of the app clears |
| 167 | // nothing. |
| 168 | t.Run("a marker-only grant for another build", func(t *testing.T) { |
| 169 | b, _ := diskBook(t) |
| 170 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: promotedAt, At: t0}) |
| 171 | mustMark(t, b, host, HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a1", At: t0}) |
| 172 | movesAcross(t, b, "blog", "v2", "h2", func() { |
| 173 | if rec, mark := grantAt(b, "v3", host, published(b, "v3", host)); rec || !mark { |
| 174 | t.Fatalf("the grant for v3 on h1 cleared record %v, marker %v; want the marker only", rec, mark) |
| 175 | } |
| 176 | }) |
| 177 | }) |
| 178 | // Fix round 1, finding 1b: a host's marker of one app replaced by another |
| 179 | // app's (the host now runs another app) moves the first app's generation. |
| 180 | t.Run("the marker replaced by another app's", func(t *testing.T) { |
| 181 | b := setup(t) |
| 182 | movesAcross(t, b, "blog", "v2", host, func() { |
| 183 | mustMark(t, b, host, HoldRan{Account: "bab", App: "shop", Build: "w1", Exec: "a2", At: t0}) |
| 184 | }) |
| 185 | }) |
| 186 | } |
| 187 | |
| 188 | // movesAcross checks that change moves bab/app's generation as build's |
| 189 | // restriction of host publishes it, that build still restricts host after |
| 190 | // it, and that a grant at the value published before it clears nothing. |
| 191 | func movesAcross(t *testing.T, b *FailureBook, app, build, host string, change func()) { |
| 192 | t.Helper() |
| 193 | e := b.PublishedEpoch("bab", app, build, host) |
| 194 | if e == 0 { |
| 195 | t.Fatal("no generation published before the change") |
| 196 | } |
| 197 | change() |
| 198 | now := b.PublishedEpoch("bab", app, build, host) |
| 199 | if now == e { |
| 200 | t.Fatalf("the generation stayed at %d across the change", e) |
| 201 | } |
| 202 | if now == 0 { |
| 203 | t.Fatalf("%s still restricts %s through its record or marker, but no generation is published", build, host) |
| 204 | } |
| 205 | if rec, mark, _, _ := b.RetryHostAt("bab", app, build, host, b.ID(), e); rec || mark { |
| 206 | t.Fatalf("a grant from before the change cleared record %v, marker %v", rec, mark) |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // A call that changes nothing (a stale failure, an unchanged one, a repeated |
| 211 | // marker of the same execution) moves nothing: a grant at the generation |
| 212 | // published before it still clears. |
| 213 | func TestANoOpOutcomeDoesNotMoveTheGeneration(t *testing.T) { |
| 214 | b, _ := diskBook(t) |
| 215 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Mode: string(activator.ModeHold), |
| 216 | HeldDown: []string{"sib"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) |
| 217 | snap := b.Snapshot() // an execution of v2 on h1 starts |
| 218 | if rec, _ := grantAt(b, "v2", "h1", published(b, "v2", "h1")); !rec { |
| 219 | t.Fatal("the retry of h1 cleared nothing") |
| 220 | } |
| 221 | e := published(b, "v2", "h2") |
| 222 | if e == 0 { |
| 223 | t.Fatal("no generation published for h2") |
| 224 | } |
| 225 | steady := func(what string) { |
| 226 | t.Helper() |
| 227 | if now := published(b, "v2", "h2"); now != e { |
| 228 | t.Fatalf("%s moved the generation from %d to %d", what, e, now) |
| 229 | } |
| 230 | } |
| 231 | if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1", Seq: snap}); res != PreRetry { |
| 232 | t.Fatalf("the failure that began before the retry = %v, want pre-retry", res) |
| 233 | } |
| 234 | steady("a pre-retry failure") |
| 235 | if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib"}); res != Unchanged { |
| 236 | t.Fatalf("a repeated failure = %v, want unchanged", res) |
| 237 | } |
| 238 | steady("an unchanged failure") |
| 239 | mustMark(t, b, "sib", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a1", At: t0}) |
| 240 | steady("a repeated marker of the same execution") |
| 241 | if res := mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Mode: string(activator.ModeHold), |
| 242 | HeldDown: []string{"sib"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}); res != Unchanged { |
| 243 | t.Fatalf("a repeated hold failure = %v, want unchanged", res) |
| 244 | } |
| 245 | steady("a repeated hold failure of the same execution") |
| 246 | if rec, _ := grantAt(b, "v2", "h2", e); !rec { |
| 247 | t.Fatal("a grant at the unchanged generation cleared nothing") |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | // Plan 2: a file an earlier binary wrote (no durable_epochs) may hold |
| 252 | // epochs that binary published and never saved. Opening it under the rule |
| 253 | // rotates the book's identity, so no grant issued before clears anything, |
| 254 | // and nothing is published until the rotation is on disk. |
| 255 | func TestABookFromBeforeTheRuleGetsANewIdentity(t *testing.T) { |
| 256 | dd := t.TempDir() |
| 257 | raw, err := json.Marshal(map[string]any{ |
| 258 | "schema": 2, |
| 259 | "book": "old", |
| 260 | "seq": 10, |
| 261 | "records": []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "h1", |
| 262 | "held_down": []string{"h1"}, "epochs": map[string]uint64{"": 11, "h1": 11}}}, |
| 263 | }) |
| 264 | if err != nil { |
| 265 | t.Fatal(err) |
| 266 | } |
| 267 | if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { |
| 268 | t.Fatal(err) |
| 269 | } |
| 270 | b, err := OpenFailureBook(dd) |
| 271 | if err != nil { |
| 272 | t.Fatal(err) |
| 273 | } |
| 274 | id := b.ID() |
| 275 | if id == "old" || id == "" { |
| 276 | t.Fatalf("identity %q after adopting the rule, want a new one", id) |
| 277 | } |
| 278 | if e := published(b, "v2", "h1"); e != 0 { |
| 279 | t.Fatalf("published %d before the rotation was saved, want 0", e) |
| 280 | } |
| 281 | for _, epoch := range []uint64{11, 12} { |
| 282 | if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h1", "old", epoch); rec || mark { |
| 283 | t.Fatalf("a grant naming the old identity at %d cleared record %v, marker %v", epoch, rec, mark) |
| 284 | } |
| 285 | } |
| 286 | if err := b.Flush(); err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | e := published(b, "v2", "h1") |
| 290 | if e <= 11 { |
| 291 | t.Fatalf("published %d after the save, want a generation past the file's 11", e) |
| 292 | } |
| 293 | again, err := OpenFailureBook(dd) |
| 294 | if err != nil { |
| 295 | t.Fatal(err) |
| 296 | } |
| 297 | if again.ID() != id || published(again, "v2", "h1") != e { |
| 298 | t.Fatalf("reopened as %q at %d, want %q at %d", again.ID(), published(again, "v2", "h1"), id, e) |
| 299 | } |
| 300 | if f := fileOf(t, dd); !f.Durable || f.Book != id { |
| 301 | t.Fatalf("file durable_epochs %v, book %q: want true and %q", f.Durable, f.Book, id) |
| 302 | } |
| 303 | if rec, _ := grantAt(again, "v2", "h1", e); !rec { |
| 304 | t.Fatal("a grant naming the new identity at the published generation cleared nothing") |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | // Fix round 1, finding 2: the book is written as schema 3, which an agent |
| 309 | // before the rule (it accepts schema 1 and 2) refuses, so it can never keep |
| 310 | // the identity while publishing unsaved epochs. A schema-3 file round-trips |
| 311 | // with its identity and owes no save. A file of schema 1 or 2 still opens, |
| 312 | // with a new identity, even one that claims durable_epochs: only an agent of |
| 313 | // this rule writes schema 3. |
| 314 | func TestTheBookIsSchema3AndOlderSchemasOpenRotated(t *testing.T) { |
| 315 | t.Run("schema 3 round-trips", func(t *testing.T) { |
| 316 | b, dd := diskBook(t) |
| 317 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1", HeldDown: []string{"h1"}}) |
| 318 | e := published(b, "v2", "h1") |
| 319 | if e == 0 { |
| 320 | t.Fatal("a saved restriction published no generation") |
| 321 | } |
| 322 | if f := fileOf(t, dd); f.Schema != 3 || !f.Durable || f.Book != b.ID() { |
| 323 | t.Fatalf("file schema %d, durable_epochs %v, book %q: want 3, true, %q", f.Schema, f.Durable, f.Book, b.ID()) |
| 324 | } |
| 325 | again, err := OpenFailureBook(dd) |
| 326 | if err != nil { |
| 327 | t.Fatalf("a schema-3 file did not open: %v", err) |
| 328 | } |
| 329 | if again.ID() != b.ID() || again.SaveOwed() || published(again, "v2", "h1") != e { |
| 330 | t.Fatalf("reopened as %q, save owed %v, at %d: want %q, false, %d", |
| 331 | again.ID(), again.SaveOwed(), published(again, "v2", "h1"), b.ID(), e) |
| 332 | } |
| 333 | if rec, _ := grantAt(again, "v2", "h1", e); !rec { |
| 334 | t.Fatal("a grant at the generation published before the reopen cleared nothing") |
| 335 | } |
| 336 | }) |
| 337 | for name, file := range map[string]map[string]any{ |
| 338 | "schema 1": {"schema": 1}, |
| 339 | "schema 2": {"schema": 2}, |
| 340 | "schema 2 with durable_epochs": {"schema": 2, "durable_epochs": true, "gens": map[string]uint64{"bab/blog": 10}}, |
| 341 | } { |
| 342 | t.Run(name, func(t *testing.T) { |
| 343 | dd := t.TempDir() |
| 344 | file["book"], file["seq"] = "old", 10 |
| 345 | file["records"] = []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "h1", "held_down": []string{"h1"}}} |
| 346 | raw, err := json.Marshal(file) |
| 347 | if err != nil { |
| 348 | t.Fatal(err) |
| 349 | } |
| 350 | if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { |
| 351 | t.Fatal(err) |
| 352 | } |
| 353 | b, err := OpenFailureBook(dd) |
| 354 | if err != nil { |
| 355 | t.Fatalf("the file did not open: %v", err) |
| 356 | } |
| 357 | if !b.HeldDown("bab", "blog", "v2", "h1") { |
| 358 | t.Fatal("the file's record was lost") |
| 359 | } |
| 360 | if id := b.ID(); id == "old" || id == "" { |
| 361 | t.Fatalf("identity %q, want a new one", id) |
| 362 | } |
| 363 | if !b.SaveOwed() || published(b, "v2", "h1") != 0 { |
| 364 | t.Fatalf("save owed %v, published %d: want the rotation owed and nothing published", b.SaveOwed(), published(b, "v2", "h1")) |
| 365 | } |
| 366 | if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h1", "old", 10); rec || mark { |
| 367 | t.Fatalf("a grant naming the old identity cleared record %v, marker %v", rec, mark) |
| 368 | } |
| 369 | if err := b.Flush(); err != nil { |
| 370 | t.Fatal(err) |
| 371 | } |
| 372 | if f := fileOf(t, dd); f.Schema != 3 || !f.Durable || f.Book != b.ID() { |
| 373 | t.Fatalf("written as schema %d, durable_epochs %v, book %q: want 3, true, %q", f.Schema, f.Durable, f.Book, b.ID()) |
| 374 | } |
| 375 | e := published(b, "v2", "h1") |
| 376 | if e == 0 { |
| 377 | t.Fatal("published nothing after the rotation was saved") |
| 378 | } |
| 379 | if rec, _ := grantAt(b, "v2", "h1", e); !rec { |
| 380 | t.Fatal("a grant naming the new identity at the published generation cleared nothing") |
| 381 | } |
| 382 | }) |
| 383 | } |
| 384 | } |
| 385 | |
| 386 | // Rule 3: a reopen starts from the disk's sequence, so every generation |
| 387 | // handed out after it is larger than every one published before it. |
| 388 | func TestAReopenNeverReissuesAPublishedGeneration(t *testing.T) { |
| 389 | b, dd := diskBook(t) |
| 390 | var before uint64 |
| 391 | read := func(b *FailureBook) uint64 { |
| 392 | t.Helper() |
| 393 | e := max(published(b, "v2", "h1"), published(b, "v3", "h1"), published(b, "v4", "h1"), published(b, "v5", "h2")) |
| 394 | if e == 0 { |
| 395 | t.Fatal("nothing published") |
| 396 | } |
| 397 | return e |
| 398 | } |
| 399 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", PromotedAt: promotedAt}) |
| 400 | before = max(before, read(b)) |
| 401 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", HeldDown: []string{"h1"}}) |
| 402 | before = max(before, read(b)) |
| 403 | mustMark(t, b, "h1", HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a1", At: t0}) |
| 404 | before = max(before, read(b)) |
| 405 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v4", Host: "sib"}) |
| 406 | before = max(before, read(b)) |
| 407 | |
| 408 | again, err := OpenFailureBook(dd) |
| 409 | if err != nil { |
| 410 | t.Fatal(err) |
| 411 | } |
| 412 | if err := again.Flush(); err != nil { |
| 413 | t.Fatal(err) |
| 414 | } |
| 415 | for i, change := range []func(){ |
| 416 | func() { mustRecord(t, again, FailureRecord{Account: "bab", App: "blog", Build: "v5", Host: "sib"}) }, |
| 417 | func() { |
| 418 | mustMark(t, again, "h2", HoldRan{Account: "bab", App: "blog", Build: "v5", Exec: "a2", At: t0}) |
| 419 | }, |
| 420 | func() { |
| 421 | if ok, err := again.ClearOnPointer("bab", "blog", "v4", t0); !ok || err != nil { |
| 422 | t.Fatalf("ClearOnPointer = %v, %v", ok, err) |
| 423 | } |
| 424 | }, |
| 425 | } { |
| 426 | change() |
| 427 | for _, c := range [][2]string{{"v2", "h1"}, {"v3", "h1"}, {"v5", "h2"}} { |
| 428 | if e := published(again, c[0], c[1]); e <= before { |
| 429 | t.Fatalf("after change %d of the reopened book, %s on %s published %d, not past %d", i+1, c[0], c[1], e, before) |
| 430 | } |
| 431 | } |
| 432 | } |
| 433 | } |
| 434 | |
| 435 | // Rule 5 and finding 3: two books opened with equal numeric state publish |
| 436 | // the same generation; a grant for one clears nothing in the other. |
| 437 | func TestAGrantForAnotherBookClearsNothing(t *testing.T) { |
| 438 | one, _ := diskBook(t) |
| 439 | other, _ := diskBook(t) |
| 440 | for _, b := range []*FailureBook{one, other} { |
| 441 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), |
| 442 | HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) |
| 443 | } |
| 444 | e := published(one, "v2", "h") |
| 445 | if e == 0 || e != published(other, "v2", "h") { |
| 446 | t.Fatalf("published %d and %d: the case needs equal generations", e, published(other, "v2", "h")) |
| 447 | } |
| 448 | if one.ID() == other.ID() { |
| 449 | t.Fatalf("two books share the identity %q", one.ID()) |
| 450 | } |
| 451 | if rec, mark, _, _ := other.RetryHostAt("bab", "blog", "v2", "h", one.ID(), e); rec || mark { |
| 452 | t.Fatalf("a grant for one book cleared record %v, marker %v in another", rec, mark) |
| 453 | } |
| 454 | if !other.HeldDown("bab", "blog", "v2", "h") { |
| 455 | t.Fatal("the other book's hold-down was lifted") |
| 456 | } |
| 457 | if rec, mark := grantAt(one, "v2", "h", e); !rec || !mark { |
| 458 | t.Fatalf("the grant cleared record %v, marker %v in its own book; want both", rec, mark) |
| 459 | } |
| 460 | } |
| 461 | |
| 462 | // Rule 5: a book's identity comes from crypto/rand only. A book that needs |
| 463 | // one and cannot get it does not open; one already on disk under the rule |
| 464 | // needs none. |
| 465 | func TestABookThatCannotGetAnIdentityDoesNotOpen(t *testing.T) { |
| 466 | durable, ddDurable := diskBook(t) |
| 467 | mustRecord(t, durable, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h"}) |
| 468 | ddOld := t.TempDir() |
| 469 | if err := os.WriteFile(filepath.Join(ddOld, FailureFile), []byte(`{"schema": 2, "book": "old", "records": []}`), 0o644); err != nil { |
| 470 | t.Fatal(err) |
| 471 | } |
| 472 | broken := errors.New("no entropy") |
| 473 | old := randRead |
| 474 | randRead = func([]byte) (int, error) { return 0, broken } |
| 475 | t.Cleanup(func() { randRead = old }) |
| 476 | for name, dd := range map[string]string{"a new book": t.TempDir(), "a book from before the rule": ddOld} { |
| 477 | if b, err := OpenFailureBook(dd); !errors.Is(err, broken) || b != nil { |
| 478 | t.Fatalf("%s opened (%v, %v) without an identity", name, b, err) |
| 479 | } |
| 480 | } |
| 481 | b, err := OpenFailureBook(ddDurable) |
| 482 | if err != nil { |
| 483 | t.Fatalf("a book with its identity on disk did not open: %v", err) |
| 484 | } |
| 485 | if b.ID() != durable.ID() { |
| 486 | t.Fatalf("identity %q, want %q", b.ID(), durable.ID()) |
| 487 | } |
| 488 | } |
| 489 | |
| 490 | // Fix wave 1, item 5 (Astra final review 1, finding 5): the sequence never |
| 491 | // wraps under one identity. The largest value it hands out is |
| 492 | // MaxUint64-2. A change that needs another refuses to save, so nothing is |
| 493 | // published and no grant clears; a snapshot at the boundary reads "not |
| 494 | // known" (0); and the file keeps the last value that landed, so a reopen |
| 495 | // refuses the same way. |
| 496 | func TestAnExhaustedSequenceFailsClosed(t *testing.T) { |
| 497 | b, dd := diskBook(t) |
| 498 | b.mu.Lock() |
| 499 | b.seq = math.MaxUint64 - 3 |
| 500 | b.mu.Unlock() |
| 501 | mustRecord(t, b, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h1"}) |
| 502 | e := published(b, "v2", "h1") |
| 503 | if e != math.MaxUint64-2 { |
| 504 | t.Fatalf("published %d, want the last usable value %d", e, uint64(math.MaxUint64-2)) |
| 505 | } |
| 506 | if s := b.Snapshot(); s != 0 { |
| 507 | t.Fatalf("Snapshot = %d at the boundary, want 0 (not known)", s) |
| 508 | } |
| 509 | if err := b.MarkHoldRan("h1", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "e1"}); err == nil { |
| 510 | t.Fatal("a change that needs a value past the boundary was saved") |
| 511 | } |
| 512 | if p := published(b, "v2", "h1"); p != 0 { |
| 513 | t.Fatalf("published %d after the sequence ran out, want 0", p) |
| 514 | } |
| 515 | if rec, mark := grantAt(b, "v2", "h1", e); rec || mark { |
| 516 | t.Fatalf("a grant cleared record %v, marker %v on an exhausted book", rec, mark) |
| 517 | } |
| 518 | again, err := OpenFailureBook(dd) |
| 519 | if err != nil { |
| 520 | t.Fatal(err) |
| 521 | } |
| 522 | again.mu.Lock() |
| 523 | seq, gen := again.seq, again.gens[genKey("bab", "blog")] |
| 524 | again.mu.Unlock() |
| 525 | if seq != math.MaxUint64-2 || gen != e { |
| 526 | t.Fatalf("the file holds seq %d and generation %d, want the last landed %d", seq, gen, e) |
| 527 | } |
| 528 | if _, err := again.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "h2"}); err == nil { |
| 529 | t.Fatal("the reopened book saved a change past the boundary") |
| 530 | } |
| 531 | if p := published(again, "v2", "h1"); p != 0 { |
| 532 | t.Fatalf("the reopened book published %d after it ran out, want 0", p) |
| 533 | } |
| 534 | } |
| 535 | |
amadan