| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/activator" |
| 11 | "github.com/carlosframework/platform/internal/status" |
| 12 | ) |
| 13 | |
| 14 | // retry is the console's signed host retry as the box applies it: a grant |
| 15 | // naming build, carrying the generation the box's status publishes for it. |
| 16 | func (h *harness) retry(host, build string) bool { |
| 17 | return h.m.RetryHost(host, build, h.book.ID(), h.book.PublishedEpoch("bab", "blog", build, host)) |
| 18 | } |
| 19 | |
| 20 | // Plan review finding 13, under the crash-epoch hard limit (readiness round |
| 21 | // 2, Task 1): a restriction's epoch is its app's generation, a persisted |
| 22 | // book sequence that moves whenever any restriction of the app changes. |
| 23 | func TestRestrictionEpochFollowsTheRecordAndTheMarker(t *testing.T) { |
| 24 | dd := t.TempDir() |
| 25 | b, err := OpenFailureBook(dd) |
| 26 | if err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | if e := b.RestrictionEpoch("bab", "blog", "v2", "a"); e != 0 { |
| 30 | t.Fatalf("epoch with nothing recorded = %d, want 0", e) |
| 31 | } |
| 32 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}) |
| 33 | e1 := b.RestrictionEpoch("bab", "blog", "v2", "a") |
| 34 | if e1 == 0 || b.RestrictionEpoch("bab", "blog", "v2", "b") != e1 { |
| 35 | t.Fatalf("after the first failure: a %d, sibling b %d; want one non-zero epoch for both", e1, b.RestrictionEpoch("bab", "blog", "v2", "b")) |
| 36 | } |
| 37 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}); res != Unchanged || |
| 38 | b.RestrictionEpoch("bab", "blog", "v2", "a") != e1 { |
| 39 | t.Fatalf("an unchanged failure moved the epoch to %d", b.RestrictionEpoch("bab", "blog", "v2", "a")) |
| 40 | } |
| 41 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", HeldDown: []string{"a"}}) |
| 42 | e2 := b.RestrictionEpoch("bab", "blog", "v2", "a") |
| 43 | if e2 <= e1 || b.RestrictionEpoch("bab", "blog", "v2", "b") <= e1 { |
| 44 | t.Fatalf("a merged hold-down: a %d, b %d; both must move past %d", e2, b.RestrictionEpoch("bab", "blog", "v2", "b"), e1) |
| 45 | } |
| 46 | b.MarkHoldRan("c", HoldRan{Account: "bab", App: "blog", Build: "v3"}) |
| 47 | e3 := b.RestrictionEpoch("bab", "blog", "v3", "c") |
| 48 | if e3 <= e2 { |
| 49 | t.Fatalf("a marker-only restriction's epoch = %d, want the app's generation moved past %d", e3, e2) |
| 50 | } |
| 51 | if e := b.RestrictionEpoch("bab", "blog", "v2", "b"); e != e3 { |
| 52 | t.Fatalf("v2's epoch for b = %d after c's marker: one generation per app, want %d", e, e3) |
| 53 | } |
| 54 | if e := b.RestrictionEpoch("bab", "blog", "v3", "b"); e != 0 { |
| 55 | t.Fatalf("b's epoch for v3 = %d: c's marker of v3 restricts only c, want 0", e) |
| 56 | } |
| 57 | if e := b.RestrictionEpoch("bab", "shop", "v3", "c"); e != 0 { |
| 58 | t.Fatalf("another app's epoch = %d, want 0", e) |
| 59 | } |
| 60 | |
| 61 | b2, err := OpenFailureBook(dd) |
| 62 | if err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | for _, c := range [][2]string{{"v2", "a"}, {"v2", "b"}, {"v3", "c"}} { |
| 66 | if got, want := b2.RestrictionEpoch("bab", "blog", c[0], c[1]), b.RestrictionEpoch("bab", "blog", c[0], c[1]); got != want { |
| 67 | t.Fatalf("after a reopen %s on %s has epoch %d, want %d", c[0], c[1], got, want) |
| 68 | } |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | // A book an earlier agent wrote has no generations: the open gives its app |
| 73 | // one, and its marker a sequence, newer than anything in the file, so its |
| 74 | // holds can still be retried (a zero epoch is never accepted). One app has |
| 75 | // one generation (the crash-epoch hard limit, rule 1), so the record's and |
| 76 | // the marker's restrictions share it. |
| 77 | func TestOpenGivesAnOlderFilesRestrictionsAnEpoch(t *testing.T) { |
| 78 | dd := t.TempDir() |
| 79 | raw, _ := json.Marshal(map[string]any{ |
| 80 | "schema": 1, |
| 81 | "records": []map[string]any{{"account": "bab", "app": "blog", "build": "v2", "host": "a"}}, |
| 82 | "hold_ran": map[string]any{"c": map[string]any{"account": "bab", "app": "blog", "build": "v3"}}, |
| 83 | "seq": 5, |
| 84 | }) |
| 85 | if err := os.WriteFile(filepath.Join(dd, FailureFile), raw, 0o644); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | b, err := OpenFailureBook(dd) |
| 89 | if err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | rec := b.RestrictionEpoch("bab", "blog", "v2", "a") |
| 93 | mark := b.RestrictionEpoch("bab", "blog", "v3", "c") |
| 94 | if rec <= 5 || rec != mark { |
| 95 | t.Fatalf("epochs record %d, marker %d: want the app's one generation, newer than the file's 5", rec, mark) |
| 96 | } |
| 97 | if m, _ := b.HoldRanOn("c"); m.Seq <= 5 { |
| 98 | t.Fatalf("the marker's sequence = %d, want one newer than the file's 5", m.Seq) |
| 99 | } |
| 100 | if b.Snapshot() <= max(rec, mark) { |
| 101 | t.Fatal("a snapshot after the open must be newer than the epochs it handed out") |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | // Plan review finding 5: a grant names the build whose restriction it lifts. |
| 106 | // One naming another build than the record's clears nothing, the marker |
| 107 | // included. |
| 108 | func TestARetryGrantForAnotherBuildClearsNothing(t *testing.T) { |
| 109 | h := newHarness(t, "a", "b") |
| 110 | h.stage("v2") |
| 111 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "c", PromotedAt: promotedAt, At: t0}) |
| 112 | h.book.MarkHoldRan("b", markV2) |
| 113 | h.install("v2", "a", "b") |
| 114 | h.pass() |
| 115 | epoch := h.book.RestrictionEpoch("bab", "blog", "v2", "b") |
| 116 | if h.m.RetryHost("b", "v9", h.book.ID(), epoch) { |
| 117 | t.Fatal("a grant for v9 cleared something") |
| 118 | } |
| 119 | if !h.book.Blocks("bab", "blog", "v2", "b", false) { |
| 120 | t.Fatal("a grant for v9 cleared v2's record for b") |
| 121 | } |
| 122 | if _, ok := h.book.HoldRanOn("b"); !ok { |
| 123 | t.Fatal("a grant for v9 cleared b's v2 marker") |
| 124 | } |
| 125 | if h.m.RetryHost("b", "v2", h.book.ID(), 0) { |
| 126 | t.Fatal("a zero epoch cleared something") |
| 127 | } |
| 128 | if !h.retry("b", "v2") { |
| 129 | t.Fatal("the grant for v2 at its epoch cleared nothing") |
| 130 | } |
| 131 | if _, ok := h.book.HoldRanOn("b"); ok || h.book.Blocks("bab", "blog", "v2", "b", false) { |
| 132 | t.Fatal("the grant for v2 left b restricted") |
| 133 | } |
| 134 | waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| 135 | } |
| 136 | |
| 137 | // The marker names B while the slot wants A: a grant for B lifts the marker |
| 138 | // and nothing of A's. |
| 139 | func TestARetryGrantOfTheMarkersBuildClearsOnlyTheMarker(t *testing.T) { |
| 140 | h := newHarness(t, "a") |
| 141 | h.stage("v2") |
| 142 | h.install("v2", "a") |
| 143 | h.sleepHost("a") |
| 144 | h.pass() |
| 145 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", At: t0}) |
| 146 | h.book.MarkHoldRan("a", HoldRan{Account: "bab", App: "blog", Build: "v3"}) |
| 147 | if !h.retry("a", "v3") { |
| 148 | t.Fatal("the grant for the marker's build cleared nothing") |
| 149 | } |
| 150 | if _, ok := h.book.HoldRanOn("a"); ok { |
| 151 | t.Fatal("the marker of v3 survived its grant") |
| 152 | } |
| 153 | if !h.book.Blocks("bab", "blog", "v2", "a", false) { |
| 154 | t.Fatal("a grant for v3 cleared v2's record for a") |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | // Plan review finding 13: a grant is for the restriction the member saw. |
| 159 | // One whose epoch predates a second failure of the same build on the same |
| 160 | // host clears nothing, and a fresh grant does. |
| 161 | func TestARetryGrantPredatingASecondFailureClearsNothing(t *testing.T) { |
| 162 | h := newHarness(t, "a", "b") |
| 163 | h.stage("v2") |
| 164 | h.failing.Store(true) |
| 165 | h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| 166 | if a.Build == "v2" && h.failing.Load() { |
| 167 | return activator.SwapFailed |
| 168 | } |
| 169 | return activator.SwapDone |
| 170 | } |
| 171 | h.install("v2", "a", "b") |
| 172 | h.pass() |
| 173 | waitFor(t, "a failure record", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| 174 | first := h.book.RestrictionEpoch("bab", "blog", "v2", "b") |
| 175 | if !h.m.RetryHost("b", "v2", h.book.ID(), first) { |
| 176 | t.Fatal("the first grant cleared nothing") |
| 177 | } |
| 178 | // b's canary fails: the same build fails again on the same host. |
| 179 | waitFor(t, "b's canary to fail", func() bool { return !h.m.attemptRunning("b") && h.book.Blocks("bab", "blog", "v2", "b", false) }) |
| 180 | if h.m.RetryHost("b", "v2", h.book.ID(), first) { |
| 181 | t.Fatal("a grant from before b's second failure cleared it") |
| 182 | } |
| 183 | if !h.book.Blocks("bab", "blog", "v2", "b", false) { |
| 184 | t.Fatal("b is not held after the stale grant") |
| 185 | } |
| 186 | h.failing.Store(false) |
| 187 | if !h.retry("b", "v2") { |
| 188 | t.Fatal("a fresh grant cleared nothing") |
| 189 | } |
| 190 | waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| 191 | } |
| 192 | |
| 193 | // The same at the book: a merge that holds the host down after the grant |
| 194 | // was signed moves the epoch, so the grant clears nothing. |
| 195 | func TestARetryGrantPredatingAHoldDownClearsNothing(t *testing.T) { |
| 196 | b := &FailureBook{} |
| 197 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h"}) |
| 198 | signed := b.RestrictionEpoch("bab", "blog", "v2", "h") |
| 199 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", HeldDown: []string{"h"}}) |
| 200 | if rec, mark, _, _ := b.RetryHostAt("bab", "blog", "v2", "h", b.ID(), signed); rec || mark { |
| 201 | t.Fatalf("a grant predating the hold-down cleared record %v, marker %v", rec, mark) |
| 202 | } |
| 203 | if !b.HeldDown("bab", "blog", "v2", "h") { |
| 204 | t.Fatal("h is no longer held down") |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | // A used grant replayed after the agent reopens its book clears nothing |
| 209 | // new: the clear moved the host's epoch, and the epoch is on disk. |
| 210 | func TestARetryGrantReplayedAfterAReopenClearsNothing(t *testing.T) { |
| 211 | dd := t.TempDir() |
| 212 | b, err := OpenFailureBook(dd) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), |
| 217 | HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a1", HoldRanAt: t0}) |
| 218 | grant := b.RestrictionEpoch("bab", "blog", "v2", "h") |
| 219 | if rec, mark, _, err := b.RetryHostAt("bab", "blog", "v2", "h", b.ID(), grant); !rec || !mark || err != nil { |
| 220 | t.Fatalf("the grant cleared record %v, marker %v (%v); want both", rec, mark, err) |
| 221 | } |
| 222 | b2, err := OpenFailureBook(dd) |
| 223 | if err != nil { |
| 224 | t.Fatal(err) |
| 225 | } |
| 226 | // The retried canary fails again, after the reopen. |
| 227 | b2.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Mode: string(activator.ModeHold), |
| 228 | HeldDown: []string{"h"}, MarkHoldRan: true, HoldRanExec: "a2", HoldRanAt: t0.Add(time.Minute), Seq: b2.Snapshot()}) |
| 229 | if rec, mark, _, _ := b2.RetryHostAt("bab", "blog", "v2", "h", b.ID(), grant); rec || mark { |
| 230 | t.Fatalf("the replayed grant cleared record %v, marker %v", rec, mark) |
| 231 | } |
| 232 | if !b2.HeldDown("bab", "blog", "v2", "h") { |
| 233 | t.Fatal("the replay lifted the second hold-down") |
| 234 | } |
| 235 | if _, ok := b2.HoldRanOn("h"); !ok { |
| 236 | t.Fatal("the replay lifted the second marker") |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | // Status publishes the epoch the console signs, beside HeldBy, on both the |
| 241 | // record arm and the marker arm. |
| 242 | func TestStatusPublishesTheHeldEpoch(t *testing.T) { |
| 243 | h := newHarness(t, "a", "b") |
| 244 | h.stage("v2", "v3") |
| 245 | h.install("v2", "a") |
| 246 | h.install("v3", "b") |
| 247 | h.sleepHost("a") |
| 248 | h.sleepHost("b") |
| 249 | h.pass() |
| 250 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "sib", Reason: "exited early", At: t0}) |
| 251 | h.book.MarkHoldRan("b", HoldRan{Account: "bab", App: "blog", Build: "v4"}) |
| 252 | r := h.m.Status() |
| 253 | if got := r.Hosts["a"]; got.HeldBy != "v2" || got.HeldEpoch == 0 || got.HeldEpoch != h.book.RestrictionEpoch("bab", "blog", "v2", "a") { |
| 254 | t.Fatalf("a = %+v: want held by v2 at its epoch %d", got, h.book.RestrictionEpoch("bab", "blog", "v2", "a")) |
| 255 | } |
| 256 | if got := r.Hosts["b"]; got.HeldBy != "v4" || got.HeldEpoch == 0 || got.HeldEpoch != h.book.RestrictionEpoch("bab", "blog", "v4", "b") { |
| 257 | t.Fatalf("b = %+v: want held by the marker's v4 at its epoch", got) |
| 258 | } |
| 259 | if got := r.Hosts["b"]; got.HeldBook == "" || got.HeldBook != h.book.ID() || r.Hosts["a"].HeldBook != h.book.ID() { |
| 260 | t.Fatalf("held books %q and %q: want the book's identity %q on both arms", r.Hosts["a"].HeldBook, got.HeldBook, h.book.ID()) |
| 261 | } |
| 262 | // Beside HeldBook, the retry protocol this box takes (readiness round 2, |
| 263 | // Task 3): its grants ride the host-retry log, which the console checks |
| 264 | // before it writes one. |
| 265 | if r.Hosts["a"].RetryV != status.HostRetryV || r.Hosts["b"].RetryV != status.HostRetryV || status.HostRetryV != 1 { |
| 266 | t.Fatalf("retry_v %d and %d: want %d on both arms", r.Hosts["a"].RetryV, r.Hosts["b"].RetryV, status.HostRetryV) |
| 267 | } |
| 268 | if !h.m.RetryHost("b", r.Hosts["b"].HeldBy, r.Hosts["b"].HeldBook, r.Hosts["b"].HeldEpoch) { |
| 269 | t.Fatal("a grant signed from the status report cleared nothing") |
| 270 | } |
| 271 | } |
| 272 | |
amadan