| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "maps" |
| 5 | "slices" |
| 6 | "testing" |
| 7 | "time" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/activator" |
| 10 | ) |
| 11 | |
| 12 | // retryHarness is a harness whose s fails v3 and whose t fails v4: each |
| 13 | // failure writes a record that holds h too, and halts that build's cohort. |
| 14 | // h serves and wants v1 throughout. |
| 15 | func retryHarness(t *testing.T) (*harness, func(build string) bool) { |
| 16 | t.Helper() |
| 17 | k := appKey{"bab", "blog"} |
| 18 | h := newHarness(t, "h", "s", "t") |
| 19 | h.stage("v2", "v3", "v4") |
| 20 | h.act.mu.Lock() |
| 21 | h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| 22 | if (a.Host == "s" && a.Build == "v3") || (a.Host == "t" && a.Build == "v4") { |
| 23 | return activator.SwapFailed |
| 24 | } |
| 25 | return activator.SwapDone |
| 26 | } |
| 27 | h.act.mu.Unlock() |
| 28 | h.pass() |
| 29 | s := h.m.scheduler(k) |
| 30 | halted := func(build string) bool { |
| 31 | s.mu.Lock() |
| 32 | defer s.mu.Unlock() |
| 33 | return s.q.cohortFor(cohortKey(build, classAdopt)).halted |
| 34 | } |
| 35 | h.install("v3", "s") |
| 36 | h.pass() |
| 37 | waitFor(t, "v3's cohort to halt", func() bool { return halted("v3") }) |
| 38 | h.install("v4", "t") |
| 39 | h.pass() |
| 40 | waitFor(t, "v4's cohort to halt", func() bool { return halted("v4") }) |
| 41 | return h, halted |
| 42 | } |
| 43 | |
| 44 | // running is h's running attempt, nil when none. |
| 45 | func (h *harness) running(host string) *dispatch { |
| 46 | s := h.m.scheduler(appKey{"bab", "blog"}) |
| 47 | s.mu.Lock() |
| 48 | defer s.mu.Unlock() |
| 49 | if d := s.q.hosts[host].running; d != nil { |
| 50 | c := *d |
| 51 | return &c |
| 52 | } |
| 53 | return nil |
| 54 | } |
| 55 | |
| 56 | // Final review round 2, finding 5 (Astra): retry entitlements are kept per |
| 57 | // host and build. B's (v3's) cohort is halted; h is retried for v3 while it |
| 58 | // wants v1, then for another restricted build C (v4) before it adopts v3. |
| 59 | // The second retry must not overwrite the first: when h comes to want v3, |
| 60 | // the book already lists h as retried for v3 (so a second retry of v3 |
| 61 | // clears nothing), and only the entitlement the first retry bound can take |
| 62 | // h through v3's halt. |
| 63 | func TestARetryForASecondBuildKeepsTheFirstBuildsCanary(t *testing.T) { |
| 64 | h, halted := retryHarness(t) |
| 65 | if !h.retry("h", "v3") { |
| 66 | t.Fatal("the retry for v3 cleared nothing") |
| 67 | } |
| 68 | if !h.retry("h", "v4") { |
| 69 | t.Fatal("the retry for v4 cleared nothing") |
| 70 | } |
| 71 | if h.retry("h", "v3") { |
| 72 | t.Fatal("a second retry of v3 cleared something: the case needs the book to refuse it") |
| 73 | } |
| 74 | if !halted("v3") || !halted("v4") { |
| 75 | t.Fatal("a cohort is no longer halted: the case needs both halts") |
| 76 | } |
| 77 | |
| 78 | gate := h.act.hold("h") |
| 79 | h.install("v3", "h") |
| 80 | h.pass() |
| 81 | waitFor(t, "h's adoption onto v3 through the halt", func() bool { |
| 82 | _, started, _ := h.act.snapshot() |
| 83 | return slices.Contains(started, "h:v3") |
| 84 | }) |
| 85 | if d := h.running("h"); d == nil || d.ID.Build != "v3" || !d.Canary || !d.Forced { |
| 86 | t.Fatalf("h's running attempt %+v: want the forced canary onto v3", d) |
| 87 | } |
| 88 | close(gate) |
| 89 | waitFor(t, "h's adoption onto v3", func() bool { return h.servingAll("v3", "h") && !h.m.attemptRunning("h") }) |
| 90 | |
| 91 | // v4's entitlement was neither overwritten nor spent by v3's adoption. |
| 92 | gate = h.act.hold("h") |
| 93 | defer close(gate) |
| 94 | h.install("v4", "h") |
| 95 | h.pass() |
| 96 | waitFor(t, "h's adoption onto v4 through the halt", func() bool { |
| 97 | _, started, _ := h.act.snapshot() |
| 98 | return slices.Contains(started, "h:v4") |
| 99 | }) |
| 100 | if d := h.running("h"); d == nil || d.ID.Build != "v4" || !d.Canary || !d.Forced { |
| 101 | t.Fatalf("h's running attempt %+v: want the forced canary onto v4", d) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | // An entitlement lasts only while the book lists the host as retried for its |
| 106 | // build. Once the record is gone (cleared by a signed pointer, or pruned when |
| 107 | // no route installs the build), the next pass drops it: the set cannot grow |
| 108 | // without bound, and a record the build may get later starts with no |
| 109 | // entitlement from a retry of the old one. |
| 110 | func TestARetryEntitlementIsDroppedWhenTheBookNoLongerListsTheHost(t *testing.T) { |
| 111 | for _, tc := range []struct { |
| 112 | name string |
| 113 | clear func(h *harness) |
| 114 | }{ |
| 115 | {"cleared by a pointer", func(h *harness) { |
| 116 | // s still wants v3 and may adopt it before h: it succeeds now, |
| 117 | // so no fresh record of v3 races h's adoption. |
| 118 | h.act.mu.Lock() |
| 119 | h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| 120 | if a.Host == "t" && a.Build == "v4" { |
| 121 | return activator.SwapFailed |
| 122 | } |
| 123 | return activator.SwapDone |
| 124 | } |
| 125 | h.act.mu.Unlock() |
| 126 | if ok, _ := h.book.ClearOnPointer("bab", "blog", "v3", t0.Add(time.Hour)); !ok { |
| 127 | h.t.Fatal("the pointer cleared nothing") |
| 128 | } |
| 129 | }}, |
| 130 | {"pruned", func(h *harness) { |
| 131 | h.install("v1", "s") // no route installs v3 any more |
| 132 | }}, |
| 133 | } { |
| 134 | t.Run(tc.name, func(t *testing.T) { |
| 135 | h, halted := retryHarness(t) |
| 136 | if !h.retry("h", "v3") { |
| 137 | t.Fatal("the retry for v3 cleared nothing") |
| 138 | } |
| 139 | if !h.retry("h", "v4") { |
| 140 | t.Fatal("the retry for v4 cleared nothing") |
| 141 | } |
| 142 | tc.clear(h) |
| 143 | h.pass() |
| 144 | if halted("v3") { |
| 145 | t.Fatal("v3's halt outlived its record") |
| 146 | } |
| 147 | s := h.m.scheduler(appKey{"bab", "blog"}) |
| 148 | s.mu.Lock() |
| 149 | got := slices.Sorted(maps.Keys(s.q.hosts["h"].forceBuilds)) |
| 150 | s.mu.Unlock() |
| 151 | if !slices.Equal(got, []string{"v4"}) { |
| 152 | t.Fatalf("h's entitlements %v, want only v4's: v3's record is gone", got) |
| 153 | } |
| 154 | |
| 155 | gate := h.act.hold("h") |
| 156 | defer close(gate) |
| 157 | h.install("v3", "h") |
| 158 | h.pass() |
| 159 | waitFor(t, "h's adoption onto v3", func() bool { |
| 160 | _, started, _ := h.act.snapshot() |
| 161 | return slices.Contains(started, "h:v3") |
| 162 | }) |
| 163 | // Whether it is the cohort's canary depends on whether s went |
| 164 | // first; either way it is not a retried host's forced canary. |
| 165 | if d := h.running("h"); d == nil || d.ID.Build != "v3" || d.Forced { |
| 166 | t.Fatalf("h's running attempt %+v: want an ordinary adoption onto v3, not a retried one", d) |
| 167 | } |
| 168 | }) |
| 169 | } |
| 170 | } |
| 171 | |
amadan