| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "sync/atomic" |
| 5 | "testing" |
| 6 | "time" |
| 7 | |
| 8 | "github.com/carlosframework/platform/internal/activator" |
| 9 | "github.com/carlosframework/platform/internal/release" |
| 10 | ) |
| 11 | |
| 12 | // Final review F8: `carlos instances retry <host>` promises that host's next |
| 13 | // adoption runs as a canary through its cohort's halt. Admission spends the |
| 14 | // allowance (forceBuilds) when the attempt starts. An attempt that then ends |
| 15 | // before its build ran (abandoned with SwapResult.Executed false, or refused |
| 16 | // as a staging error) spent nothing of it: the allowance comes back, so the |
| 17 | // canary still runs once the host's deferral ends, with no second retry |
| 18 | // (which the book would refuse anyway: the host is already in Retried). An |
| 19 | // attempt whose build ran spends it, abandoned or not. |
| 20 | func TestARetryAllowanceComesBackOnlyWhenNothingRan(t *testing.T) { |
| 21 | for _, tc := range []struct { |
| 22 | name string |
| 23 | staging bool // b's first retry attempt is refused as a staging error |
| 24 | executed bool // b's first retry attempt is abandoned after its build ran |
| 25 | deferral time.Duration |
| 26 | }{ |
| 27 | {"abandoned before its build ran", false, false, abandonDeferral}, |
| 28 | {"refused as a staging error", true, false, stagingDeferral}, |
| 29 | {"abandoned after its build ran", false, true, abandonDeferral}, |
| 30 | } { |
| 31 | t.Run(tc.name, func(t *testing.T) { |
| 32 | h := newHarness(t, "a", "b") |
| 33 | h.stage("v2") |
| 34 | var bTries atomic.Int32 |
| 35 | h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| 36 | switch { |
| 37 | case a.Build != "v2": |
| 38 | return activator.SwapDone |
| 39 | case a.Host == "a": |
| 40 | return activator.SwapFailed // the canary that halts the cohort |
| 41 | case bTries.Add(1) == 1 && !tc.staging: |
| 42 | return activator.SwapAbandoned |
| 43 | } |
| 44 | return activator.SwapDone |
| 45 | } |
| 46 | h.act.executed = func(_ activator.Attempt, out activator.SwapOutcome) bool { |
| 47 | return out != activator.SwapAbandoned || tc.executed |
| 48 | } |
| 49 | h.install("v2", "a", "b") |
| 50 | h.pass() |
| 51 | waitFor(t, "the halt", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| 52 | if tc.staging { |
| 53 | h.mu.Lock() |
| 54 | h.mismatch["v2"] = true |
| 55 | h.mu.Unlock() |
| 56 | } |
| 57 | if !h.retry("b", "v2") { |
| 58 | t.Fatal("RetryHost did not clear b") |
| 59 | } |
| 60 | if tc.staging { |
| 61 | waitFor(t, "the staging refusal", func() bool { return h.touched.Load() > 0 }) |
| 62 | h.mu.Lock() |
| 63 | h.mismatch["v2"] = false |
| 64 | h.mu.Unlock() |
| 65 | } else { |
| 66 | waitFor(t, "b's first retry attempt", func() bool { return bTries.Load() >= 1 }) |
| 67 | } |
| 68 | // Its slot is free and its deferral set (done) before the next pass. |
| 69 | waitFor(t, "b's slot to free", func() bool { return !h.m.attemptRunning("b") }) |
| 70 | before, _, _ := h.act.snapshot() |
| 71 | h.advance(tc.deferral + time.Second) |
| 72 | h.pass() |
| 73 | if tc.executed { |
| 74 | time.Sleep(100 * time.Millisecond) |
| 75 | if calls, _, _ := h.act.snapshot(); len(calls) != len(before) { |
| 76 | t.Fatalf("calls %+v: an attempt whose build ran spent the retry, yet b ran again through the halt", calls[len(before):]) |
| 77 | } |
| 78 | return |
| 79 | } |
| 80 | waitFor(t, "b's retry canary", func() bool { return h.servingAll("v2", "b") }) |
| 81 | calls, _, _ := h.act.snapshot() |
| 82 | if c := calls[len(calls)-1].a; c.Host != "b" || !c.Canary || c.Cause != activator.CauseRetry { |
| 83 | t.Fatalf("attempt after the re-armed retry = %+v, want b's retry canary", c) |
| 84 | } |
| 85 | if h.servingAll("v2", "a") { |
| 86 | t.Fatal("the re-armed retry lifted the halt for another host") |
| 87 | } |
| 88 | }) |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | // Final review wave 2, N4: only proof that nothing ran gives the allowance |
| 93 | // back. A worker attempt abandoned when its caller left can still have had |
| 94 | // its respawn exec the build (the activator joins the respawn, and past its |
| 95 | // bound reports Executed true: unknown). Executed true spends the |
| 96 | // allowance, whether the build surely ran or only may have; Executed false, |
| 97 | // the activator's proof that nothing ran, gives it back. |
| 98 | func TestARetryAllowanceIsSpentWhenAWorkerAttemptMayHaveRun(t *testing.T) { |
| 99 | for _, tc := range []struct { |
| 100 | name string |
| 101 | executed bool |
| 102 | }{ |
| 103 | {"execution unknown", true}, |
| 104 | {"proven unexecuted", false}, |
| 105 | } { |
| 106 | t.Run(tc.name, func(t *testing.T) { |
| 107 | h := workerHarness(t, "a", "b") |
| 108 | h.stage("v2") |
| 109 | var bTries atomic.Int32 |
| 110 | h.act.mu.Lock() |
| 111 | h.act.worker = func(a activator.Attempt, _ *release.StagedIdentity) activator.SwapResult { |
| 112 | switch { |
| 113 | case a.Build != "v2": |
| 114 | return activator.SwapResult{Outcome: activator.SwapDone} |
| 115 | case a.Host == "a": |
| 116 | return activator.SwapResult{Outcome: activator.SwapFailed, Reason: "exited within 10s of starting", Executed: true} |
| 117 | case bTries.Add(1) == 1: |
| 118 | return activator.SwapResult{Outcome: activator.SwapAbandoned, Reason: "swap: an intentional stop cancelled the attempt", Executed: tc.executed} |
| 119 | } |
| 120 | return activator.SwapResult{Outcome: activator.SwapDone, Executed: true} |
| 121 | } |
| 122 | h.act.mu.Unlock() |
| 123 | h.install("v2", "a", "b") |
| 124 | h.pass() |
| 125 | waitFor(t, "the halt", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| 126 | waitFor(t, "a's slot to free", func() bool { return !h.m.attemptRunning("a") }) |
| 127 | if !h.retry("b", "v2") { |
| 128 | t.Fatal("RetryHost did not clear b") |
| 129 | } |
| 130 | waitFor(t, "b's first retry attempt", func() bool { return bTries.Load() >= 1 }) |
| 131 | waitFor(t, "b's slot to free", func() bool { return !h.m.attemptRunning("b") }) |
| 132 | h.advance(abandonDeferral + time.Second) |
| 133 | h.pass() |
| 134 | if tc.executed { |
| 135 | time.Sleep(100 * time.Millisecond) |
| 136 | if n := bTries.Load(); n != 1 { |
| 137 | t.Fatalf("b ran %d attempts: an attempt that may have run its build spent the retry, yet b ran again through the halt", n) |
| 138 | } |
| 139 | return |
| 140 | } |
| 141 | waitFor(t, "b's re-armed retry canary", func() bool { return bTries.Load() == 2 }) |
| 142 | }) |
| 143 | } |
| 144 | } |
| 145 | |
amadan