amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package rollout
2
3import (
4 "os"
5 "path/filepath"
6 "slices"
7 "testing"
8 "time"
9)
10
11var t0 = time.Unix(1_700_000_000, 0).UTC()
12
13// promotedAt is when the pointer the failures below happened under was signed.
14var promotedAt = t0.Add(-time.Hour)
15
16func rec(build, host string) FailureRecord {
17 return FailureRecord{Account: "bab", App: "blog", Build: build, Host: host, Cause: "release", Mode: "overlap",
18 Reason: "health check failed", PromotedAt: promotedAt, At: t0}
19}
20
21func TestFailureRecordBlocksOnlyInstancesThatHaveNotServedTheBuild(t *testing.T) {
22 b := &FailureBook{}
23 added, err := b.Record(rec("v2", "a.example.com"))
24 if err != nil || added != Recorded {
25 t.Fatalf("Record = %v, %v", added, err)
26 }
27 if !b.Blocks("bab", "blog", "v2", "b.example.com", false) {
28 t.Fatal("an instance that has not served v2 must be held off it")
29 }
30 if b.Blocks("bab", "blog", "v2", "c.example.com", true) {
31 t.Fatal("an instance already serving v2 must not be blocked (spec §8.4)")
32 }
33 if b.Blocks("bab", "blog", "v3", "b.example.com", false) {
34 t.Fatal("a newer installed build no longer matches the record")
35 }
36 if b.Blocks("bab", "other", "v2", "b.example.com", false) {
37 t.Fatal("the scope is (app, build, box), not the build name alone")
38 }
39 if added, _ := b.Record(rec("v2", "b.example.com")); added == Recorded {
40 t.Fatal("the first failure stands; a second is not a second record")
41 }
42}
43
44// Review focus 3: the record outlives an agent restart.
45func TestFailureBookSurvivesReopen(t *testing.T) {
46 dir := t.TempDir()
47 b, err := OpenFailureBook(dir)
48 if err != nil {
49 t.Fatal(err)
50 }
51 r := rec("v2", "a.example.com")
52 r.HeldDown = []string{"a.example.com"}
53 if _, err := b.Record(r); err != nil {
54 t.Fatal(err)
55 }
56 again, err := OpenFailureBook(dir)
57 if err != nil {
58 t.Fatal(err)
59 }
60 if !again.Blocks("bab", "blog", "v2", "b.example.com", false) {
61 t.Fatal("the reopened book lost the record")
62 }
63 if !again.HeldDown("bab", "blog", "v2", "a.example.com") {
64 t.Fatal("the reopened book lost the held-down host")
65 }
66}
67
68func TestACorruptFailureFileIsAnError(t *testing.T) {
69 dir := t.TempDir()
70 if err := os.WriteFile(filepath.Join(dir, FailureFile), []byte("{nope"), 0o644); err != nil {
71 t.Fatal(err)
72 }
73 if _, err := OpenFailureBook(dir); err == nil {
74 t.Fatal("a book that cannot be read must refuse, not start empty")
75 }
76}
77
78func TestRetryClearing(t *testing.T) {
79 b := &FailureBook{}
80 r := rec("v2", "a.example.com")
81 r.HeldDown = []string{"a.example.com"}
82 b.Record(r)
83
84 // carlos instances retry: one host only.
85 if ok, _ := b.RetryHost("bab", "blog", "v2", "a.example.com"); !ok {
86 t.Fatal("a host retry clears that host")
87 }
88 if b.Blocks("bab", "blog", "v2", "a.example.com", false) || b.HeldDown("bab", "blog", "v2", "a.example.com") {
89 t.Fatal("the retried host is still held")
90 }
91 if !b.Blocks("bab", "blog", "v2", "b.example.com", false) {
92 t.Fatal("a host retry cleared another host")
93 }
94 // The retried host fails the build again: it is held again.
95 b.Record(rec("v2", "a.example.com"))
96 if !b.Blocks("bab", "blog", "v2", "a.example.com", false) {
97 t.Fatal("a second failure must hold the retried host again")
98 }
99
100 // carlos promote --retry (spec rev 14 §8.4): a pointer naming the build
101 // with a NEWER PromotedAt clears the record; the pointer it failed under,
102 // seen again, does not.
103 if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt); ok {
104 t.Fatal("the pointer the build failed under must not clear the record")
105 }
106 if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(-time.Minute)); ok {
107 t.Fatal("an older pointer must not clear the record")
108 }
109 if ok, _ := b.ClearOnPointer("bab", "blog", "v2", promotedAt.Add(time.Second)); !ok {
110 t.Fatal("a re-signed pointer for the same build clears the record")
111 }
112 if _, ok := b.Lookup("bab", "blog", "v2"); ok {
113 t.Fatal("record still present after the retry")
114 }
115}
116
117func TestPruneKeepsRecordsSomeRouteStillInstalls(t *testing.T) {
118 b := &FailureBook{}
119 b.Record(rec("v2", "a.example.com"))
120 b.Record(rec("v3", "a.example.com"))
121 if err := b.Prune(func(_, _, build string) bool { return build == "v3" }); err != nil {
122 t.Fatal(err)
123 }
124 if _, ok := b.Lookup("bab", "blog", "v2"); ok {
125 t.Fatal("v2 is installed nowhere and should have been pruned")
126 }
127 if _, ok := b.Lookup("bab", "blog", "v3"); !ok {
128 t.Fatal("v3 is still installed and must stay held (spec §8.4)")
129 }
130}
131
132// Review fix 1: a record handed out must not share its host lists with the
133// book, or a status read racing `carlos instances retry` sees a half-edited
134// list.
135func TestRecordsHandedOutAreCopies(t *testing.T) {
136 b := &FailureBook{}
137 in := rec("v2", "a.example.com")
138 in.HeldDown = []string{"a.example.com", "b.example.com", "c.example.com"}
139 if _, err := b.Record(in); err != nil {
140 t.Fatal(err)
141 }
142 in.HeldDown[0] = "scribbled-by-the-caller"
143 if got, _ := b.Lookup("bab", "blog", "v2"); got.HeldDown[0] != "a.example.com" {
144 t.Fatalf("Record kept the caller's HeldDown array: %v", got.HeldDown)
145 }
146
147 look, _ := b.Lookup("bab", "blog", "v2")
148 look.HeldDown[1] = "scribbled-on-lookup"
149 list := b.List()
150 list[0].HeldDown[2] = "scribbled-on-list"
151 forApp := b.ForApp("bab", "blog")
152 forApp[0].HeldDown[0] = "scribbled-on-forapp"
153
154 want := []string{"a.example.com", "b.example.com", "c.example.com"}
155 got, _ := b.Lookup("bab", "blog", "v2")
156 if !slices.Equal(got.HeldDown, want) {
157 t.Fatalf("book's HeldDown = %v, want %v: a handed-out record shares its array", got.HeldDown, want)
158 }
159
160 // Retried is shared the same way.
161 b.RetryHost("bab", "blog", "v2", "b.example.com")
162 look, _ = b.Lookup("bab", "blog", "v2")
163 look.Retried[0] = "scribbled-retried"
164 list = b.List()
165 list[0].Retried[0] = "scribbled-retried"
166 got, _ = b.Lookup("bab", "blog", "v2")
167 if !slices.Equal(got.Retried, []string{"b.example.com"}) {
168 t.Fatalf("book's Retried = %v", got.Retried)
169 }
170}
171
172// The mutators edit copy-on-write, so a slice taken from the book before an
173// edit (as a reader holding the old record would) never changes under it.
174func TestMutatorsDoNotEditSlicesInPlace(t *testing.T) {
175 b := &FailureBook{}
176 in := rec("v2", "a.example.com")
177 in.HeldDown = []string{"a.example.com", "b.example.com"}
178 b.Record(in)
179
180 b.mu.Lock()
181 oldHeld := b.recs[0].HeldDown
182 b.mu.Unlock()
183 b.RetryHost("bab", "blog", "v2", "a.example.com")
184 if !slices.Equal(oldHeld, []string{"a.example.com", "b.example.com"}) {
185 t.Fatalf("RetryHost edited the HeldDown array in place: %v", oldHeld)
186 }
187
188 b.mu.Lock()
189 oldRetried := b.recs[0].Retried
190 b.mu.Unlock()
191 b.Record(rec("v2", "a.example.com")) // spends a.example.com's retry
192 if !slices.Equal(oldRetried, []string{"a.example.com"}) {
193 t.Fatalf("Record edited the Retried array in place: %v", oldRetried)
194 }
195}
196
197// failBook returns a book whose file cannot be written (the path is a
198// directory), plus a func that makes it writable again.
199func failBook(t *testing.T) (*FailureBook, string, func()) {
200 t.Helper()
201 dir := t.TempDir()
202 b, err := OpenFailureBook(dir)
203 if err != nil {
204 t.Fatal(err)
205 }
206 p := filepath.Join(dir, FailureFile)
207 if err := os.Mkdir(p, 0o755); err != nil {
208 t.Fatal(err)
209 }
210 return b, dir, func() {
211 if err := os.Remove(p); err != nil {
212 t.Fatal(err)
213 }
214 }
215}
216
217// Review fix 2 (spec §8.4: the record survives agent restarts): a failed save
218// is retried by the next call, even one that adds nothing new.
219func TestFailedSaveIsRetriedByTheNextRecord(t *testing.T) {
220 b, dir, heal := failBook(t)
221 if _, err := b.Record(rec("v2", "a.example.com")); err == nil {
222 t.Fatal("the first Record must report the failed save")
223 }
224 heal()
225 if added, err := b.Record(rec("v2", "a.example.com")); err != nil || added != Unchanged {
226 t.Fatalf("the repeat Record = %v, %v; want unchanged, nil", added, err)
227 }
228 again, err := OpenFailureBook(dir)
229 if err != nil {
230 t.Fatal(err)
231 }
232 if !again.Blocks("bab", "blog", "v2", "b.example.com", false) {
233 t.Fatal("the no-op Record did not retry the save: a restart would lose the record")
234 }
235}
236
237func TestFailedSaveIsRetriedByPrune(t *testing.T) {
238 b, dir, heal := failBook(t)
239 if _, err := b.Record(rec("v2", "a.example.com")); err == nil {
240 t.Fatal("the first Record must report the failed save")
241 }
242 if err := b.Prune(func(_, _, _ string) bool { return true }); err == nil {
243 t.Fatal("Prune must report that the save still fails")
244 }
245 heal()
246 if err := b.Prune(func(_, _, _ string) bool { return true }); err != nil {
247 t.Fatal(err)
248 }
249 again, err := OpenFailureBook(dir)
250 if err != nil {
251 t.Fatal(err)
252 }
253 if _, ok := again.Lookup("bab", "blog", "v2"); !ok {
254 t.Fatal("a Prune that drops nothing did not retry the save")
255 }
256}
257
258func TestFailedSaveIsRetriedAfterRetryHostAndClearOnPointer(t *testing.T) {
259 b, dir, heal := failBook(t)
260 b.Record(rec("v2", "a.example.com"))
261 b.Record(rec("v3", "a.example.com"))
262 if _, err := b.RetryHost("bab", "blog", "v2", "x.example.com"); err == nil {
263 t.Fatal("RetryHost must report the failed save")
264 }
265 heal()
266 // Nothing to clear here, but the pending save is still owed.
267 if ok, err := b.ClearOnPointer("bab", "blog", "v3", promotedAt); ok || err != nil {
268 t.Fatalf("ClearOnPointer = %v, %v", ok, err)
269 }
270 again, err := OpenFailureBook(dir)
271 if err != nil {
272 t.Fatal(err)
273 }
274 if got := again.List(); len(got) != 2 || !slices.Equal(got[0].Retried, []string{"x.example.com"}) {
275 t.Fatalf("reopened book = %+v", got)
276 }
277
278 // And a failing ClearOnPointer is itself retried.
279 if err := os.Remove(filepath.Join(dir, FailureFile)); err != nil {
280 t.Fatal(err)
281 }
282 if err := os.Mkdir(filepath.Join(dir, FailureFile), 0o755); err != nil {
283 t.Fatal(err)
284 }
285 if ok, err := b.ClearOnPointer("bab", "blog", "v3", promotedAt.Add(time.Second)); !ok || err == nil {
286 t.Fatalf("ClearOnPointer = %v, %v; want true and the save error", ok, err)
287 }
288 heal()
289 if _, err := b.RetryHost("bab", "blog", "v2", "x.example.com"); err != nil { // no-op
290 t.Fatal(err)
291 }
292 again, err = OpenFailureBook(dir)
293 if err != nil {
294 t.Fatal(err)
295 }
296 if _, ok := again.Lookup("bab", "blog", "v3"); ok {
297 t.Fatal("the cleared record came back after a restart")
298 }
299}
300