amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package release
2
3import (
4 "bytes"
5 "context"
6 "crypto/ed25519"
7 "errors"
8 "os"
9 "path/filepath"
10 "strings"
11 "testing"
12
13 "github.com/carlosframework/platform/internal/hostwrite"
14 "github.com/carlosframework/platform/internal/registry"
15)
16
17func groupFixtureRoute(t *testing.T, reg *registry.Registry, host, socket, addr, unit string) {
18 t.Helper()
19 if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "console",
20 Channel: "canary/x", Socket: socket, Addr: addr, Unit: unit}); err != nil {
21 t.Fatalf("upsert %s: %v", host, err)
22 }
23}
24
25func TestProcessGroupKeysOnTheEffectiveUpstream(t *testing.T) {
26 reg := testRegistry(t)
27 groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service")
28 groupFixtureRoute(t, reg, "console.example.com", "", "127.0.0.1:8090", "")
29 groupFixtureRoute(t, reg, "other.example.com", "", "127.0.0.1:9000", "")
30 groupFixtureRoute(t, reg, "sock.example.com", "/run/x.sock", "127.0.0.1:8090", "")
31 bound, _ := reg.Get("platform.example.com")
32 got := processGroup(reg, *bound)
33 if len(got) != 1 || got[0].Host != "console.example.com" {
34 t.Fatalf("group = %+v, want only console.example.com (socket wins over addr; 9000 is another process)", got)
35 }
36}
37
38func TestProcessGroupNeverMatchesEmptyUpstreams(t *testing.T) {
39 reg := testRegistry(t)
40 groupFixtureRoute(t, reg, "a.example.com", "", "", "a.service")
41 groupFixtureRoute(t, reg, "b.example.com", "", "", "")
42 bound, _ := reg.Get("a.example.com")
43 if got := processGroup(reg, *bound); len(got) != 0 {
44 t.Fatalf("group = %+v, want none: two empty upstreams are not one process", got)
45 }
46}
47
48// The flagship shape: three hosts, one process, the unit runs the LAST
49// host's link. Hosts before it were repointed and stamped; the bound host's
50// restart reports the unit did not come up.
51func TestNotUpRollsTheWholeGroupBackAndRefusesEveryHost(t *testing.T) {
52 ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
53 if err := reg.SetUnit("console.example.com", ""); err != nil {
54 t.Fatal(err)
55 }
56 groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service")
57 if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
58 t.Fatal(err)
59 }
60 if err := PointLive(opt, "platform.example.com", "bab", "old", "console"); err != nil {
61 t.Fatal(err)
62 }
63 bake := t.TempDir()
64 restart := func(h string) error {
65 if h == "platform.example.com" {
66 return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
67 }
68 return nil
69 }
70 var rolledBackHost string
71 rb := func(h string) error { rolledBackHost = h; return nil }
72 var out bytes.Buffer
73 withCapturedStdout(t, &out, func() {
74 if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(rb)); err != nil {
75 t.Fatalf("Pass: %v", err)
76 }
77 })
78 for _, h := range []string{"console.example.com", "platform.example.com"} {
79 if v, _ := LiveVersion(opt, h); v != "old" {
80 t.Fatalf("%s live = %q, want old", h, v)
81 }
82 rt, _ := reg.Get(h)
83 if rt.Version == "new" {
84 t.Fatalf("%s still stamped new: %+v", h, rt)
85 }
86 if r, _ := ReadRefusal(bake, h); r == nil || r.Version != "new" {
87 t.Fatalf("%s refusal = %+v, want one naming new", h, r)
88 }
89 }
90 if rolledBackHost != "platform.example.com" {
91 t.Fatalf("rollback restart ran for %q, want the bound host", rolledBackHost)
92 }
93 for _, h := range []string{"console.example.com", "platform.example.com"} {
94 if r, _ := ReadRefusal(bake, h); !r.Recovered || r.RestoredVersion != "old" || r.RecoveryError != "" {
95 t.Fatalf("%s refusal = %+v, want recovered onto old", h, r)
96 }
97 }
98}
99
100// notUpGroupFixture is the flagship shape again: console.example.com is
101// served by platform.example.com's console.service, the bound route has a
102// history (old over older), and the canary pointer names the rejected new.
103func notUpGroupFixture(t *testing.T) (context.Context, *registry.Registry, Store, ed25519.PublicKey, string) {
104 t.Helper()
105 ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
106 if err := reg.SetUnit("console.example.com", ""); err != nil {
107 t.Fatal(err)
108 }
109 groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service")
110 if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
111 t.Fatal(err)
112 }
113 if err := PointLive(opt, "platform.example.com", "bab", "old", "console"); err != nil {
114 t.Fatal(err)
115 }
116 if err := reg.SetVersions("platform.example.com", "old", "older"); err != nil {
117 t.Fatal(err)
118 }
119 return ctx, reg, s, pub, opt
120}
121
122// Final review F6 (and the ledger's Task 3 minor): a sibling rolled back with
123// its process is stamped with the BOUND route's previous version. Its own
124// PrevVersion names restored by then (this pass stamped it new over old).
125func TestRollbackGroupStampsTheBoundRoutesHistory(t *testing.T) {
126 ctx, reg, s, pub, opt := notUpGroupFixture(t)
127 restart := func(h string) error {
128 if h == "platform.example.com" {
129 return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
130 }
131 return nil
132 }
133 var out bytes.Buffer
134 withCapturedStdout(t, &out, func() {
135 if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(t.TempDir()), WithRollbackRestart(func(string) error { return nil })); err != nil {
136 t.Fatalf("Pass: %v", err)
137 }
138 })
139 if rt := mustRoute(t, reg, "console.example.com"); rt.Version != "old" || rt.PrevVersion != "older" {
140 t.Fatalf("sibling stamp = %q over %q, want old over older (bound's history)", rt.Version, rt.PrevVersion)
141 }
142}
143
144// Final review F6: the rollback restart failed. Every record says so, with
145// the error, so the watcher does not publish a recovery that never happened.
146func TestRollbackRestartFailureIsRecordedAsNotRecovered(t *testing.T) {
147 ctx, reg, s, pub, opt := notUpGroupFixture(t)
148 restart := func(h string) error {
149 if h == "platform.example.com" {
150 return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
151 }
152 return nil
153 }
154 rb := func(string) error {
155 return errors.New("rollback did not come up either: console.service did not come up: unit entered failed state")
156 }
157 bake := t.TempDir()
158 var out bytes.Buffer
159 withCapturedStdout(t, &out, func() {
160 if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(rb)); err != nil {
161 t.Fatalf("Pass: %v", err)
162 }
163 })
164 for _, h := range []string{"console.example.com", "platform.example.com"} {
165 r, _ := ReadRefusal(bake, h)
166 if r == nil || r.Recovered || r.RestoredVersion != "old" ||
167 !strings.Contains(r.RecoveryError, "rollback did not come up either") {
168 t.Fatalf("%s refusal = %+v, want not recovered, restored old, carrying the restart's error", h, r)
169 }
170 }
171}
172
173// Final review F6: the BOUND route's own symlink rollback failed, so the
174// process still runs (or will run) the rejected build. Siblings are not
175// repointed onto a version their process is not running, and the records
176// say there was no recovery.
177func TestBoundRollbackFailureLeavesSiblingsAlone(t *testing.T) {
178 ctx, reg, s, pub, opt := notUpGroupFixture(t)
179 restart := func(h string) error {
180 if h == "platform.example.com" {
181 // A non-empty directory where PointLive's temp link goes: the
182 // repoint back to old fails for this host only.
183 if err := os.MkdirAll(filepath.Join(opt, "live", "platform.example.com.tmp", "x"), 0o755); err != nil {
184 t.Fatal(err)
185 }
186 return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
187 }
188 return nil
189 }
190 rbCalled := false
191 bake := t.TempDir()
192 var out bytes.Buffer
193 withCapturedStdout(t, &out, func() {
194 if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake),
195 WithRollbackRestart(func(string) error { rbCalled = true; return nil })); err != nil {
196 t.Fatalf("Pass: %v", err)
197 }
198 })
199 if v, _ := LiveVersion(opt, "platform.example.com"); v != "new" {
200 t.Fatalf("bound live = %q; the fixture meant its rollback to fail", v)
201 }
202 if v, _ := LiveVersion(opt, "console.example.com"); v != "new" {
203 t.Fatalf("sibling live = %q, want it left on new: its process never went back to old", v)
204 }
205 if rbCalled {
206 t.Fatal("the rollback restart ran with the link still on the rejected build")
207 }
208 r, _ := ReadRefusal(bake, "platform.example.com")
209 if r == nil || r.Recovered || r.RestoredVersion != "" || !strings.Contains(r.RecoveryError, "could not roll the symlink back to old") {
210 t.Fatalf("bound refusal = %+v, want not recovered with the repoint's error", r)
211 }
212}
213
214// Final review F4: a same-upstream route following ANOTHER channel is a
215// separate promotion. It is neither repointed nor refused with the bound
216// route's PromotedAt.
217func TestRollbackGroupLeavesAnotherChannelAlone(t *testing.T) {
218 ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
219 if err := reg.SetUnit("console.example.com", "console.service"); err != nil {
220 t.Fatal(err)
221 }
222 if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
223 t.Fatal(err)
224 }
225 if err := reg.Upsert(registry.Route{Host: "zz-beta.example.com", Kind: "instance", App: "console",
226 Channel: "beta", Addr: "127.0.0.1:8090"}); err != nil {
227 t.Fatal(err)
228 }
229 if err := StageArtifact(opt, "bab", "new", "console", []byte("bad build")); err != nil {
230 t.Fatal(err)
231 }
232 if err := PointLive(opt, "zz-beta.example.com", "bab", "new", "console"); err != nil {
233 t.Fatal(err)
234 }
235 if got := processGroup(reg, mustRoute(t, reg, "console.example.com")); len(got) != 0 {
236 t.Fatalf("group = %+v, want none: zz-beta follows another channel", got)
237 }
238 restart := func(h string) error {
239 if h == "console.example.com" {
240 return &UnitNotUpError{Unit: "console.service", Reason: "not ready after 30s"}
241 }
242 return nil
243 }
244 bake := t.TempDir()
245 var out bytes.Buffer
246 withCapturedStdout(t, &out, func() {
247 if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(func(string) error { return nil })); err != nil {
248 t.Fatalf("Pass: %v", err)
249 }
250 })
251 if v, _ := LiveVersion(opt, "zz-beta.example.com"); v != "new" {
252 t.Fatalf("another channel's route was rolled back to %q", v)
253 }
254 if r, _ := ReadRefusal(bake, "zz-beta.example.com"); r != nil {
255 t.Fatalf("another channel's route was refused with the bound route's promotion: %+v", r)
256 }
257 if r, _ := ReadRefusal(bake, "console.example.com"); r == nil {
258 t.Fatal("the bound route has no refusal")
259 }
260}
261
262func mustRoute(t *testing.T, reg *registry.Registry, host string) registry.Route {
263 t.Helper()
264 rt, err := reg.Get(host)
265 if err != nil || rt == nil {
266 t.Fatalf("get %s: %+v, %v", host, rt, err)
267 }
268 return *rt
269}
270
271func TestRollbackGroupLeavesOtherUpstreamAlone(t *testing.T) {
272 ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com")
273 if err := reg.SetUnit("console.example.com", "console.service"); err != nil {
274 t.Fatal(err)
275 }
276 if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil {
277 t.Fatal(err)
278 }
279 groupFixtureRoute(t, reg, "zz-other.example.com", "", "127.0.0.1:9000", "")
280 if err := StageArtifact(opt, "bab", "new", "console", []byte("bad build")); err != nil {
281 t.Fatal(err)
282 }
283 if err := PointLive(opt, "zz-other.example.com", "bab", "new", "console"); err != nil {
284 t.Fatal(err)
285 }
286 restart := func(h string) error {
287 if h == "console.example.com" {
288 return &UnitNotUpError{Unit: "console.service", Reason: "not ready after 30s"}
289 }
290 return nil
291 }
292 bake := t.TempDir()
293 if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(func(string) error { return nil })); err != nil {
294 t.Fatalf("Pass: %v", err)
295 }
296 if v, _ := LiveVersion(opt, "zz-other.example.com"); v != "new" {
297 t.Fatalf("another process's route was rolled back to %q", v)
298 }
299 if r, _ := ReadRefusal(bake, "zz-other.example.com"); r != nil {
300 t.Fatalf("another process's route was refused: %+v", r)
301 }
302}
303
304// scriptedWork is one host's release selection under lifecycle host writes.
305// It counts With calls, refuses every effect once stale, and lets a test go
306// stale right after a chosen call's effect has landed.
307type scriptedWork struct {
308 name string
309 calls int
310 stale bool
311 after func(call int)
312}
313
314func (w *scriptedWork) With(_ context.Context, id hostwrite.Identity, fn func() error) error {
315 w.calls++
316 if id.Release == "" {
317 return errors.New("missing release identity")
318 }
319 if w.stale {
320 return errors.New(w.name + ": release selection changed")
321 }
322 err := fn()
323 if w.after != nil {
324 w.after(w.calls)
325 }
326 return err
327}
328
329// scriptedSelector hands out each host's works in order, one per Select, and
330// keeps the last for any later Select.
331type scriptedSelector struct {
332 works map[string][]*scriptedWork
333 selects map[string]int
334}
335
336func (s *scriptedSelector) Select(_ context.Context, host, _ string) (hostwrite.Work, error) {
337 list := s.works[host]
338 if len(list) == 0 {
339 return nil, errors.New("no work for " + host)
340 }
341 i := min(s.selects[host], len(list)-1)
342 s.selects[host]++
343 return list[i], nil
344}
345
346func claimWrites(sel *scriptedSelector) PassOption {
347 return WithHostWrites(hostwrite.Config{Enabled: true, Selector: sel})
348}
349
350// boundNotUp is the selected restart for notUpGroupFixture: the bound route's
351// unit does not come up, the sibling's restart is the shared process's no-op.
352func boundNotUp(restarts *int) hostwrite.Callback {
353 return func(_ context.Context, s hostwrite.Selection) error {
354 *restarts++
355 if s.Host == "platform.example.com" {
356 return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"}
357 }
358 return nil
359 }
360}
361
362// Merge review P1: authority lost after the group rolled back, before the
363// rollback start. The start is skipped, but every group host is still refused
364// (not recovered, carrying the authority error), so the next pass, with fresh
365// authority and an empty poisoned map, does not adopt the rejected build.
366func TestClaimAuthorityLostBeforeRollbackStartStillRefusesTheGroup(t *testing.T) {
367 ctx, reg, s, pub, opt := notUpGroupFixture(t)
368 bake := t.TempDir()
369 bound := &scriptedWork{name: "platform"}
370 // Bound's calls: 1 forward repoint, 2 the pre-restart check, 3 the
371 // rollback repoint, 4 the pre-start check. Stale after 3 has landed.
372 bound.after = func(call int) {
373 if call == 3 {
374 bound.stale = true
375 }
376 }
377 sibling := &scriptedWork{name: "console"}
378 sel := &scriptedSelector{selects: map[string]int{}, works: map[string][]*scriptedWork{
379 "platform.example.com": {bound}, "console.example.com": {sibling}}}
380 restarts, starts := 0, 0
381 start := func(ctx context.Context, s hostwrite.Selection) error { starts++; return nil }
382 var out bytes.Buffer
383 withCapturedStdout(t, &out, func() {
384 if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("legacy restart"); return nil },
385 BakeDir(bake), claimWrites(sel), WithSelectedRestarts(boundNotUp(&restarts), start),
386 WithRollbackRestart(func(string) error { t.Fatal("legacy rollback restart"); return nil })); err != nil {
387 t.Fatalf("Pass: %v", err)
388 }
389 })
390 if !bound.stale {
391 t.Fatalf("the fixture never reached the rollback repoint:\n%s", out.String())
392 }
393 if starts != 0 {
394 t.Fatalf("rollback start ran %d times after authority changed", starts)
395 }
396 for _, h := range []string{"console.example.com", "platform.example.com"} {
397 if v, _ := LiveVersion(opt, h); v != "old" {
398 t.Fatalf("%s live = %q, want old (rolled back before authority changed)", h, v)
399 }
400 r, err := ReadRefusal(bake, h)
401 if err != nil || r == nil || r.Version != "new" || r.Recovered || r.RestoredVersion != "old" ||
402 !strings.Contains(r.RecoveryError, "release authority changed before rollback start") {
403 t.Fatalf("%s refusal = %+v, %v; want new refused, not recovered, with the authority error\n%s", h, r, err, out.String())
404 }
405 }
406
407 // The next pass: fresh, valid authority for both hosts, nothing poisoned.
408 sel2 := &scriptedSelector{selects: map[string]int{}, works: map[string][]*scriptedWork{
409 "platform.example.com": {{name: "platform2"}}, "console.example.com": {{name: "console2"}}}}
410 restarts = 0
411 out.Reset()
412 withCapturedStdout(t, &out, func() {
413 if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("legacy restart"); return nil },
414 BakeDir(bake), claimWrites(sel2), WithSelectedRestarts(boundNotUp(&restarts), start),
415 WithRollbackRestart(func(string) error { t.Fatal("legacy rollback restart"); return nil })); err != nil {
416 t.Fatalf("second Pass: %v", err)
417 }
418 })
419 if restarts != 0 || starts != 0 {
420 t.Fatalf("second pass restarted %d / started %d: it re-adopted the rejected build\n%s", restarts, starts, out.String())
421 }
422 for _, h := range []string{"console.example.com", "platform.example.com"} {
423 if v, _ := LiveVersion(opt, h); v != "old" {
424 t.Fatalf("%s live = %q after the second pass, want old", h, v)
425 }
426 if rt := mustRoute(t, reg, h); rt.Version == "new" {
427 t.Fatalf("%s stamped new by the second pass", h)
428 }
429 }
430}
431
432// Merge review P1: the sibling's installation changes between its rollback
433// repoint and its stamp. The stamp is refused under the sibling's one
434// selection (the pass's original), never re-selected and handed to the
435// successor.
436func TestClaimSiblingStampRefusedWhenItsSelectionGoesStale(t *testing.T) {
437 ctx, reg, s, pub, opt := notUpGroupFixture(t)
438 bake := t.TempDir()
439 bound := &scriptedWork{name: "platform"}
440 sibling := &scriptedWork{name: "console"}
441 // Sibling's calls: 1 forward repoint, 2 the pre-restart check, 3 the
442 // forward stamp, 4 the rollback repoint. Stale after 4 has landed.
443 sibling.after = func(call int) {
444 if call == 4 {
445 sibling.stale = true
446 }
447 }
448 successor := &scriptedWork{name: "console-successor"}
449 sel := &scriptedSelector{selects: map[string]int{}, works: map[string][]*scriptedWork{
450 "platform.example.com": {bound}, "console.example.com": {sibling, successor}}}
451 restarts, starts := 0, 0
452 start := func(ctx context.Context, s hostwrite.Selection) error {
453 return s.With(ctx, func() error { starts++; return nil })
454 }
455 var out bytes.Buffer
456 withCapturedStdout(t, &out, func() {
457 if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("legacy restart"); return nil },
458 BakeDir(bake), claimWrites(sel), WithSelectedRestarts(boundNotUp(&restarts), start),
459 WithRollbackRestart(func(string) error { t.Fatal("legacy rollback restart"); return nil })); err != nil {
460 t.Fatalf("Pass: %v", err)
461 }
462 })
463 if n := sel.selects["console.example.com"]; n != 1 || successor.calls != 0 {
464 t.Fatalf("sibling selected %d times, successor used %d times: want the pass's one selection only", n, successor.calls)
465 }
466 if !sibling.stale {
467 t.Fatalf("the fixture never reached the sibling's rollback repoint:\n%s", out.String())
468 }
469 if v, _ := LiveVersion(opt, "console.example.com"); v != "old" {
470 t.Fatalf("sibling live = %q, want old (the repoint landed before the change)", v)
471 }
472 if rt := mustRoute(t, reg, "console.example.com"); rt.Version != "new" {
473 t.Fatalf("sibling stamped %q over %q: the stamp should have been refused", rt.Version, rt.PrevVersion)
474 }
475 if !strings.Contains(out.String(), "console: release selection changed") {
476 t.Fatalf("the refused stamp was not logged:\n%s", out.String())
477 }
478 if starts != 1 {
479 t.Fatalf("bound's rollback start ran %d times, want 1", starts)
480 }
481 for _, h := range []string{"console.example.com", "platform.example.com"} {
482 if r, _ := ReadRefusal(bake, h); r == nil || r.Version != "new" {
483 t.Fatalf("%s refusal = %+v, want one naming new", h, r)
484 }
485 }
486}
487