amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package instancectl
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "net"
8 "net/http"
9 "net/http/httptest"
10 "os"
11 "path/filepath"
12 "strings"
13 "sync"
14 "testing"
15 "time"
16
17 "github.com/carlosframework/platform/internal/version"
18)
19
20type fakeCtl struct {
21 mu sync.Mutex
22 calls []string
23 // answers maps a leading argument pattern ("show -p InvocationID") to output.
24 answers map[string]string
25 fail map[string]error
26 // onStart, if set, runs once, synchronously, the first time args[0] is
27 // "start", before run returns to its caller. A test uses it to
28 // simulate a client that disconnects the instant systemctl start
29 // answers (fix round 1, item 4).
30 onStart func()
31 // before, keyed by a verb (args[0]), runs once, synchronously, before
32 // that verb's call is recorded: a test holds a handler there.
33 before map[string]func()
34 // unseenCancel models a handler that has not yet seen its client go
35 // (#292): its systemctl calls run although the request's context is
36 // done, the race exec.CommandContext cannot close on its own.
37 unseenCancel bool
38}
39
40func (f *fakeCtl) run(ctx context.Context, args ...string) (string, error) {
41 f.mu.Lock()
42 var pre func()
43 if len(args) > 0 && f.before[args[0]] != nil {
44 pre = f.before[args[0]]
45 delete(f.before, args[0])
46 }
47 f.mu.Unlock()
48 if pre != nil {
49 pre()
50 }
51 f.mu.Lock()
52 line := strings.Join(args, " ")
53 f.calls = append(f.calls, line)
54 var hook func()
55 if len(args) > 0 && args[0] == "start" && f.onStart != nil {
56 hook, f.onStart = f.onStart, nil
57 }
58 unseen := f.unseenCancel
59 f.mu.Unlock()
60 if hook != nil {
61 hook()
62 }
63 // Production's Systemctl is exec.CommandContext, which fails outright
64 // once its context is done rather than completing anyway; the fake
65 // must behave the same way, or a test cannot tell a handler that used
66 // the request's context apart from one that used a detached one (fix
67 // round 1, item 4).
68 if ctx.Err() != nil && !unseen {
69 return "", ctx.Err()
70 }
71 f.mu.Lock()
72 defer f.mu.Unlock()
73 for k, err := range f.fail {
74 if strings.HasPrefix(line, k) {
75 return "", err
76 }
77 }
78 for k, out := range f.answers {
79 if strings.HasPrefix(line, k) {
80 return out, nil
81 }
82 }
83 return "", nil
84}
85
86func (f *fakeCtl) joined() string {
87 f.mu.Lock()
88 defer f.mu.Unlock()
89 return strings.Join(f.calls, "\n")
90}
91
92// newServer starts a Server behind a real unix socket. Any configure hooks
93// run before the accept loop starts, so a test that wants a non-default
94// PeerOK (or anything else) sets it up front rather than racing Serve's
95// first Accept (codex review fix round 2 item 1).
96func newServer(t *testing.T, ctl *fakeCtl, configure ...func(*Server)) (*Server, *http.Client) {
97 t.Helper()
98 s, sock := serveOnSocket(t, ctl, configure...)
99 client := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
100 return (&net.Dialer{}).DialContext(ctx, "unix", sock)
101 }}}
102 return s, client
103}
104
105// serveOnSocket is newServer without the http.Client: it returns the
106// socket path so a test of the agent-side Client can dial it the way
107// production does.
108func serveOnSocket(t *testing.T, ctl *fakeCtl, configure ...func(*Server)) (*Server, string) {
109 t.Helper()
110 root := t.TempDir()
111 s := &Server{
112 Systemctl: ctl.run,
113 PeerOK: func(net.Conn) error { return nil },
114 CgroupRoot: filepath.Join(root, "cgroup"),
115 ProbeOut: filepath.Join(root, "probe.out"),
116 FixtureOK: func() error { return nil },
117 ProbeEvery: time.Hour,
118 Logf: t.Logf,
119 // The same binary as the Client this test process is, as in
120 // production, where both are /opt/carlos/carlos.
121 Build: version.Build,
122 }
123 for _, configure := range configure {
124 configure(s)
125 }
126 dir, err := os.MkdirTemp("", "ic")
127 if err != nil {
128 t.Fatal(err)
129 }
130 t.Cleanup(func() { os.RemoveAll(dir) })
131 sock := filepath.Join(dir, "c.sock")
132 ln, err := net.Listen("unix", sock)
133 if err != nil {
134 t.Skipf("cannot bind a unix socket here (%v); run with TMPDIR=/tmp/g outside the sandbox", err)
135 }
136 ctx, cancel := context.WithCancel(context.Background())
137 t.Cleanup(cancel)
138 go s.Serve(ctx, ln) //nolint:errcheck
139 t.Cleanup(func() { ln.Close() })
140 return s, sock
141}
142
143func post(t *testing.T, c *http.Client, path string) *http.Response {
144 t.Helper()
145 resp, err := c.Post("http://unix"+path, "", nil)
146 if err != nil {
147 t.Fatalf("POST %s: %v", path, err)
148 }
149 return resp
150}
151
152func TestStartSetsTheTierThenStartsThenReadsTheInvocation(t *testing.T) {
153 ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "abc123\n"}}
154 _, c := newServer(t, ctl)
155 resp := post(t, c, "/start?host=bill-2.example.com&role=run&memory_max=536870912")
156 if resp.StatusCode != 200 {
157 t.Fatalf("status %d", resp.StatusCode)
158 }
159 var out StartResponse
160 if err := json.NewDecoder(resp.Body).Decode(&out); err != nil || out.InvocationID != "abc123" {
161 t.Errorf("response = %+v, %v", out, err)
162 }
163 want := "set-property --runtime carlos-instances-bill\\x2d2.example.com.slice MemoryMax=536870912 MemoryHigh=483183819 MemorySwapMax=536870912 CPUWeight=100 CPUQuota=100% IOWeight=100\n" +
164 "start carlos-instance@bill\\x2d2.example.com.service\n" +
165 "show -p InvocationID --value carlos-instance@bill\\x2d2.example.com.service"
166 if got := ctl.joined(); got != want {
167 t.Errorf("systemctl calls:\n%s\nwant:\n%s", got, want)
168 }
169}
170
171func TestStartRefusesBadInputsWithoutCallingSystemctl(t *testing.T) {
172 ctl := &fakeCtl{}
173 _, c := newServer(t, ctl)
174 for _, q := range []string{
175 "host=Bad_Host&role=run&memory_max=536870912",
176 "host=h.example.com&role=exec&memory_max=536870912",
177 "host=h.example.com&role=run&memory_max=1",
178 "host=h.example.com&role=seed&memory_max=536870912", // seed is not a /start role
179 "role=run&memory_max=536870912",
180 } {
181 resp := post(t, c, "/start?"+q)
182 if resp.StatusCode != 400 {
183 t.Errorf("%s: status %d, want 400", q, resp.StatusCode)
184 }
185 }
186 if ctl.joined() != "" {
187 t.Errorf("systemctl was called: %s", ctl.joined())
188 }
189}
190
191func TestStartReportsSystemctlFailureAs502(t *testing.T) {
192 ctl := &fakeCtl{fail: map[string]error{"start ": errors.New("Job for carlos-instance@h.example.com.service failed")}}
193 _, c := newServer(t, ctl)
194 resp := post(t, c, "/start?host=h.example.com&role=sidecar&memory_max=1073741824")
195 if resp.StatusCode != 502 {
196 t.Errorf("status %d, want 502", resp.StatusCode)
197 }
198 if !strings.Contains(ctl.joined(), "MemoryMax=1073741824 MemoryHigh=966367638 MemorySwapMax=1073741824 CPUWeight=100 CPUQuota=200% IOWeight=100") {
199 t.Errorf("children tier not applied: %s", ctl.joined())
200 }
201}
202
203func TestSeedBlocksReadsTheRetainedResultWaitsAndStops(t *testing.T) {
204 ctl := &fakeCtl{answers: map[string]string{
205 "show -p InvocationID,Result,ExecMainStatus,ExecMainCode": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n",
206 }}
207 s, c := newServer(t, ctl)
208 // An empty cgroup: cgroup.events says populated 0.
209 ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
210 if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
211 t.Fatal(err)
212 }
213 if err := os.WriteFile(ev, []byte("populated 0\nfrozen 0\n"), 0o644); err != nil {
214 t.Fatal(err)
215 }
216 resp := post(t, c, "/seed?host=h.example.com&memory_max=536870912")
217 if resp.StatusCode != 200 {
218 t.Fatalf("status %d", resp.StatusCode)
219 }
220 var out SeedResponse
221 if err := json.NewDecoder(resp.Body).Decode(&out); err != nil || out.InvocationID != "feed" || out.Result != "success" || out.Status != "0" || out.Code != "exited" {
222 t.Errorf("response = %+v, %v", out, err)
223 }
224 calls := ctl.joined()
225 for _, want := range []string{"set-property --runtime carlos-instances-h.example.com.slice", "start carlos-instance-seed@h.example.com.service", "show -p InvocationID,Result,ExecMainStatus,ExecMainCode carlos-instance-seed@h.example.com.service", "stop carlos-instance-seed@h.example.com.service"} {
226 if !strings.Contains(calls, want) {
227 t.Errorf("missing %q in\n%s", want, calls)
228 }
229 }
230 if strings.Index(calls, "show -p") > strings.Index(calls, "stop ") {
231 t.Errorf("stop before show:\n%s", calls)
232 }
233}
234
235func TestSeedFailsWhenTheCgroupStaysPopulated(t *testing.T) {
236 ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID,Result": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n"}}
237 s, c := newServer(t, ctl)
238 s.cgroupWaitBudget = 200 * time.Millisecond
239 ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
240 if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
241 t.Fatal(err)
242 }
243 if err := os.WriteFile(ev, []byte("populated 1\n"), 0o644); err != nil {
244 t.Fatal(err)
245 }
246 resp := post(t, c, "/seed?host=h.example.com&memory_max=536870912")
247 if resp.StatusCode != 502 {
248 t.Errorf("status %d, want 502 while processes remain", resp.StatusCode)
249 }
250 // stop is issued (on the detached context) before the wait gives up, not
251 // only after: it is the action that actually kills the survivors (final
252 // review, plan 2, item 4).
253 if !strings.Contains(ctl.joined(), "stop carlos-instance-seed@h.example.com.service") {
254 t.Errorf("stop was not issued while the cgroup stayed populated: %s", ctl.joined())
255 }
256}
257
258// TestSeedStopRunsDespiteACancelledClientContext proves the WHOLE
259// post-start sequence (the second InvocationID read, show, the cleanup
260// stop, WaitCgroupEmpty) runs on a context detached from the request, not
261// only the final stop (fix round 1, item 4). fakeCtl.run now honours
262// ctx.Err(), and its onStart hook cancels the request context the instant
263// the first "start" answers, simulating a client that hung up right then.
264// Before this fix, "show" ran on the request context: a cancelled one
265// failed it outright, the handler 502'd there, and the cleanup stop after
266// it was never reached. Confirmed empirically: temporarily swapping in
267// commit 7576db8e's handleSeed (the pre-fix-round-1 version, which used
268// ctx throughout after start) and running just this test failed with
269// "status 502, body show: context canceled" and never reached a stop
270// call, exactly the reported bug. The swap was reverted immediately
271// after, see the report for both runs.
272func TestSeedStopRunsDespiteACancelledClientContext(t *testing.T) {
273 ctx, cancel := context.WithCancel(context.Background())
274 ctl := &fakeCtl{
275 answers: map[string]string{
276 "show -p InvocationID,Result,ExecMainStatus,ExecMainCode": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n",
277 },
278 onStart: cancel,
279 }
280 root := t.TempDir()
281 s := &Server{
282 Systemctl: ctl.run,
283 PeerOK: func(net.Conn) error { return nil },
284 CgroupRoot: filepath.Join(root, "cgroup"),
285 Logf: t.Logf,
286 }
287 s.cgroupWaitBudget = time.Second
288 ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
289 if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
290 t.Fatal(err)
291 }
292 if err := os.WriteFile(ev, []byte("populated 0\n"), 0o644); err != nil {
293 t.Fatal(err)
294 }
295 r := httptest.NewRequest("POST", "/seed?host=h.example.com&memory_max=536870912", nil).WithContext(ctx)
296 w := httptest.NewRecorder()
297 s.handleSeed(w, r)
298 if w.Code != http.StatusOK {
299 t.Fatalf("status %d, body %s, want 200 despite the cancelled client context", w.Code, w.Body.String())
300 }
301 var out SeedResponse
302 if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.InvocationID != "feed" || out.Result != "success" {
303 t.Errorf("response = %+v, %v", out, err)
304 }
305 if !strings.Contains(ctl.joined(), "stop carlos-instance-seed@h.example.com.service") {
306 t.Errorf("stop did not run despite the cancelled request context:\n%s", ctl.joined())
307 }
308}
309
310// TestSeedStopsAndRestartsAStaleInvocation proves the stale-result guard:
311// when start leaves the SAME (non-empty) InvocationID in place, the
312// handler stops the abandoned run and starts again so it reads its own
313// result, never one left behind by a prior request (final review, plan 2,
314// item 4).
315func TestSeedStopsAndRestartsAStaleInvocation(t *testing.T) {
316 var mu sync.Mutex
317 var calls []string
318 invocation := "stale00000000000000000000000001"
319 starts := 0
320 root := t.TempDir()
321 systemctl := func(_ context.Context, args ...string) (string, error) {
322 mu.Lock()
323 calls = append(calls, strings.Join(args, " "))
324 mu.Unlock()
325 switch args[0] {
326 case "start":
327 starts++
328 if starts == 2 {
329 // The retry actually moves the invocation on, as a real
330 // systemd start would after the stale unit was stopped.
331 invocation = "fresh0000000000000000000000002"
332 }
333 return "", nil
334 case "show":
335 if len(args) >= 3 && args[2] == "InvocationID" {
336 return invocation + "\n", nil
337 }
338 return "InvocationID=" + invocation + "\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n", nil
339 default:
340 return "", nil
341 }
342 }
343 s := &Server{
344 Systemctl: systemctl,
345 PeerOK: func(net.Conn) error { return nil },
346 CgroupRoot: filepath.Join(root, "cgroup"),
347 Logf: t.Logf,
348 }
349 s.cgroupWaitBudget = time.Second
350 ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
351 if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
352 t.Fatal(err)
353 }
354 if err := os.WriteFile(ev, []byte("populated 0\n"), 0o644); err != nil {
355 t.Fatal(err)
356 }
357 r := httptest.NewRequest("POST", "/seed?host=h.example.com&memory_max=536870912", nil)
358 w := httptest.NewRecorder()
359 s.handleSeed(w, r)
360 if w.Code != http.StatusOK {
361 t.Fatalf("status %d, body %s", w.Code, w.Body.String())
362 }
363 if starts != 2 {
364 t.Errorf("starts = %d, want 2 (a stale invocation must trigger a stop and retry)", starts)
365 }
366 mu.Lock()
367 joined := strings.Join(calls, "\n")
368 mu.Unlock()
369 if !strings.Contains(joined, "stop carlos-instance-seed@h.example.com.service") {
370 t.Errorf("no stop issued for the stale invocation:\n%s", joined)
371 }
372 var out SeedResponse
373 if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.InvocationID != invocation {
374 t.Errorf("response = %+v, %v, want the FRESH invocation %s", out, err, invocation)
375 }
376}
377
378func TestStopResetAndTheirTolerances(t *testing.T) {
379 ctl := &fakeCtl{fail: map[string]error{"reset-failed carlos-instance-sidecar@": errors.New("Unit carlos-instance-sidecar@h.example.com.service not loaded.")}}
380 _, c := newServer(t, ctl)
381 if resp := post(t, c, "/stop?host=h.example.com&role=seed"); resp.StatusCode != 204 {
382 t.Errorf("stop status %d", resp.StatusCode)
383 }
384 if resp := post(t, c, "/reset?host=h.example.com&role=sidecar"); resp.StatusCode != 204 {
385 t.Errorf("reset of an unloaded unit: status %d, want 204 (tolerated)", resp.StatusCode)
386 }
387 if resp := post(t, c, "/reset?host=h.example.com&role=nope"); resp.StatusCode != 400 {
388 t.Errorf("bad role: status %d", resp.StatusCode)
389 }
390 if !strings.Contains(ctl.joined(), "stop carlos-instance-seed@h.example.com.service") {
391 t.Errorf("stop not issued: %s", ctl.joined())
392 }
393}
394
395func TestSweepStopsEveryInstanceUnitAndJob(t *testing.T) {
396 ctl := &fakeCtl{answers: map[string]string{
397 "list-units": "carlos-instance@a.example.com.service loaded active running carlos exec-backed instance a.example.com\n" +
398 "carlos-instance-sidecar@a.example.com.service loaded inactive dead ...\n" +
399 "carlos-instance-seed@_aaaa\\x2d0123456789abcdef.service loaded activating start ...\n",
400 "list-jobs": "12 carlos-instance@b.example.com.service start waiting\n",
401 "show -p ActiveState --value": "inactive\n",
402 }}
403 s, c := newServer(t, ctl)
404 s.cgroupWaitBudget = time.Second
405 resp := post(t, c, "/sweep")
406 if resp.StatusCode != 200 {
407 t.Fatalf("status %d", resp.StatusCode)
408 }
409 var out SweepResponse
410 if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
411 t.Fatal(err)
412 }
413 got := strings.Join(out.Units, " ")
414 for _, want := range []string{"carlos-instance@a.example.com.service", "carlos-instance-seed@_aaaa\\x2d0123456789abcdef.service", "carlos-instance@b.example.com.service"} {
415 if !strings.Contains(got, want) {
416 t.Errorf("sweep did not stop %s: %v", want, out.Units)
417 }
418 }
419 if strings.Contains(got, "sidecar@a") {
420 t.Errorf("sweep stopped an inactive unit with no job: %v", out.Units)
421 }
422 if len(out.Failed) != 0 {
423 t.Errorf("nothing should have failed: %v", out.Failed)
424 }
425 calls := ctl.joined()
426 if !strings.Contains(calls, "stop carlos-instance@a.example.com.service") || !strings.Contains(calls, "stop carlos-instance@b.example.com.service") {
427 t.Errorf("stops not issued:\n%s", calls)
428 }
429}
430
431// TestSweepStopsOthersAndReportsAStuckUnitAsFailed proves a sweep never
432// aborts on the first unit whose cgroup won't empty: the other targets are
433// still stopped and reported swept, and the stuck one is reported failed
434// rather than losing the whole response (codex review fix round 2 item 3).
435func TestSweepStopsOthersAndReportsAStuckUnitAsFailed(t *testing.T) {
436 ctl := &fakeCtl{answers: map[string]string{
437 "list-units": "carlos-instance@a.example.com.service loaded active running carlos exec-backed instance a.example.com\n" +
438 "carlos-instance@stuck.example.com.service loaded active running carlos exec-backed instance stuck.example.com\n",
439 "list-jobs": "",
440 "show -p ActiveState --value": "inactive\n",
441 }}
442 s, c := newServer(t, ctl)
443 s.cgroupWaitBudget = 200 * time.Millisecond
444 ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-stuck.example.com.slice", "carlos-instance@stuck.example.com.service", "cgroup.events")
445 if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
446 t.Fatal(err)
447 }
448 if err := os.WriteFile(ev, []byte("populated 1\n"), 0o644); err != nil {
449 t.Fatal(err)
450 }
451 resp := post(t, c, "/sweep")
452 if resp.StatusCode != 502 {
453 t.Fatalf("status %d, want 502 while a unit is stuck", resp.StatusCode)
454 }
455 var out SweepResponse
456 if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
457 t.Fatal(err)
458 }
459 got := strings.Join(out.Units, " ")
460 if !strings.Contains(got, "carlos-instance@a.example.com.service") {
461 t.Errorf("the other unit was not stopped and reported: %v", out.Units)
462 }
463 if strings.Contains(got, "stuck") {
464 t.Errorf("the stuck unit must not be reported as swept: %v", out.Units)
465 }
466 if _, ok := out.Failed["carlos-instance@stuck.example.com.service"]; !ok {
467 t.Errorf("the stuck unit is missing from failed: %v", out.Failed)
468 }
469 calls := ctl.joined()
470 if !strings.Contains(calls, "stop carlos-instance@a.example.com.service") || !strings.Contains(calls, "stop carlos-instance@stuck.example.com.service") {
471 t.Errorf("both units were not stopped:\n%s", calls)
472 }
473}
474
475// TestReadyBeforeTheFirstProbeNamesTheReason pins that /ready answers with
476// an explicit "not yet run" line before RunProbe has ever completed, not
477// an empty Lines that could be misread as every check having passed
478// (final review, plan 2, item 6).
479func TestReadyBeforeTheFirstProbeNamesTheReason(t *testing.T) {
480 ctl := &fakeCtl{}
481 _, c := newServer(t, ctl)
482 resp, err := c.Get("http://unix/ready")
483 if err != nil || resp.StatusCode != 503 {
484 t.Fatalf("status: %v, %v", resp, err)
485 }
486 var got ProbeReport
487 if err := json.NewDecoder(resp.Body).Decode(&got); err != nil {
488 t.Fatal(err)
489 }
490 if got.Ready || len(got.Lines) != 1 || got.Lines[0].Check != "probe" || got.Lines[0].Verdict != "fail" || got.Lines[0].Detail != "not yet run" {
491 t.Errorf("body = %+v", got)
492 }
493}
494
495func TestReadyReflectsTheProbeVerdict(t *testing.T) {
496 ctl := &fakeCtl{}
497 s, c := newServer(t, ctl)
498 // The fake "start carlos-instance-probe.service" writes the report the
499 // unit's StandardOutput=truncate: would.
500 report := ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "pid1", Verdict: "fail", Detail: "pid=4242"}}}
501 ctl.answers = map[string]string{}
502 ctl.fail = map[string]error{}
503 writeReport := func(r ProbeReport) {
504 b, _ := json.Marshal(r)
505 if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {
506 t.Fatal(err)
507 }
508 }
509 writeReport(report)
510 s.RunProbe(context.Background())
511 resp, err := c.Get("http://unix/ready")
512 if err != nil || resp.StatusCode != 503 {
513 t.Fatalf("not-ready: %v %v", resp, err)
514 }
515 var got ProbeReport
516 if err := json.NewDecoder(resp.Body).Decode(&got); err != nil || got.Ready || len(got.Lines) != 1 {
517 t.Errorf("body = %+v, %v", got, err)
518 }
519 writeReport(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})
520 s.RunProbe(context.Background())
521 if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
522 t.Fatalf("ready: %v %v", resp, err)
523 }
524 if !strings.Contains(ctl.joined(), "start carlos-instance-probe.service") {
525 t.Errorf("probe unit not started: %s", ctl.joined())
526 }
527 // A missing fixture is never a pass.
528 s.FixtureOK = func() error { return errors.New("probe fixture absent") }
529 s.RunProbe(context.Background())
530 if resp, _ := c.Get("http://unix/ready"); resp.StatusCode != 503 {
531 t.Errorf("fixture absent yet ready")
532 }
533}
534
535func TestPeerCheckClosesTheConnectionWithoutAResponse(t *testing.T) {
536 ctl := &fakeCtl{}
537 _, c := newServer(t, ctl, func(s *Server) {
538 s.PeerOK = func(net.Conn) error { return errors.New("not the agent") }
539 })
540 _, err := c.Get("http://unix/ready")
541 if err == nil {
542 t.Fatal("a refused peer got an HTTP response")
543 }
544 if ctl.joined() != "" {
545 t.Errorf("systemctl ran for a refused peer: %s", ctl.joined())
546 }
547}
548
549func TestWaitCgroupEmptyTreatsAnAbsentCgroupAsEmpty(t *testing.T) {
550 root := t.TempDir()
551 if err := WaitCgroupEmpty(context.Background(), root, "carlos-instances-h.slice", "carlos-instance@h.service", 10*time.Millisecond); err != nil {
552 t.Errorf("absent cgroup: %v", err)
553 }
554}
555
556// fakeRun answers inspectMountpoints' unshare invocations by the
557// mountpoint named in the command: the listing under each raw directory,
558// or an exec failure.
559type fakeRun struct {
560 listings map[string]string // "/data/instances" -> ls -A output
561 err error
562 calls []string
563}
564
565func (f *fakeRun) run(_ context.Context, args ...string) (string, error) {
566 line := strings.Join(args, " ")
567 f.calls = append(f.calls, line)
568 if f.err != nil {
569 return "", f.err
570 }
571 for where, out := range f.listings {
572 if strings.Contains(line, "ls -A "+where) {
573 return out, nil
574 }
575 }
576 return "", nil
577}
578
579func probeOK(t *testing.T, s *Server) {
580 t.Helper()
581 b, _ := json.Marshal(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})
582 if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {
583 t.Fatal(err)
584 }
585 s.RunProbe(context.Background())
586}
587
588// Both raw directories empty: two ok lines, and readiness follows the
589// probe alone.
590func TestMountpointInspectionCleanIsReady(t *testing.T) {
591 run := &fakeRun{}
592 s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })
593 s.InspectMountpoints(context.Background())
594 probeOK(t, s)
595 if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
596 t.Fatalf("ready = %v %v, want 204", resp, err)
597 }
598 rep, ready := s.Ready()
599 if !ready || len(rep.Lines) != 3 || rep.Lines[0].Check != "mountpoint-data-instances" || rep.Lines[0].Verdict != "ok" || rep.Lines[1].Check != "mountpoint-run-carlos-instances" {
600 t.Fatalf("report = %+v", rep)
601 }
602 if len(run.calls) != 2 || !strings.HasPrefix(run.calls[0], "unshare -m --propagation private sh -c umount /data/instances") {
603 t.Errorf("calls = %q", run.calls)
604 }
605}
606
607// Files under the raw /data/instances: the report carries a failing
608// mountpoint line and /ready is 503 even after a passing probe, on every
609// probe after it.
610func TestMountpointHeldFilesBlockReadiness(t *testing.T) {
611 run := &fakeRun{listings: map[string]string{"/data/instances": "a.example.com\nb.example.com\n"}}
612 s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })
613 s.InspectMountpoints(context.Background())
614 for i := 0; i < 2; i++ {
615 probeOK(t, s)
616 resp, err := c.Get("http://unix/ready")
617 if err != nil || resp.StatusCode != 503 {
618 t.Fatalf("probe %d: ready = %v %v, want 503", i, resp, err)
619 }
620 var got ProbeReport
621 if err := json.NewDecoder(resp.Body).Decode(&got); err != nil || got.Ready {
622 t.Fatalf("body = %+v %v", got, err)
623 }
624 l := got.Lines[0]
625 if l.Check != "mountpoint-data-instances" || l.Verdict != "fail" || !strings.Contains(l.Detail, "mountpoint held files while unmounted: 2 entries under /data/instances") {
626 t.Errorf("line = %+v", l)
627 }
628 if got.Lines[1].Verdict != "ok" || got.Lines[2].Check != "pid1" {
629 t.Errorf("lines = %+v", got.Lines)
630 }
631 }
632}
633
634// unshare cannot run: both lines are unverified, in the report, and
635// readiness follows the probe alone.
636func TestMountpointInspectionUnverifiedDoesNotBlock(t *testing.T) {
637 run := &fakeRun{err: errors.New("exec: \"unshare\": executable file not found in $PATH")}
638 s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })
639 s.InspectMountpoints(context.Background())
640 probeOK(t, s)
641 if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
642 t.Fatalf("ready = %v %v, want 204", resp, err)
643 }
644 rep, _ := s.Ready()
645 if len(rep.Lines) != 3 || rep.Lines[0].Verdict != "unverified" || !strings.Contains(rep.Lines[0].Detail, "unshare") {
646 t.Fatalf("report = %+v", rep)
647 }
648 // A failing probe is still a failing probe.
649 b, _ := json.Marshal(ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "pid1", Verdict: "fail"}}})
650 os.WriteFile(s.ProbeOut, b, 0o644)
651 s.RunProbe(context.Background())
652 if resp, _ := c.Get("http://unix/ready"); resp.StatusCode != 503 {
653 t.Error("a failing probe read as ready")
654 }
655}
656
657// Discussion #256: carlos-instance-control.service ran the previous
658// release's binary for three weeks and served the new release's units.
659// /ready now asks, on every call, whether this process still runs the
660// installed binary, and a mismatch is a failing line whatever the probe
661// said; it names the restart that clears it.
662func TestReadyRefusesWhenTheRunningBinaryIsNotTheInstalledOne(t *testing.T) {
663 ctl := &fakeCtl{}
664 stale := errors.New("instance control is running a binary that is no longer /opt/carlos/carlos; restart carlos-instance-control.service")
665 var exeErr error
666 s, c := newServer(t, ctl, func(s *Server) {
667 s.ExeCurrent = func() error { return exeErr }
668 s.Build = "abc1234"
669 })
670 b, _ := json.Marshal(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})
671 if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {
672 t.Fatal(err)
673 }
674 s.RunProbe(context.Background())
675 resp, err := c.Get("http://unix/ready")
676 if err != nil || resp.StatusCode != 204 {
677 t.Fatalf("current binary: %v %v", resp, err)
678 }
679 if got := resp.Header.Get(BuildHeader); got != "abc1234" {
680 t.Fatalf("%s = %q, want the service's build", BuildHeader, got)
681 }
682 exeErr = stale
683 resp, err = c.Get("http://unix/ready")
684 if err != nil || resp.StatusCode != 503 {
685 t.Fatalf("stale binary: %v %v", resp, err)
686 }
687 if got := resp.Header.Get(BuildHeader); got != "abc1234" {
688 t.Fatalf("%s on a 503 = %q, want the service's build", BuildHeader, got)
689 }
690 var got ProbeReport
691 if err := json.NewDecoder(resp.Body).Decode(&got); err != nil {
692 t.Fatal(err)
693 }
694 if got.Ready || len(got.Lines) != 2 || got.Lines[1].Check != "binary" || got.Lines[1].Verdict != "fail" || !strings.Contains(got.Lines[1].Detail, "restart carlos-instance-control.service") {
695 t.Fatalf("body = %+v", got)
696 }
697 // The stored report is not what grew: the next call carries one
698 // binary line, not two.
699 resp, _ = c.Get("http://unix/ready")
700 got = ProbeReport{}
701 _ = json.NewDecoder(resp.Body).Decode(&got)
702 if len(got.Lines) != 2 {
703 t.Fatalf("second call = %+v, want the probe line and one binary line", got)
704 }
705 exeErr = nil
706 if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
707 t.Fatalf("after the restart: %v %v", resp, err)
708 }
709}
710
711func TestStopResetAndSweepCoverTheCmdRole(t *testing.T) {
712 ctl := &fakeCtl{answers: map[string]string{
713 "list-units": "carlos-instance-cmd@a.example.com.service loaded active exited carlos exec-backed instance a.example.com (cmd)\n",
714 "show -p ActiveState --value": "inactive\n",
715 }}
716 s, c := newServer(t, ctl)
717 s.cgroupWaitBudget = time.Second
718 if resp := post(t, c, "/stop?host=a.example.com&role=cmd"); resp.StatusCode != http.StatusNoContent {
719 t.Fatalf("stop status %d", resp.StatusCode)
720 }
721 if resp := post(t, c, "/reset?host=a.example.com&role=cmd"); resp.StatusCode/100 != 2 {
722 t.Fatalf("reset status %d", resp.StatusCode)
723 }
724 if resp := post(t, c, "/sweep"); resp.StatusCode != http.StatusOK {
725 t.Fatalf("sweep status %d", resp.StatusCode)
726 }
727 calls := ctl.joined()
728 for _, want := range []string{
729 "stop carlos-instance-cmd@a.example.com.service",
730 "reset-failed carlos-instance-cmd@a.example.com.service",
731 "carlos-instance-cmd@*.service",
732 } {
733 if !strings.Contains(calls, want) {
734 t.Errorf("missing %q in\n%s", want, calls)
735 }
736 }
737}
738