amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package instancectl
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "sync"
12 "testing"
13 "time"
14
15 "github.com/carlosframework/platform/internal/cgroup"
16)
17
18const genHost = "bill-2.example.com"
19
20var genUnit = GenUnitName(genHost, 7)
21
22// genShow is startGen's one look at systemd: the unit's drop-in lines and
23// its host slice's tier, in one call.
24var genShow = "show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax,CPUWeight,CPUQuotaPerSecUSec,IOWeight " + genUnit + " " + SliceName(genHost)
25
26// genSliceAtTier is that call's second block for a slice already at the
27// default tier.
28const genSliceAtTier = "\nSlice=carlos.slice\nNeedDaemonReload=no\nMemoryMax=536870912\nMemoryHigh=483183819\nMemorySwapMax=536870912\nCPUWeight=100\nCPUQuotaPerSecUSec=1s\nIOWeight=100\n"
29
30// genTierShow is systemd's answer for the default tier's memory lines.
31const genTierShow = "MemoryMax=536870912\nMemoryHigh=483183819\nMemorySwapMax=536870912\n"
32
33func genServer(t *testing.T, ctl *fakeCtl) (*Server, string, string) {
34 t.Helper()
35 dropins, exe := t.TempDir(), t.TempDir()
36 s, sock := serveOnSocket(t, ctl, func(s *Server) { s.DropInDir = dropins; s.ExeRoot = exe })
37 return s, sock, dropins
38}
39
40func genCtl() *fakeCtl {
41 return &fakeCtl{answers: map[string]string{
42 "show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax": "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow,
43 "show -p InvocationID --value": "abc123\n",
44 "list-units": "",
45 "list-jobs": "",
46 "show -p ActiveState --value": "inactive\n",
47 }}
48}
49
50func TestGenDropInText(t *testing.T) {
51 want := `# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).
52[Service]
53Slice=carlos-instances-bill\x2d2.example.com.slice
54BindPaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
55ReadWritePaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
56WorkingDirectory=/data/instances/bill-2.example.com
57BindReadOnlyPaths=/run/carlos-control/exe/bill-2.example.com@7/app:/opt/carlos/app /run/carlos-control/exe/bill-2.example.com@7/live:/opt/carlos/live
58StandardOutput=file:/run/carlos/logs/bill-2.example.com@7.run.sock
59MemoryMax=536870912
60MemoryHigh=483183819
61MemorySwapMax=536870912
62`
63 if got := GenDropIn(genHost, 7, cgroup.TierDefault); got != want {
64 t.Errorf("GenDropIn =\n%s\nwant\n%s", got, want)
65 }
66}
67
68// The wake path's case: a generation's unit is new, so systemd reads its
69// drop-in on the first reference and the box-wide reload is skipped.
70func TestGenStartWritesTheDropInChecksItThenStartsWithoutAReload(t *testing.T) {
71 ctl := genCtl()
72 s, sock, dropins := genServer(t, ctl)
73 inv, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
74 if err != nil || inv != "abc123" {
75 t.Fatalf("StartGen = %q, %v", inv, err)
76 }
77 raw, err := os.ReadFile(filepath.Join(dropins, genUnit+".d", DropInName))
78 if err != nil || string(raw) != GenDropIn(genHost, 7, cgroup.TierDefault) {
79 t.Fatalf("drop-in = %q, %v", raw, err)
80 }
81 if target, err := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 7), "app")); err != nil || target != "/opt/carlos/live/"+genHost {
82 t.Fatalf("exe app link = %q, %v", target, err)
83 }
84 want := strings.Join([]string{
85 genShow,
86 `set-property --runtime carlos-instances-bill\x2d2.example.com.slice MemoryMax=536870912 MemoryHigh=483183819 MemorySwapMax=536870912 CPUWeight=100 CPUQuota=100% IOWeight=100`,
87 "start " + genUnit,
88 "show -p InvocationID --value " + genUnit,
89 }, "\n")
90 if got := ctl.joined(); got != want {
91 t.Errorf("calls =\n%s\nwant\n%s", got, want)
92 }
93}
94
95// staleUntilReload makes systemd hold a stale copy of the unit (answer)
96// until a daemon-reload, after which the drop-in reads as applied.
97func staleUntilReload(ctl *fakeCtl, answer string) {
98 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = answer
99 ctl.before = map[string]func(){"daemon-reload": func() {
100 ctl.mu.Lock()
101 defer ctl.mu.Unlock()
102 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow
103 }}
104}
105
106// systemd still holding the unit loaded from older files says so with
107// NeedDaemonReload, or shows a slice the drop-in does not name. Either way
108// the start reloads, checks again, and only then starts.
109func TestGenStartReloadsWhenSystemdHoldsAStaleCopy(t *testing.T) {
110 for name, answer := range map[string]string{
111 "files changed since load": "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=yes\n" + genTierShow,
112 "loaded without drop-in": `Slice=system-carlos\x2dinstance\x2dgen.slice` + "\nNeedDaemonReload=no\n" + genTierShow,
113 "show failed to answer": "",
114 } {
115 t.Run(name, func(t *testing.T) {
116 ctl := genCtl()
117 staleUntilReload(ctl, answer)
118 _, sock, _ := genServer(t, ctl)
119 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
120 t.Fatal(err)
121 }
122 want := strings.Join([]string{
123 genShow,
124 "daemon-reload",
125 genShow,
126 }, "\n")
127 if got := ctl.joined(); !strings.HasPrefix(got, want+"\n") || !strings.Contains(got, "\nstart "+genUnit) {
128 t.Errorf("calls =\n%s\nwant them to begin\n%s\nand then start", got, want)
129 }
130 })
131 }
132}
133
134// systemd's change check is by mtime, so a reused generation still loaded
135// with an older tier, whose rewritten drop-in is not newer (a clock that
136// went back), reads NeedDaemonReload=no. The drop-in's other lines follow
137// from the unit's name; its memory lines are checked against the tier.
138func TestGenStartReloadsWhenSystemdHoldsAnotherTier(t *testing.T) {
139 ctl := genCtl()
140 staleUntilReload(ctl, "Slice="+SliceName(genHost)+"\nNeedDaemonReload=no\nMemoryMax=1073741824\nMemoryHigh=966367641\nMemorySwapMax=1073741824\n")
141 _, sock, _ := genServer(t, ctl)
142 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
143 t.Fatal(err)
144 }
145 if got := ctl.joined(); !strings.Contains(got, "daemon-reload\n") {
146 t.Errorf("started on systemd's stale tier without a reload:\n%s", got)
147 }
148}
149
150// A reload that leaves systemd still saying its files changed proves
151// nothing about what it loaded: refuse rather than start on a guess.
152func TestGenStartRefusesWhenTheReloadDidNotSettle(t *testing.T) {
153 ctl := genCtl()
154 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=yes\n" + genTierShow
155 s, sock, dropins := genServer(t, ctl)
156 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
157 t.Fatal("started a generation systemd still held a stale copy of")
158 }
159 if strings.Contains(ctl.joined(), "start "+genUnit) {
160 t.Errorf("the unit was started:\n%s", ctl.joined())
161 }
162 if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
163 t.Errorf("files left behind: drop-in %v exe %v", dropIn, exe)
164 }
165}
166
167// A successor's start during an overlap: the slice holds the incumbent,
168// and its raised limit is the swap's. The start leaves it alone.
169func TestGenStartLeavesAPopulatedSliceAlone(t *testing.T) {
170 ctl := genCtl()
171 s, sock, _ := genServer(t, ctl)
172 events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
173 if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
174 t.Fatal(err)
175 }
176 if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil {
177 t.Fatal(err)
178 }
179 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
180 t.Fatal(err)
181 }
182 if strings.Contains(ctl.joined(), "set-property") {
183 t.Errorf("a successor's start touched the live slice:\n%s", ctl.joined())
184 }
185}
186
187// Review focus 1, nested: an empty slice is reset to the tier.
188func TestGenStartResetsAnEmptySlice(t *testing.T) {
189 ctl := genCtl()
190 s, sock, _ := genServer(t, ctl)
191 events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
192 if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
193 t.Fatal(err)
194 }
195 if err := os.WriteFile(events, []byte("populated 0\nfrozen 0\n"), 0o644); err != nil {
196 t.Fatal(err)
197 }
198 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
199 t.Fatal(err)
200 }
201 if !strings.Contains(ctl.joined(), "set-property --runtime "+SliceName(genHost)+" MemoryMax=536870912") {
202 t.Errorf("a cold start left the slice as it was:\n%s", ctl.joined())
203 }
204}
205
206// Review focus 5: a drop-in systemd did not apply means the default slice
207// and an empty /data. Refuse, and leave nothing behind.
208func TestGenStartRefusesWhenTheDropInDidNotApply(t *testing.T) {
209 ctl := genCtl()
210 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = `Slice=system-carlos\x2dinstance\x2dgen.slice` + "\nNeedDaemonReload=no\n" + genTierShow
211 s, sock, dropins := genServer(t, ctl)
212 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
213 t.Fatal("started a generation whose drop-in did not apply")
214 }
215 if strings.Contains(ctl.joined(), "start "+genUnit) {
216 t.Errorf("the unit was started:\n%s", ctl.joined())
217 }
218 if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
219 t.Errorf("drop-in left behind: %v", err)
220 }
221 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
222 t.Errorf("exe dir left behind: %v", err)
223 }
224}
225
226func TestGenStartRefusedWithoutADropInDir(t *testing.T) {
227 ctl := genCtl()
228 _, sock := serveOnSocket(t, ctl)
229 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil || !strings.Contains(err.Error(), "501") {
230 t.Fatalf("StartGen without DropInDir: %v, want a 501", err)
231 }
232 if ctl.joined() != "" {
233 t.Errorf("systemctl was called:\n%s", ctl.joined())
234 }
235}
236
237func TestGenParametersRefusedWhereTheyDoNotBelong(t *testing.T) {
238 ctl := genCtl()
239 _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
240 for _, path := range []string{
241 "/start?host=" + genHost + "&role=sidecar&memory_max=536870912&gen=7",
242 "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=0",
243 "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=07",
244 "/seed?host=" + genHost + "&memory_max=536870912&gen=7",
245 "/stop?host=" + genHost + "&role=run&kill=1",
246 "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=7&kill=1",
247 } {
248 if resp := post(t, c, path); resp.StatusCode != 400 {
249 t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
250 }
251 }
252 if ctl.joined() != "" {
253 t.Errorf("systemctl was called:\n%s", ctl.joined())
254 }
255}
256
257func TestGenStopWithKillKillsStopsWaitsAndRemovesFiles(t *testing.T) {
258 ctl := genCtl()
259 s, sock, dropins := genServer(t, ctl)
260 if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
261 t.Fatal(err)
262 }
263 if err := s.ensureGenExe(genHost, 7); err != nil {
264 t.Fatal(err)
265 }
266 if err := NewClient(sock).StopGen(context.Background(), genHost, 7, true); err != nil {
267 t.Fatal(err)
268 }
269 if want := "kill --signal=SIGKILL " + genUnit + "\nstop " + genUnit + "\nreset-failed " + genUnit; ctl.joined() != want {
270 t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
271 }
272 if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
273 t.Errorf("drop-in survived the stop: %v", err)
274 }
275 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
276 t.Errorf("exe dir survived the stop: %v", err)
277 }
278}
279
280func TestParentSetsOnlyTheSliceMemory(t *testing.T) {
281 ctl := genCtl()
282 _, sock := serveOnSocket(t, ctl)
283 c := NewClient(sock)
284 if err := c.Parent(context.Background(), genHost, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault).MemoryMax); err != nil {
285 t.Fatal(err)
286 }
287 want := `set-property --runtime carlos-instances-bill\x2d2.example.com.slice MemoryMax=1073741824 MemoryHigh=966367638 MemorySwapMax=1073741824`
288 if ctl.joined() != want {
289 t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
290 }
291 if err := c.Parent(context.Background(), genHost, 64<<20); err == nil {
292 t.Error("a parent limit that is no tier sum was accepted")
293 }
294}
295
296func TestSweepStopsGenerationUnitsAndRemovesTheirFiles(t *testing.T) {
297 ctl := genCtl()
298 ctl.answers["list-units"] = genUnit + " loaded active running carlos exec-backed instance generation\n"
299 s, sock, dropins := genServer(t, ctl)
300 if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
301 t.Fatal(err)
302 }
303 if err := s.ensureGenExe(genHost, 7); err != nil {
304 t.Fatal(err)
305 }
306 out, err := NewClient(sock).Sweep(context.Background())
307 if err != nil || len(out.Units) != 1 || out.Units[0] != genUnit {
308 raw, _ := json.Marshal(out)
309 t.Fatalf("sweep = %s, %v", raw, err)
310 }
311 if !strings.Contains(ctl.joined(), UnitPrefixGen+"*.service") {
312 t.Errorf("list-units did not ask for generation units:\n%s", ctl.joined())
313 }
314 if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
315 t.Errorf("drop-in survived the sweep: %v", err)
316 }
317 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
318 t.Errorf("exe dir survived the sweep: %v", err)
319 }
320}
321
322// A start that failed leaves nothing behind: the unit is stopped and the
323// drop-in and exe layout are removed.
324func TestGenStartFailureStopsTheUnitAndRemovesTheFiles(t *testing.T) {
325 ctl := genCtl()
326 ctl.fail = map[string]error{"start ": errors.New("boom")}
327 s, sock, dropins := genServer(t, ctl)
328 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
329 t.Fatal("a failed start was reported as started")
330 }
331 if !strings.Contains(ctl.joined(), "stop "+genUnit) {
332 t.Errorf("the failed unit was not stopped:\n%s", ctl.joined())
333 }
334 if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
335 t.Errorf("drop-in left behind: %v", err)
336 }
337 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
338 t.Errorf("exe dir left behind: %v", err)
339 }
340}
341
342func TestGenResetResetsTheGenerationUnitOnly(t *testing.T) {
343 ctl := genCtl()
344 _, sock := serveOnSocket(t, ctl)
345 if err := NewClient(sock).ResetGen(context.Background(), genHost, 7); err != nil {
346 t.Fatal(err)
347 }
348 if want := "reset-failed " + genUnit; ctl.joined() != want {
349 t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
350 }
351}
352
353func TestGenAndParentInputsAreValidated(t *testing.T) {
354 ctl := genCtl()
355 _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
356 for _, path := range []string{
357 "/start?host=" + genHost + "&role=run&memory_max=12345&gen=7",
358 "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=",
359 "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=-1",
360 "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=18446744073709551616",
361 "/stop?host=" + genHost + "&role=run&gen=7&kill=true",
362 "/stop?host=" + genHost + "&role=seed&gen=7",
363 "/reset?host=" + genHost + "&role=run&gen=0",
364 "/reset?host=" + genHost + "&role=sidecar&gen=7",
365 "/parent?host=not_a_host&memory_max=536870912",
366 "/parent?host=" + genHost,
367 "/parent?host=" + genHost + "&memory_max=-536870912",
368 "/parent?host=" + genHost + "&memory_max=0",
369 } {
370 if resp := post(t, c, path); resp.StatusCode != 400 {
371 t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
372 }
373 }
374 if ctl.joined() != "" {
375 t.Errorf("systemctl was called:\n%s", ctl.joined())
376 }
377}
378
379// genFilesPresent reports whether the generation's drop-in directory and
380// its executable layout are on disk.
381func genFilesPresent(t *testing.T, s *Server, dropins string) (dropIn, exe bool) {
382 t.Helper()
383 _, err1 := os.Stat(filepath.Join(dropins, genUnit+".d"))
384 _, err2 := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7)))
385 return err1 == nil, err2 == nil
386}
387
388// populateGenCgroup makes the generation unit's cgroup report processes,
389// so no stop can prove it empty.
390func populateGenCgroup(t *testing.T, s *Server) {
391 t.Helper()
392 events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), genUnit, "cgroup.events")
393 if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
394 t.Fatal(err)
395 }
396 if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil {
397 t.Fatal(err)
398 }
399}
400
401// systemctl start succeeded, then the invocation lookup failed: the unit
402// may be running. It is killed and proven empty before its files go, and
403// the answer is an ordinary start failure.
404func TestGenStartWhoseInvocationLookupFailsIsKilledBeforeItsFilesGo(t *testing.T) {
405 ctl := genCtl()
406 ctl.fail = map[string]error{"show -p InvocationID": errors.New("bus timeout")}
407 s, sock, dropins := genServer(t, ctl)
408 _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
409 if err == nil || errors.Is(err, ErrGenUnproven) {
410 t.Fatalf("StartGen = %v, want an ordinary start failure", err)
411 }
412 if want := "start " + genUnit + "\nshow -p InvocationID --value " + genUnit + "\nkill --signal=SIGKILL " + genUnit + "\nstop " + genUnit + "\nreset-failed " + genUnit; !strings.HasSuffix(ctl.joined(), want) {
413 t.Errorf("calls =\n%s\nwant them to end\n%s", ctl.joined(), want)
414 }
415 if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
416 t.Errorf("files left after a proven stop: drop-in %v exe %v", dropIn, exe)
417 }
418}
419
420// The same, but the generation's cgroup never empties: termination is
421// unproven, so the files stay and the answer says so.
422func TestGenStartWhoseCleanupIsUnprovenKeepsTheFiles(t *testing.T) {
423 ctl := genCtl()
424 ctl.fail = map[string]error{"show -p InvocationID": errors.New("bus timeout")}
425 s, sock, dropins := genServer(t, ctl)
426 s.cgroupWaitBudget = 100 * time.Millisecond
427 populateGenCgroup(t, s)
428 _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
429 if !errors.Is(err, ErrGenUnproven) {
430 t.Fatalf("StartGen = %v, want ErrGenUnproven", err)
431 }
432 if dropIn, exe := genFilesPresent(t, s, dropins); !dropIn || !exe {
433 t.Errorf("files removed while termination is unproven: drop-in %v exe %v", dropIn, exe)
434 }
435}
436
437// A failed systemctl start whose cleanup stop also fails: nothing proves
438// the unit gone, so the files stay.
439func TestGenStartFailureWhoseStopFailsKeepsTheFiles(t *testing.T) {
440 ctl := genCtl()
441 ctl.fail = map[string]error{"start ": errors.New("boom"), "stop ": errors.New("stop timed out")}
442 s, sock, dropins := genServer(t, ctl)
443 _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
444 if !errors.Is(err, ErrGenUnproven) {
445 t.Fatalf("StartGen = %v, want ErrGenUnproven", err)
446 }
447 if dropIn, exe := genFilesPresent(t, s, dropins); !dropIn || !exe {
448 t.Errorf("files removed after a failed stop: drop-in %v exe %v", dropIn, exe)
449 }
450}
451
452// Nothing was started when the drop-in cannot be written: the exe layout
453// already made goes, and systemctl is never asked to start.
454func TestGenStartDropInWriteFailureRemovesTheFiles(t *testing.T) {
455 ctl := genCtl()
456 s, sock, dropins := genServer(t, ctl)
457 // A file where the unit's drop-in directory goes: MkdirAll fails.
458 if err := os.WriteFile(filepath.Join(dropins, genUnit+".d"), nil, 0o644); err != nil {
459 t.Fatal(err)
460 }
461 _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
462 if err == nil || errors.Is(err, ErrGenUnproven) {
463 t.Fatalf("StartGen = %v, want an ordinary failure", err)
464 }
465 if ctl.joined() != "" {
466 t.Errorf("systemctl was called:\n%s", ctl.joined())
467 }
468 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
469 t.Errorf("exe dir left behind: %v", err)
470 }
471}
472
473func TestGenStartDaemonReloadFailureRemovesTheFiles(t *testing.T) {
474 ctl := genCtl()
475 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=yes\n" + genTierShow
476 ctl.fail = map[string]error{"daemon-reload": errors.New("bus gone")}
477 s, sock, dropins := genServer(t, ctl)
478 _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
479 if err == nil || errors.Is(err, ErrGenUnproven) {
480 t.Fatalf("StartGen = %v, want an ordinary failure", err)
481 }
482 if strings.Contains(ctl.joined(), "start "+genUnit) {
483 t.Errorf("the unit was started:\n%s", ctl.joined())
484 }
485 if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
486 t.Errorf("files left behind: drop-in %v exe %v", dropIn, exe)
487 }
488}
489
490// A generation that stopped while nobody removed its files (its stop's
491// answer was lost, the agent restarted) is inactive, so the sweep has no
492// unit to stop. Its leftovers still go once systemd reports it inactive
493// and its cgroup is empty.
494func TestSweepRemovesTheFilesOfAnInactiveGeneration(t *testing.T) {
495 ctl := genCtl()
496 s, sock, dropins := genServer(t, ctl)
497 if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
498 t.Fatal(err)
499 }
500 if err := s.ensureGenExe(genHost, 7); err != nil {
501 t.Fatal(err)
502 }
503 out, err := NewClient(sock).Sweep(context.Background())
504 if err != nil || len(out.Units) != 0 {
505 t.Fatalf("sweep = %+v, %v", out, err)
506 }
507 if !strings.Contains(ctl.joined(), "show -p ActiveState --value "+genUnit) {
508 t.Errorf("the leftover unit's state was not checked:\n%s", ctl.joined())
509 }
510 if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
511 t.Errorf("leftovers of an inactive generation survived the sweep: drop-in %v exe %v", dropIn, exe)
512 }
513}
514
515// Task 15 (Astra IMPL-2-R2 residual 4): a generation can leave only its
516// executable directory under ExeRoot, when its drop-in went and the exe's
517// removal failed (removeGenFiles ignores errors). The sweep listed only
518// DropInDir, so that leftover stayed in /run for good. It now walks
519// ExeRoot too, with the same checks: the unit inactive or failed, its
520// cgroup empty, and no start of it under way. A host's own directory (no
521// '@') and a name that is not a generation key are never touched.
522func TestSweepRemovesAnExeOnlyGenerationLeftover(t *testing.T) {
523 ctl := genCtl()
524 s, sock, dropins := genServer(t, ctl)
525 if err := s.ensureGenExe(genHost, 7); err != nil {
526 t.Fatal(err)
527 }
528 if err := s.ensureHostExe(genHost); err != nil {
529 t.Fatal(err)
530 }
531 odd := []string{genHost + "@0", genHost + "@x", "@7", genHost + "@07"}
532 for _, name := range odd {
533 if err := os.Mkdir(filepath.Join(s.ExeRoot, name), 0o755); err != nil {
534 t.Fatal(err)
535 }
536 }
537 if _, err := NewClient(sock).Sweep(context.Background()); err != nil {
538 t.Fatal(err)
539 }
540 if !strings.Contains(ctl.joined(), "show -p ActiveState --value "+genUnit) {
541 t.Errorf("the exe-only leftover's unit state was not checked:\n%s", ctl.joined())
542 }
543 if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
544 t.Errorf("an exe-only leftover of an inactive generation survived the sweep: drop-in %v exe %v", dropIn, exe)
545 }
546 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err == nil {
547 t.Error("the generation's exe directory is still there")
548 }
549 if _, err := os.Lstat(filepath.Join(s.ExeRoot, genHost, "app")); err != nil {
550 t.Errorf("the host's own exe directory was touched: %v", err)
551 }
552 for _, name := range odd {
553 if _, err := os.Stat(filepath.Join(s.ExeRoot, name)); err != nil {
554 t.Errorf("%s, not a generation key, was removed: %v", name, err)
555 }
556 }
557}
558
559// Task 15 fix round 1 (d): a leftover the sweep cannot remove is logged as
560// not removed, with the removal's error. removeGenFiles dropped every
561// error, so the sweep logged "removed the files of" over an exe directory
562// still on disk.
563func TestSweepLogsAnExeLeftoverItCouldNotRemove(t *testing.T) {
564 if os.Geteuid() == 0 {
565 t.Skip("root removes from a read-only directory")
566 }
567 var (
568 mu sync.Mutex
569 logged []string
570 )
571 ctl := genCtl()
572 exe := t.TempDir()
573 s, _ := serveOnSocket(t, ctl, func(s *Server) {
574 s.DropInDir, s.ExeRoot = t.TempDir(), exe
575 s.Logf = func(f string, a ...any) {
576 mu.Lock()
577 logged = append(logged, fmt.Sprintf(f, a...))
578 mu.Unlock()
579 }
580 })
581 if err := s.ensureGenExe(genHost, 7); err != nil {
582 t.Fatal(err)
583 }
584 if err := os.Chmod(exe, 0o555); err != nil {
585 t.Fatal(err)
586 }
587 t.Cleanup(func() { _ = os.Chmod(exe, 0o755) })
588 s.sweepGenLeftovers(context.Background(), nil)
589 if _, err := os.Stat(filepath.Join(exe, GenKey(genHost, 7))); err != nil {
590 t.Fatalf("the fixture's exe directory went after all: %v", err)
591 }
592 mu.Lock()
593 all := strings.Join(logged, "\n")
594 mu.Unlock()
595 if strings.Contains(all, "removed the files of "+genUnit) {
596 t.Errorf("the sweep logged a removal that failed:\n%s", all)
597 }
598 if !strings.Contains(all, "could not remove the files of "+genUnit) || !strings.Contains(all, "permission denied") {
599 t.Errorf("the sweep did not log the failed removal and its error:\n%s", all)
600 }
601}
602
603// An exe-only leftover is kept while its unit is not proven gone (active,
604// or its cgroup populated) and while a start of that unit is preparing it:
605// the start lays the exe out before it writes the drop-in, so for a moment
606// a live start looks exactly like an exe-only leftover.
607func TestSweepKeepsAnExeOnlyGenerationNotProvenGone(t *testing.T) {
608 for name, setup := range map[string]func(*fakeCtl, *Server){
609 "active": func(c *fakeCtl, _ *Server) { c.answers["show -p ActiveState --value"] = "active\n" },
610 "populated": func(_ *fakeCtl, s *Server) { populateGenCgroup(t, s) },
611 "starting": func(_ *fakeCtl, s *Server) { s.markGenStarting(genUnit, true) },
612 } {
613 t.Run(name, func(t *testing.T) {
614 ctl := genCtl()
615 s, sock, _ := genServer(t, ctl)
616 setup(ctl, s)
617 if err := s.ensureGenExe(genHost, 7); err != nil {
618 t.Fatal(err)
619 }
620 if _, err := NewClient(sock).Sweep(context.Background()); err != nil {
621 t.Fatal(err)
622 }
623 if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err != nil {
624 t.Errorf("the exe of a generation not proven gone was removed: %v", err)
625 }
626 })
627 }
628}
629
630// Leftovers whose unit is not proven gone stay: one systemd reports
631// active, and one whose cgroup still holds processes.
632func TestSweepKeepsTheFilesOfAGenerationNotProvenGone(t *testing.T) {
633 for name, setup := range map[string]func(*fakeCtl, *Server){
634 "active": func(c *fakeCtl, _ *Server) { c.answers["show -p ActiveState --value"] = "active\n" },
635 "populated": func(_ *fakeCtl, s *Server) { populateGenCgroup(t, s) },
636 } {
637 t.Run(name, func(t *testing.T) {
638 ctl := genCtl()
639 s, sock, dropins := genServer(t, ctl)
640 setup(ctl, s)
641 if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
642 t.Fatal(err)
643 }
644 if err := s.ensureGenExe(genHost, 7); err != nil {
645 t.Fatal(err)
646 }
647 if _, err := NewClient(sock).Sweep(context.Background()); err != nil {
648 t.Fatal(err)
649 }
650 if dropIn, exe := genFilesPresent(t, s, dropins); !dropIn || !exe {
651 t.Errorf("files of a generation not proven gone were removed: drop-in %v exe %v", dropIn, exe)
652 }
653 })
654 }
655}
656
657// genLogServer is genServer with the server's log captured.
658func genLogServer(t *testing.T, ctl *fakeCtl) (*Server, string, *syncLog) {
659 t.Helper()
660 logs := &syncLog{}
661 dropins, exe := t.TempDir(), t.TempDir()
662 s, sock := serveOnSocket(t, ctl, func(s *Server) { s.DropInDir = dropins; s.ExeRoot = exe; s.Logf = logs.logf })
663 return s, sock, logs
664}
665
666type syncLog struct {
667 mu sync.Mutex
668 lines []string
669}
670
671func (l *syncLog) logf(format string, args ...any) {
672 l.mu.Lock()
673 defer l.mu.Unlock()
674 l.lines = append(l.lines, fmt.Sprintf(format, args...))
675}
676
677func (l *syncLog) text() string {
678 l.mu.Lock()
679 defer l.mu.Unlock()
680 return strings.Join(l.lines, "\n")
681}
682
683func TestStopGenResetsTheFailedStateAfterAProvenStop(t *testing.T) {
684 for _, kill := range []bool{false, true} {
685 t.Run(fmt.Sprintf("kill=%v", kill), func(t *testing.T) {
686 ctl := genCtl()
687 _, sock, _ := genLogServer(t, ctl)
688 if err := NewClient(sock).StopGen(context.Background(), genHost, 7, kill); err != nil {
689 t.Fatal(err)
690 }
691 want := "stop " + genUnit + "\nreset-failed " + genUnit
692 if kill {
693 want = "kill --signal=SIGKILL " + genUnit + "\n" + want
694 }
695 if ctl.joined() != want {
696 t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
697 }
698 })
699 }
700}
701
702// reset-failed is cosmetic: the stop succeeded, so the answer stays 204 and
703// the failure is logged; a unit that is no longer loaded is not logged.
704func TestStopGenAnswersNoContentWhenResetFailedFails(t *testing.T) {
705 for _, kill := range []bool{false, true} {
706 t.Run(fmt.Sprintf("kill=%v", kill), func(t *testing.T) {
707 ctl := genCtl()
708 ctl.fail = map[string]error{"reset-failed ": errors.New("bus timeout")}
709 s, sock, logs := genLogServer(t, ctl)
710 if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
711 t.Fatal(err)
712 }
713 if err := NewClient(sock).StopGen(context.Background(), genHost, 7, kill); err != nil {
714 t.Fatalf("StopGen = %v, want success", err)
715 }
716 if want := "instance-control: reset-failed " + genUnit + ": bus timeout"; !strings.Contains(logs.text(), want) {
717 t.Errorf("log =\n%s\nwant it to contain %q", logs.text(), want)
718 }
719 if _, err := os.Stat(filepath.Join(s.DropInDir, genUnit+".d")); !os.IsNotExist(err) {
720 t.Errorf("drop-in survived the stop: %v", err)
721 }
722 })
723 }
724}
725
726func TestStopGenIgnoresResetFailedOnAnUnloadedUnit(t *testing.T) {
727 ctl := genCtl()
728 ctl.fail = map[string]error{"reset-failed ": errors.New("Unit " + genUnit + " not loaded.")}
729 _, sock, logs := genLogServer(t, ctl)
730 if err := NewClient(sock).StopGen(context.Background(), genHost, 7, false); err != nil {
731 t.Fatal(err)
732 }
733 if strings.Contains(logs.text(), "reset-failed") {
734 t.Errorf("an unloaded unit was logged: %s", logs.text())
735 }
736}
737
738func TestStopGenWhoseStopFailsDoesNotResetFailed(t *testing.T) {
739 for _, kill := range []bool{false, true} {
740 t.Run(fmt.Sprintf("kill=%v", kill), func(t *testing.T) {
741 ctl := genCtl()
742 ctl.fail = map[string]error{"stop ": errors.New("stop timed out")}
743 _, sock, _ := genLogServer(t, ctl)
744 if err := NewClient(sock).StopGen(context.Background(), genHost, 7, kill); err == nil {
745 t.Fatal("StopGen succeeded though the stop failed")
746 }
747 if strings.Contains(ctl.joined(), "reset-failed") {
748 t.Errorf("reset-failed after a failed stop:\n%s", ctl.joined())
749 }
750 })
751 }
752}
753
754func TestStopGenWhoseCgroupNeverEmptiesDoesNotResetFailed(t *testing.T) {
755 ctl := genCtl()
756 s, sock, _ := genLogServer(t, ctl)
757 s.cgroupWaitBudget = 100 * time.Millisecond
758 populateGenCgroup(t, s)
759 if err := NewClient(sock).StopGen(context.Background(), genHost, 7, true); err == nil {
760 t.Fatal("StopGen succeeded though the cgroup never emptied")
761 }
762 if strings.Contains(ctl.joined(), "reset-failed") {
763 t.Errorf("reset-failed after an unproven kill:\n%s", ctl.joined())
764 }
765}
766
767func TestAbandonedGenStartResetsFailedOnlyAfterAProvenKill(t *testing.T) {
768 ctl := genCtl()
769 ctl.fail = map[string]error{"start ": errors.New("boom")}
770 _, sock, _ := genLogServer(t, ctl)
771 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil || errors.Is(err, ErrGenUnproven) {
772 t.Fatalf("StartGen = %v, want an ordinary start failure", err)
773 }
774 if want := "kill --signal=SIGKILL " + genUnit + "\nstop " + genUnit + "\nreset-failed " + genUnit; !strings.HasSuffix(ctl.joined(), want) {
775 t.Errorf("calls =\n%s\nwant them to end\n%s", ctl.joined(), want)
776 }
777
778 ctl = genCtl()
779 ctl.fail = map[string]error{"start ": errors.New("boom"), "stop ": errors.New("stop timed out")}
780 _, sock, _ = genLogServer(t, ctl)
781 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); !errors.Is(err, ErrGenUnproven) {
782 t.Fatalf("StartGen = %v, want ErrGenUnproven", err)
783 }
784 if strings.Contains(ctl.joined(), "reset-failed") {
785 t.Errorf("reset-failed after a failed kill:\n%s", ctl.joined())
786 }
787}
788
789// A cold start whose slice already carries the tier (it kept it from the
790// last generation: set-property --runtime lasts until a reboot) skips the
791// set-property: one systemctl call fewer on a wake. The same show read it.
792func TestGenStartSkipsTheTierWhenTheSliceAlreadyHasIt(t *testing.T) {
793 ctl := genCtl()
794 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow + genSliceAtTier
795 _, sock, _ := genServer(t, ctl)
796 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
797 t.Fatal(err)
798 }
799 if got := ctl.joined(); strings.Contains(got, "set-property") {
800 t.Errorf("the slice already had the tier but was set again:\n%s", got)
801 }
802}
803
804// Any one value off, or the slice missing from the answer, and the tier is
805// set as before.
806func TestGenStartSetsTheTierWhenTheSliceDiffers(t *testing.T) {
807 for name, block := range map[string]string{
808 "memory": strings.Replace(genSliceAtTier, "MemoryMax=536870912", "MemoryMax=1073741824", 1),
809 "cpu quota": strings.Replace(genSliceAtTier, "CPUQuotaPerSecUSec=1s", "CPUQuotaPerSecUSec=2s", 1),
810 "io weight": strings.Replace(genSliceAtTier, "IOWeight=100", "IOWeight=[not set]", 1),
811 "no slice block": "",
812 } {
813 t.Run(name, func(t *testing.T) {
814 ctl := genCtl()
815 ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow + block
816 _, sock, _ := genServer(t, ctl)
817 if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
818 t.Fatal(err)
819 }
820 if got := ctl.joined(); !strings.Contains(got, "set-property --runtime "+SliceName(genHost)) {
821 t.Errorf("the slice differs (%s) but the tier was not set:\n%s", name, got)
822 }
823 })
824 }
825}
826
827func TestCPUQuotaShowsAsSystemdShowsIt(t *testing.T) {
828 for pct, want := range map[int]string{50: "500ms", 100: "1s", 150: "1.500000s", 200: "2s", 250: "2.500000s"} {
829 if got := cpuQuotaShown(pct); got != want {
830 t.Errorf("cpuQuotaShown(%d) = %q, want %q (systemd 257)", pct, got, want)
831 }
832 }
833}
834