carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package activator
4
5import (
6 "context"
7 "encoding/json"
8 "errors"
9 "net"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "strconv"
14 "strings"
15 "sync"
16 "syscall"
17 "testing"
18 "time"
19
20 "github.com/carlosframework/platform/internal/cgroup"
21 "github.com/carlosframework/platform/internal/hostbox"
22 "github.com/carlosframework/platform/internal/hostclaim"
23 "github.com/carlosframework/platform/internal/hostguard"
24 "github.com/carlosframework/platform/internal/hostwrite"
25 "github.com/carlosframework/platform/internal/instancectl"
26 "github.com/carlosframework/platform/internal/instancefs"
27 "golang.org/x/sys/unix"
28)
29
30type fakeRoleRoot struct {
31 mu sync.Mutex
32 events []string
33 launches []hostbox.RoleLaunchRequest
34 stops []hostbox.RoleLaunchRequest
35 grant func(hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error)
36 stopErr error
37 capability string
38}
39
40func (f *fakeRoleRoot) Capability(context.Context) (string, error) { return f.capability, nil }
41
42func (f *fakeRoleRoot) Launch(_ context.Context, q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) {
43 f.mu.Lock()
44 f.events = append(f.events, "launch")
45 f.launches = append(f.launches, q)
46 grant := f.grant
47 f.mu.Unlock()
48 return grant(q)
49}
50
51func (f *fakeRoleRoot) Stop(_ context.Context, q hostbox.RoleLaunchRequest) error {
52 f.mu.Lock()
53 defer f.mu.Unlock()
54 f.events = append(f.events, "stop")
55 f.stops = append(f.stops, q)
56 return f.stopErr
57}
58
59func selfStart(t *testing.T) string {
60 t.Helper()
61 start, err := roleProcessStart(os.Getpid())
62 if err != nil {
63 t.Fatal(err)
64 }
65 return start
66}
67
68type rootLaunchFixture struct {
69 launch RoleLaunch
70 root *fakeRoleRoot
71 attempt hostclaim.WakeRetryAttempt
72 build func() *exec.Cmd
73 exe string
74 exeID hostbox.LaunchFile
75 launchDir string
76 runDir string
77 logSocket string
78 beforeRuns int
79}
80
81func newRootLaunchFixture(t *testing.T) *rootLaunchFixture {
82 t.Helper()
83 target := hostguard.FenceTarget{Region: "eu", Host: "tenant.test", Incarnation: "inc", Stamp: 2, Session: 3, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "ctrl", ClaimETag: "etag", DirectoryRevision: "7"}
84 work, err := (hostwrite.Config{}).Select(t.Context(), target.Host, "spawn")
85 if err != nil {
86 t.Fatal(err)
87 }
88 base := shortTempDir(t)
89 f := &rootLaunchFixture{launchDir: filepath.Join(base, "data"), runDir: filepath.Join(base, "run"), logSocket: filepath.Join(base, "logs", "t.run.sock"), exe: filepath.Join(base, "app")}
90 for _, d := range []string{f.launchDir, filepath.Dir(f.logSocket)} {
91 if err := os.MkdirAll(d, 0o700); err != nil {
92 t.Fatal(err)
93 }
94 }
95 if err := os.WriteFile(f.exe, []byte("build"), 0o555); err != nil {
96 t.Fatal(err)
97 }
98 var st unix.Stat_t
99 if err := unix.Stat(f.exe, &st); err != nil {
100 t.Fatal(err)
101 }
102 f.exeID = hostbox.LaunchFile{Device: uint64(st.Dev), Inode: st.Ino}
103 f.attempt = hostclaim.WakeRetryAttempt{ID: "attempt-1", InvocationID: "inv-1", LaunchInputs: "inputs-1", Session: 3, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "ctrl"}
104 start := selfStart(t)
105 f.root = &fakeRoleRoot{capability: hostbox.RoleLaunchProtocol}
106 f.root.grant = func(q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) {
107 return hostbox.RoleLaunchGrant{ReservationID: q.Reservation.ID, Unit: "carlos-instance@tenant.test.service", UnitInvocationID: "unit-1", PID: os.Getpid(), ProcessStart: start, Group: hostbox.LaunchGroup{RelativePath: q.Reservation.Group.RelativePath, Device: 3, Inode: 4, InvocationID: "unit-1"}}, nil
108 }
109 f.launch = RoleLaunch{Work: work, Target: target, Root: f.root, LaunchDir: f.launchDir, RunDir: f.runDir, LogSocket: f.logSocket, MemoryMax: cgroup.TierDefault.MemoryMax, Executable: func(context.Context, *exec.Cmd) (*os.File, error) { return os.Open(f.exe) }}
110 f.build = func() *exec.Cmd {
111 cmd := exec.Command("/opt/carlos/live/tenant.test", "--socket", filepath.Join(f.runDir, "app.sock"), "--db", filepath.Join(f.launchDir, "app.db"))
112 cmd.Env = []string{"CARLOS_ADMIN_TOKEN=secret", "A=1"}
113 return cmd
114 }
115 return f
116}
117
118func (f *rootLaunchFixture) before() error {
119 f.root.mu.Lock()
120 defer f.root.mu.Unlock()
121 f.root.events = append(f.root.events, "before")
122 f.beforeRuns++
123 return nil
124}
125
126func TestRootRoleLaunchCarriesTheReservation(t *testing.T) {
127 f := newRootLaunchFixture(t)
128 finished := 0
129 f.launch.RegisterProducer = func(_ context.Context, role string) (RoleProducer, error) {
130 if role != "main" {
131 t.Fatalf("producer role %q", role)
132 }
133 return RoleProducer{Context: t.Context(), Finish: func(error) { finished++ }}, nil
134 }
135 cmd, prepared, settle, producer, err := f.launch.Start(t.Context(), "main", f.attempt, f.build, f.before)
136 if err != nil {
137 t.Fatal(err)
138 }
139 if cmd == nil || cmd.Process == nil || cmd.Process.Pid != os.Getpid() || prepared != nil || settle == nil || producer.Finish == nil {
140 t.Fatalf("start returned cmd=%v prepared=%v settle=%v", cmd, prepared, settle != nil)
141 }
142 if got := strings.Join(f.root.events, ","); got != "before,launch" {
143 t.Fatalf("order=%s", got)
144 }
145 q := f.root.launches[0]
146 r := q.Reservation
147 if q.V != hostbox.RoleLaunchV || q.Op != hostbox.RoleLaunchStart || q.Executable != f.exe || q.MemoryMax != cgroup.TierDefault.MemoryMax || r == nil {
148 t.Fatalf("request=%+v", q)
149 }
150 want := hostbox.LaunchReservation{ID: "attempt-1:main", Role: "main", InvocationID: "inv-1", Inputs: "inputs-1", Target: f.launch.Target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup("tenant.test")}, Executable: f.exeID}
151 if *r != want {
152 t.Fatalf("reservation=%+v want %+v", *r, want)
153 }
154 raw, err := instancefs.ReadBeneath(f.launchDir, instancectl.LaunchFileName(instancectl.RoleRun))
155 if err != nil {
156 t.Fatal(err)
157 }
158 var file instancectl.Launch
159 if err := json.Unmarshal(raw, &file); err != nil {
160 t.Fatal(err)
161 }
162 if file.Schema != instancectl.LaunchSchema || file.Host != "tenant.test" || strings.Join(file.Argv, " ") != "/opt/carlos/live/tenant.test --socket "+filepath.Join(f.runDir, "app.sock")+" --db "+filepath.Join(f.launchDir, "app.db") || strings.Join(file.Env, ",") != "CARLOS_ADMIN_TOKEN=secret,A=1" {
163 t.Fatalf("launch file=%+v", file)
164 }
165 if info, err := os.Stat(filepath.Join(f.launchDir, instancectl.LaunchFileName(instancectl.RoleRun))); err != nil || info.Mode().Perm() != 0o600 {
166 t.Fatalf("launch file mode=%v %v", info, err)
167 }
168 // The unit's stdout socket listens before root starts it.
169 conn, err := net.Dial("unix", f.logSocket)
170 if err != nil {
171 t.Fatalf("log listener: %v", err)
172 }
173 conn.Close()
174 if err := settle(t.Context()); err != nil {
175 t.Fatal(err)
176 }
177 if len(f.root.stops) != 1 || f.root.stops[0].Op != hostbox.RoleLaunchStop || *f.root.stops[0].Reservation != want {
178 t.Fatalf("stops=%+v", f.root.stops)
179 }
180 if _, err := net.Dial("unix", f.logSocket); err == nil {
181 t.Fatal("log listener outlived the settled role")
182 }
183}
184
185func TestRootRoleLaunchRefusesShapesItCannotCarry(t *testing.T) {
186 for name, mutate := range map[string]func(*rootLaunchFixture, *string){
187 "seed role": func(_ *rootLaunchFixture, role *string) { *role = "seed" },
188 "sidecar role": func(_ *rootLaunchFixture, role *string) { *role = "sidecar" },
189 "socket elsewhere": func(f *rootLaunchFixture, _ *string) {
190 f.build = argsBuild("--socket", "/run/carlos/other.sock", "--db", filepath.Join(f.launchDir, "app.db"))
191 },
192 "db elsewhere": func(f *rootLaunchFixture, _ *string) {
193 f.build = argsBuild("--socket", filepath.Join(f.runDir, "app.sock"), "--db", "/data/other.db")
194 },
195 "no socket": func(f *rootLaunchFixture, _ *string) {
196 f.build = argsBuild("--db", filepath.Join(f.launchDir, "app.db"))
197 },
198 "extra flag": func(f *rootLaunchFixture, _ *string) {
199 f.build = argsBuild("--socket", filepath.Join(f.runDir, "app.sock"), "--db", filepath.Join(f.launchDir, "app.db"), "--exec", "/bin/sh")
200 },
201 "no launch dir": func(f *rootLaunchFixture, _ *string) { f.launch.LaunchDir = "" },
202 "no log socket": func(f *rootLaunchFixture, _ *string) { f.launch.LogSocket = "relative.sock" },
203 "no tier": func(f *rootLaunchFixture, _ *string) { f.launch.MemoryMax = 0 },
204 "attempt session": func(f *rootLaunchFixture, _ *string) { f.attempt.Session = 4 },
205 "attempt controller": func(f *rootLaunchFixture, _ *string) { f.attempt.ControllerID = "ctrl-2" },
206 "attempt id": func(f *rootLaunchFixture, _ *string) { f.attempt.ID = "" },
207 } {
208 f := newRootLaunchFixture(t)
209 role := "main"
210 mutate(f, &role)
211 cmd, _, settle, _, err := f.launch.Start(t.Context(), role, f.attempt, f.build, f.before)
212 if err == nil || cmd != nil || settle != nil {
213 t.Fatalf("%s started: %v", name, err)
214 }
215 if len(f.root.launches) != 0 || f.beforeRuns != 0 {
216 t.Fatalf("%s reached root or the before-start hook", name)
217 }
218 if _, err := os.Stat(filepath.Join(f.launchDir, instancectl.LaunchFileName(instancectl.RoleRun))); err == nil {
219 t.Fatalf("%s wrote a launch file", name)
220 }
221 }
222}
223
224func argsBuild(args ...string) func() *exec.Cmd {
225 return func() *exec.Cmd { return exec.Command("/opt/carlos/live/tenant.test", args...) }
226}
227
228func TestRootRoleLaunchBurnsEveryLaunchItDoesNotHold(t *testing.T) {
229 // A refusal or an unanswered launch is followed by a stop of the same
230 // reservation, so a late or uncertain start can never run unowned.
231 f := newRootLaunchFixture(t)
232 finished := []error{}
233 f.launch.RegisterProducer = func(context.Context, string) (RoleProducer, error) {
234 return RoleProducer{Context: t.Context(), Finish: func(err error) { finished = append(finished, err) }}, nil
235 }
236 f.root.grant = func(hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) {
237 return hostbox.RoleLaunchGrant{}, errors.New("connection reset")
238 }
239 cmd, _, settle, _, err := f.launch.Start(t.Context(), "main", f.attempt, f.build, f.before)
240 if err == nil || cmd != nil || settle != nil || len(f.root.stops) != 1 || *f.root.stops[0].Reservation != *f.root.launches[0].Reservation {
241 t.Fatalf("unanswered launch: err=%v settle=%v stops=%d", err, settle != nil, len(f.root.stops))
242 }
243 if len(finished) != 1 || finished[0] != nil {
244 t.Fatalf("settled producer retained a launch error: %v", finished)
245 }
246 // When the burning stop also fails, the caller keeps a settle to retry.
247 f = newRootLaunchFixture(t)
248 f.root.grant = func(hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) {
249 return hostbox.RoleLaunchGrant{}, errors.New("connection reset")
250 }
251 finished = nil
252 f.launch.RegisterProducer = func(context.Context, string) (RoleProducer, error) {
253 return RoleProducer{Context: t.Context(), Finish: func(err error) { finished = append(finished, err) }}, nil
254 }
255 f.root.stopErr = errors.New("root unreachable")
256 _, _, settle, _, err = f.launch.Start(t.Context(), "main", f.attempt, f.build, f.before)
257 if err == nil || settle == nil {
258 t.Fatalf("uncertain launch dropped its cleanup: %v", err)
259 }
260 if len(finished) != 0 {
261 t.Fatal("unproven unit finished its producer")
262 }
263 f.root.stopErr = nil
264 if err := settle(t.Context()); err != nil || len(finished) != 1 || finished[0] != nil {
265 t.Fatalf("settle retry did not complete producer: %v %v", err, finished)
266 }
267 // A grant for another process or reservation is never adopted.
268 for name, bad := range map[string]func(*hostbox.RoleLaunchGrant){
269 "other reservation": func(g *hostbox.RoleLaunchGrant) { g.ReservationID = "attempt-2:main" },
270 "reused pid": func(g *hostbox.RoleLaunchGrant) { g.ProcessStart = "1" },
271 "other group": func(g *hostbox.RoleLaunchGrant) { g.Group.RelativePath = "carlos.slice/x.service" },
272 } {
273 f := newRootLaunchFixture(t)
274 good := f.root.grant
275 f.root.grant = func(q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) {
276 g, err := good(q)
277 bad(&g)
278 return g, err
279 }
280 cmd, _, _, _, err := f.launch.Start(t.Context(), "main", f.attempt, f.build, f.before)
281 if err == nil || cmd != nil || len(f.root.stops) != 1 {
282 t.Fatalf("%s adopted: %v stops=%d", name, err, len(f.root.stops))
283 }
284 }
285}
286
287func TestRootClaimProcessStopsThroughRootAndWatchesExit(t *testing.T) {
288 child := exec.Command("sleep", "30")
289 if err := child.Start(); err != nil {
290 t.Skip("no sleep binary:", err)
291 }
292 reaped := make(chan struct{})
293 go func() { _ = child.Wait(); close(reaped) }()
294 proc, err := os.FindProcess(child.Process.Pid)
295 if err != nil {
296 t.Fatal(err)
297 }
298 cmd := &exec.Cmd{Path: "/opt/carlos/live/tenant.test", Process: proc}
299 p := newClaimProcessFor(cmd, nil, true)
300 if p.Exited() {
301 t.Fatal("running root process reported exited")
302 }
303 stopped := 0
304 p.settle = func(context.Context) error {
305 stopped++
306 return child.Process.Signal(syscall.SIGKILL)
307 }
308 ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
309 defer cancel()
310 if err := p.Stop(ctx); err != nil {
311 t.Fatal(err)
312 }
313 <-reaped
314 if stopped != 1 || !p.Exited() {
315 t.Fatalf("stopped=%d exited=%v", stopped, p.Exited())
316 }
317 if err := p.Stop(ctx); err != nil || stopped != 1 {
318 t.Fatalf("second stop=%v stopped=%d", err, stopped)
319 }
320 // Without a root settle there is no way to stop it: refuse, never assume.
321 other := exec.Command("sleep", "30")
322 if err := other.Start(); err != nil {
323 t.Fatal(err)
324 }
325 defer func() { _ = other.Process.Kill(); _ = other.Wait() }()
326 proc, _ = os.FindProcess(other.Process.Pid)
327 q := newClaimProcessFor(&exec.Cmd{Process: proc}, nil, true)
328 if err := q.Stop(ctx); !errors.Is(err, hostwrite.ErrUnavailable) {
329 t.Fatalf("unsettleable root process: %v", err)
330 }
331}
332
333func TestRootRoleHealthAndPeerReadTheUnitReservation(t *testing.T) {
334 raw, err := os.ReadFile("/proc/self/cgroup")
335 if err != nil || !strings.HasPrefix(string(raw), "0::/") {
336 t.Skip("no unified cgroup membership")
337 }
338 rel := strings.TrimSpace(strings.TrimPrefix(strings.SplitN(string(raw), "\n", 2)[0], "0::/"))
339 if !hostbox.UnitGroupPath(rel) {
340 t.Skip("test process is not in a service cgroup:", rel)
341 }
342 var st unix.Stat_t
343 if err := unix.Stat(filepath.Join("/sys/fs/cgroup", rel), &st); err != nil {
344 t.Skip("cgroup tree not visible:", err)
345 }
346 ctx := t.Context()
347 journal, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launch.db"))
348 if err != nil {
349 t.Fatal(err)
350 }
351 defer journal.Close()
352 f := newRootLaunchFixture(t)
353 target := f.launch.Target
354 if err := journal.Prelaunch(ctx, hostbox.LaunchInventory{ControllerPID: os.Getpid(), ProcessStart: selfStart(t), Target: target, Groups: []hostbox.LaunchGroup{{RelativePath: "carlos/parent", Device: 1, Inode: 2}}, NoState: true}); err != nil {
355 t.Fatal(err)
356 }
357 r := hostbox.LaunchReservation{ID: "attempt-1:main", Role: "main", InvocationID: "inv-1", Inputs: "inputs-1", Target: target, Group: hostbox.LaunchGroup{RelativePath: rel}, Executable: f.exeID}
358 if err := journal.IntendUnit(ctx, r); err != nil {
359 t.Fatal(err)
360 }
361 if err := journal.StartedUnit(ctx, r, hostbox.LaunchGroup{RelativePath: rel, Device: uint64(st.Dev), Inode: st.Ino, InvocationID: "unit-1"}, os.Getpid(), selfStart(t)); err != nil {
362 t.Fatal(err)
363 }
364 proc, _ := os.FindProcess(os.Getpid())
365 cmd := &exec.Cmd{Process: proc}
366 launch := RoleLaunch{Journal: journal, Target: target, CgroupRoot: "/sys/fs/cgroup", Root: f.root}
367 if err := launch.ExactHealth(ctx, cmd, f.attempt); err != nil {
368 t.Fatalf("exact health: %v", err)
369 }
370 a, b, err := socketPair()
371 if err != nil {
372 t.Fatal(err)
373 }
374 defer a.Close()
375 defer b.Close()
376 if err := launch.PeerInRole(ctx, a, cmd); err != nil {
377 t.Fatalf("peer in role: %v", err)
378 }
379 if err := (RoleLaunch{Journal: journal, Target: target, CgroupRoot: "/sys/fs/cgroup"}).PeerInRole(ctx, a, cmd); err == nil {
380 t.Fatal("peer accepted without the isolated root protocol")
381 }
382 other := f.attempt
383 other.InvocationID = "inv-2"
384 if err := launch.ExactHealth(ctx, cmd, other); err == nil {
385 t.Fatal("other attempt reported healthy")
386 }
387}
388
389func socketPair() (net.Conn, net.Conn, error) {
390 fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0)
391 if err != nil {
392 return nil, nil, err
393 }
394 fa, fb := os.NewFile(uintptr(fds[0]), "a"), os.NewFile(uintptr(fds[1]), "b")
395 defer fa.Close()
396 defer fb.Close()
397 a, err := net.FileConn(fa)
398 if err != nil {
399 return nil, nil, err
400 }
401 b, err := net.FileConn(fb)
402 if err != nil {
403 a.Close()
404 return nil, nil, err
405 }
406 return a, b, nil
407}
408
409// shortTempDir keeps unix socket paths under the kernel's length limit.
410func shortTempDir(t *testing.T) string {
411 t.Helper()
412 dir, err := os.MkdirTemp("", "rl")
413 if err != nil {
414 t.Fatal(err)
415 }
416 t.Cleanup(func() { os.RemoveAll(dir) })
417 return dir
418}
419
420type plan14LaunchWork struct {
421 held bool
422 calls int
423 stale bool
424}
425
426func (w *plan14LaunchWork) With(_ context.Context, _ hostwrite.Identity, fn func() error) error {
427 if w.held {
428 return hostwrite.ErrUnavailable
429 }
430 w.calls++
431 if w.stale {
432 return hostwrite.ErrUnavailable
433 }
434 w.held = true
435 defer func() { w.held = false }()
436 return fn()
437}
438func TestPlan14LaunchInitializationOutsideLock(t *testing.T) {
439 for _, stale := range []bool{false, true} {
440 t.Run(strconv.FormatBool(stale), func(t *testing.T) {
441 f := newRootLaunchFixture(t)
442 work := &plan14LaunchWork{}
443 f.launch.Work = work
444 calls := 0
445 f.launch.InitializeUnlocked = func(context.Context) error {
446 if work.held {
447 t.Fatal("root RPC under agent lock")
448 }
449 calls++
450 work.stale = stale
451 return nil
452 }
453 _, _, settle, _, err := f.launch.Start(t.Context(), "main", f.attempt, f.build, f.before)
454 if stale {
455 if err == nil || len(f.root.launches) != 0 {
456 t.Fatal("stale initialization applied")
457 }
458 } else {
459 if err != nil {
460 t.Fatal(err)
461 }
462 if err = settle(t.Context()); err != nil {
463 t.Fatal(err)
464 }
465 }
466 if calls != 1 || work.calls != 1 {
467 t.Fatal(calls, work.calls)
468 }
469 })
470 }
471}
472