| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "testing" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/activator" |
| 14 | "github.com/carlosframework/platform/internal/cgroup" |
| 15 | "github.com/carlosframework/platform/internal/hostbox" |
| 16 | "github.com/carlosframework/platform/internal/hostclaim" |
| 17 | "github.com/carlosframework/platform/internal/hostwrite" |
| 18 | "github.com/carlosframework/platform/internal/instancectl" |
| 19 | "github.com/carlosframework/platform/internal/provision" |
| 20 | "golang.org/x/sys/unix" |
| 21 | ) |
| 22 | |
| 23 | type plan11LaunchClient struct{ fixture *launchFixture } |
| 24 | |
| 25 | func (c plan11LaunchClient) Capability(context.Context) (string, error) { |
| 26 | return hostbox.RoleLaunchProtocol, nil |
| 27 | } |
| 28 | func (c plan11LaunchClient) Launch(ctx context.Context, q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 29 | r, e := c.fixture.root.Serve(ctx, c.fixture.peer, q) |
| 30 | if e != nil { |
| 31 | return hostbox.RoleLaunchGrant{}, e |
| 32 | } |
| 33 | return *r.Grant, nil |
| 34 | } |
| 35 | func (c plan11LaunchClient) Stop(ctx context.Context, q hostbox.RoleLaunchRequest) error { |
| 36 | _, e := c.fixture.root.Serve(ctx, c.fixture.peer, q) |
| 37 | return e |
| 38 | } |
| 39 | |
| 40 | func TestPlan11FailedIsolatedStartCanRetryAndDrain(t *testing.T) { |
| 41 | f := newLaunchFixture(t, servingOp()) |
| 42 | dir, err := os.MkdirTemp(os.TempDir(), "p11r-") |
| 43 | if err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 47 | exe := filepath.Join(dir, "app") |
| 48 | if err := os.WriteFile(exe, []byte("test executable"), 0700); err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | var st unix.Stat_t |
| 52 | if err := unix.Stat(exe, &st); err != nil { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | f.kernel.path = exe |
| 56 | f.kernel.want = hostbox.LaunchFile{Device: uint64(st.Dev), Inode: st.Ino} |
| 57 | f.kernel.observed.Executable = f.kernel.want |
| 58 | f.kernel.observed.Running = f.kernel.want |
| 59 | f.units.started.MainPID = os.Getpid() |
| 60 | f.kernel.observed.ProcessStart, err = claimConstructionProcessStart() |
| 61 | if err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | producers, err := NewClaimProducers(f.target, "start-42", f.journal.RecordProducer, f.journal.CompleteProducer) |
| 65 | if err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | work, err := (hostwrite.Config{}).Select(t.Context(), f.target.Host, "spawn") |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | f.root.DataRoot, f.root.runRoot = dir, filepath.Join(dir, "run") |
| 73 | launchDir := provision.InstanceDir(dir, f.target.Host) |
| 74 | if err := os.MkdirAll(launchDir, 0700); err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | launch := activator.RoleLaunch{Root: plan11LaunchClient{f}, Target: f.target, Journal: f.journal, Work: work, LaunchDir: launchDir, RunDir: filepath.Join(f.root.runRoot, provision.InstanceID(f.target.Host)), LogSocket: filepath.Join(dir, "log.sock"), MemoryMax: cgroup.TierDefault.MemoryMax, ParentPath: "carlos/claim-parent"} |
| 78 | launch.Executable = func(context.Context, *exec.Cmd) (*os.File, error) { return os.Open(exe) } |
| 79 | launch.RegisterProducer = func(ctx context.Context, role string) (activator.RoleProducer, error) { |
| 80 | ctx, finish, err := producers.Register(ctx, f.target, role) |
| 81 | return activator.RoleProducer{Context: ctx, Finish: finish}, err |
| 82 | } |
| 83 | build := func() *exec.Cmd { |
| 84 | cmd := exec.Command(exe, "--socket", filepath.Join(launch.RunDir, "app.sock"), "--db", filepath.Join(launchDir, "data.db")) |
| 85 | cmd.Env = []string{"CARLOS_ADMIN_TOKEN=test"} |
| 86 | return cmd |
| 87 | } |
| 88 | attempt := hostclaim.WakeRetryAttempt{ID: "first", InvocationID: "inv-first", LaunchInputs: "inputs", Session: f.target.Session, Box: f.target.Box, Enrollment: f.target.Enrollment, BootID: f.target.BootID, ControllerID: f.target.ControllerID} |
| 89 | launch.Spec = &hostbox.LaunchSpec{Identity: f.req.Spec.Identity, Executable: exe, Digest: f.kernel.digest, Args: build().Args[1:], Env: build().Env} |
| 90 | attempt.LaunchInputs, _ = launch.Spec.Hash() |
| 91 | f.claims.set(func(c *hostclaim.Claim) { a := attempt; c.Holder.WakeRetry.Attempt = &a }) |
| 92 | f.units.startErr = errors.New("first unit start failed") |
| 93 | if _, _, _, _, err := launch.Start(t.Context(), "main", attempt, build, func() error { return nil }); err == nil { |
| 94 | t.Fatal("failed start reported success") |
| 95 | } |
| 96 | f.units.startErr = nil |
| 97 | attempt.ID = "second" |
| 98 | attempt.InvocationID = "inv-second" |
| 99 | f.claims.set(func(c *hostclaim.Claim) { |
| 100 | a := attempt |
| 101 | c.Holder.WakeRetry.Attempt = &a |
| 102 | c.Holder.WakeRetry.AttemptsReserved = 2 |
| 103 | }) |
| 104 | _, _, settle, producer, err := launch.Start(t.Context(), "main", attempt, build, func() error { return nil }) |
| 105 | if err != nil { |
| 106 | t.Fatalf("retry after settled failure: %v", err) |
| 107 | } |
| 108 | if err := settle(t.Context()); err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | producer.Finish(nil) |
| 112 | a := &claimWakeAttempt{owner: &ClaimWakeAssembly{Producers: func(context.Context, hostclaim.Observed) (*ClaimProducers, error) { return producers, nil }}, observed: f.claims.obs, inputs: ClaimLaunchInputs{DBPath: filepath.Join(dir, "data.db")}, prelaunch: &launch, writersJournal: f.journal, writersCgroup: &writersCgroupFake{}} |
| 113 | if witness, err := a.StopWriters(t.Context()); err != nil || !witness.Valid() { |
| 114 | t.Fatalf("writers drain after retry: %+v %v", witness, err) |
| 115 | } |
| 116 | before, _ := f.units.counts() |
| 117 | refused := *f.req.Reservation |
| 118 | refused.ID, refused.InvocationID = "after-stop:main", "after-stop" |
| 119 | refused.Executable = f.kernel.want |
| 120 | q := f.req |
| 121 | q.Reservation, q.Executable = &refused, exe |
| 122 | if _, err := f.root.Serve(t.Context(), f.peer, q); err == nil { |
| 123 | t.Fatal("root launched after writers drain") |
| 124 | } |
| 125 | if after, _ := f.units.counts(); after != before { |
| 126 | t.Fatal("closed writers reached unit start") |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func TestPlan11LateRootStopDoesNotStopAnotherInvocation(t *testing.T) { |
| 131 | f := newLaunchFixture(t, servingOp()) |
| 132 | if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | // Gone reports that this reservation's invocation has already ended. |
| 136 | if _, err := f.root.Serve(t.Context(), f.peer, hostbox.RoleLaunchRequest{V: 1, Op: hostbox.RoleLaunchStop, Reservation: f.req.Reservation}); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | if _, stops := f.units.counts(); stops != 0 { |
| 140 | t.Fatal("late stop touched the host's replacement invocation") |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | func TestPlan11NewLaunchSettlesOldUnstartedReservation(t *testing.T) { |
| 145 | f := newLaunchFixture(t, servingOp()) |
| 146 | old := *f.req.Reservation |
| 147 | if err := f.journal.IntendUnit(t.Context(), old); err != nil { |
| 148 | t.Fatal(err) |
| 149 | } |
| 150 | // A different controller replaces one that crashed after recording intent. |
| 151 | next := old |
| 152 | next.ID, next.InvocationID = "next:main", "next-inv" |
| 153 | next.Target.ControllerID = "controller-2" |
| 154 | nextPeer := f.peer |
| 155 | nextPeer.ControllerID = next.Target.ControllerID |
| 156 | parent, err := claimContainmentParent("/sys/fs/cgroup/system.slice/carlos-edge.service", next.Target) |
| 157 | if err != nil { |
| 158 | t.Fatal(err) |
| 159 | } |
| 160 | if err := f.journal.Prelaunch(t.Context(), hostbox.LaunchInventory{ControllerPID: nextPeer.PID, ProcessStart: nextPeer.ProcessStart, Target: next.Target, Groups: []hostbox.LaunchGroup{{RelativePath: parent, Device: 1, Inode: 9}}, NoState: true}); err != nil { |
| 161 | t.Fatal(err) |
| 162 | } |
| 163 | q := f.req |
| 164 | q.Reservation = &next |
| 165 | f.claims.set(func(c *hostclaim.Claim) { |
| 166 | a := *c.Holder.WakeRetry.Attempt |
| 167 | a.ID = "next" |
| 168 | a.InvocationID = next.InvocationID |
| 169 | a.ControllerID = next.Target.ControllerID |
| 170 | c.Holder.WakeRetry.Attempt = &a |
| 171 | c.Holder.WakeRetry.AttemptsReserved = 2 |
| 172 | }) |
| 173 | if _, err := f.root.Serve(t.Context(), nextPeer, q); err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | previous, err := f.journal.UnitReservation(t.Context(), old.Target, old.ID) |
| 177 | if err != nil || previous.State != hostbox.UnitSettled { |
| 178 | t.Fatalf("old intent survived proven-empty start: %+v %v", previous, err) |
| 179 | } |
| 180 | f.units.goneErr = instancectl.ErrClaimUnitBusy |
| 181 | if _, err := f.root.Serve(t.Context(), f.peer, hostbox.RoleLaunchRequest{V: 1, Op: hostbox.RoleLaunchStop, Reservation: &old}); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | if _, stops := f.units.counts(); stops != 0 { |
| 185 | t.Fatal("old intended reservation stopped replacement") |
| 186 | } |
| 187 | } |
| 188 | |
amadan