| 1 | //go:build linux |
| 2 | |
| 3 | package activator |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "net" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "slices" |
| 14 | "strconv" |
| 15 | "strings" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/hostbox" |
| 20 | "github.com/carlosframework/platform/internal/hostclaim" |
| 21 | "github.com/carlosframework/platform/internal/hostwrite" |
| 22 | "github.com/carlosframework/platform/internal/instancectl" |
| 23 | "github.com/carlosframework/platform/internal/instancefs" |
| 24 | "golang.org/x/sys/unix" |
| 25 | ) |
| 26 | |
| 27 | // startIsolated is Start through the reservation-bearing root protocol |
| 28 | // (claim Plan 11, R10). The agent writes the role's launch file and opens |
| 29 | // its log socket, exactly as main's nested start does, then asks root to |
| 30 | // run the reservation. It never builds a process of its own: the command it |
| 31 | // returns names the unit's main process root reported, and the settle it |
| 32 | // returns is root's stop of that same reservation. |
| 33 | func (l RoleLaunch) startIsolated(ctx context.Context, role string, attempt hostclaim.WakeRetryAttempt, build func() *exec.Cmd, before func() error) (*exec.Cmd, *cgroup.Prepared, func(context.Context) error, RoleProducer, error) { |
| 34 | fail := func(err error) (*exec.Cmd, *cgroup.Prepared, func(context.Context) error, RoleProducer, error) { |
| 35 | return nil, nil, nil, RoleProducer{}, err |
| 36 | } |
| 37 | if role != "main" { |
| 38 | return fail(hostbox.ErrRoleLaunchShape) |
| 39 | } |
| 40 | if l.Work == nil || l.Executable == nil || build == nil || before == nil || l.MemoryMax <= 0 || |
| 41 | !cleanAbs(l.LaunchDir) || !cleanAbs(l.RunDir) || !cleanAbs(l.LogSocket) || |
| 42 | attempt.ID == "" || attempt.InvocationID == "" || attempt.LaunchInputs == "" || attempt.Session != l.Target.Session || attempt.Box != l.Target.Box || |
| 43 | attempt.Enrollment != l.Target.Enrollment || attempt.BootID != l.Target.BootID || attempt.ControllerID != l.Target.ControllerID { |
| 44 | return fail(hostwrite.ErrUnavailable) |
| 45 | } |
| 46 | if l.InitializeUnlocked != nil { |
| 47 | if err := l.InitializeUnlocked(ctx); err != nil { |
| 48 | return fail(err) |
| 49 | } |
| 50 | } |
| 51 | host := l.Target.Host |
| 52 | var cmd *exec.Cmd |
| 53 | var executable *os.File |
| 54 | defer func() { |
| 55 | if executable != nil { |
| 56 | executable.Close() |
| 57 | } |
| 58 | }() |
| 59 | reservation := hostbox.LaunchReservation{ID: attempt.ID + ":" + role, Role: role, InvocationID: attempt.InvocationID, Inputs: attempt.LaunchInputs, Target: l.Target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(host)}} |
| 60 | if l.Unit != nil { |
| 61 | if !instancectl.ValidBoundClaimUnit(l.Unit.Unit) { |
| 62 | return fail(hostbox.ErrRoleLaunchShape) |
| 63 | } |
| 64 | reservation.Group.RelativePath = instancectl.BoundClaimUnitControlGroup(host, l.Unit.Unit) |
| 65 | } |
| 66 | err := l.Work.With(ctx, l.Identity, func() error { |
| 67 | cmd = build() |
| 68 | if cmd == nil { |
| 69 | return hostwrite.ErrUnavailable |
| 70 | } |
| 71 | var argv []string |
| 72 | var socket string |
| 73 | var err error |
| 74 | if l.Unit != nil { |
| 75 | if l.Spec == nil || len(cmd.Args) == 0 || !slices.Equal(cmd.Args[1:], l.Unit.Args) || !slices.Equal(cmd.Env, l.Unit.Env) || !slices.Equal(l.Spec.Args, l.Unit.Args) || !slices.Equal(l.Spec.Env, l.Unit.Env) { |
| 76 | return hostbox.ErrRoleLaunchShape |
| 77 | } |
| 78 | argv = append([]string{filepath.Join(instancectl.LiveDir, host)}, l.Unit.Args...) |
| 79 | } else { |
| 80 | argv, socket, err = isolatedArgv(cmd, host, l.LaunchDir, l.RunDir) |
| 81 | if err != nil { |
| 82 | return err |
| 83 | } |
| 84 | } |
| 85 | if l.Initialize != nil { |
| 86 | if err = l.Initialize(ctx); err != nil { |
| 87 | return err |
| 88 | } |
| 89 | } |
| 90 | if executable, err = l.Executable(ctx, cmd); err != nil { |
| 91 | return err |
| 92 | } |
| 93 | if executable == nil { |
| 94 | return hostwrite.ErrUnavailable |
| 95 | } |
| 96 | var stat unix.Stat_t |
| 97 | if err = unix.Fstat(int(executable.Fd()), &stat); err != nil { |
| 98 | return err |
| 99 | } |
| 100 | if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Mode&0111 == 0 || !cleanAbs(executable.Name()) { |
| 101 | return hostwrite.ErrUnavailable |
| 102 | } |
| 103 | reservation.Executable = hostbox.LaunchFile{Device: uint64(stat.Dev), Inode: stat.Ino} |
| 104 | // The nested start's own preparation (isolated-exec design §6.1): the |
| 105 | // run directory, no stale socket, and the launch file the trampoline |
| 106 | // execs. The file grants nothing the tenant could not write itself. |
| 107 | if err = os.MkdirAll(l.RunDir, 0o750|os.ModeSetgid); err != nil { |
| 108 | return err |
| 109 | } |
| 110 | if socket != "" { |
| 111 | if err = os.Remove(socket); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 112 | return err |
| 113 | } |
| 114 | } |
| 115 | raw, err := json.MarshalIndent(instancectl.Launch{Schema: instancectl.LaunchSchema, Host: host, Argv: argv, Env: append([]string(nil), cmd.Env...)}, "", " ") |
| 116 | if err != nil { |
| 117 | return err |
| 118 | } |
| 119 | return instancefs.WriteAtomic(l.LaunchDir, instancectl.LaunchFileName(instancectl.RoleRun), append(raw, '\n'), 0o600) |
| 120 | }) |
| 121 | if err != nil { |
| 122 | return fail(err) |
| 123 | } |
| 124 | request := hostbox.RoleLaunchRequest{Spec: l.Spec, V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Reservation: &reservation, Executable: executable.Name(), MemoryMax: l.MemoryMax} |
| 125 | stopRequest := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStop, Reservation: &reservation} |
| 126 | if err = request.Validate(); err != nil { |
| 127 | return fail(err) |
| 128 | } |
| 129 | logs, err := listenLogs(l.LogSocket, nil) |
| 130 | if err != nil { |
| 131 | return fail(err) |
| 132 | } |
| 133 | var producer RoleProducer |
| 134 | if l.RegisterProducer != nil { |
| 135 | if producer, err = l.RegisterProducer(ctx, role); err != nil { |
| 136 | logs.Close() |
| 137 | return fail(err) |
| 138 | } |
| 139 | } |
| 140 | finish := func(err error) { |
| 141 | if producer.Finish != nil { |
| 142 | producer.Finish(err) |
| 143 | } |
| 144 | } |
| 145 | // This hook runs after strict preparation and immediately before Start. |
| 146 | if err = before(); err != nil { |
| 147 | logs.Close() |
| 148 | finish(nil) |
| 149 | return fail(err) |
| 150 | } |
| 151 | settle := func(ctx context.Context) error { |
| 152 | if err := l.Root.Stop(ctx, stopRequest); err != nil { |
| 153 | return err |
| 154 | } |
| 155 | finish(nil) |
| 156 | return logs.Close() |
| 157 | } |
| 158 | grant, err := l.Root.Launch(ctx, request) |
| 159 | if err == nil { |
| 160 | err = grant.Check(reservation) |
| 161 | } |
| 162 | var process *os.Process |
| 163 | if err == nil { |
| 164 | process, err = pinRootProcess(grant.PID, grant.ProcessStart) |
| 165 | } |
| 166 | if err != nil { |
| 167 | // A refused, unanswered or unusable launch is stopped by its own |
| 168 | // reservation, which root then refuses ever to start. Only a stop |
| 169 | // that also failed leaves the caller a settle to retry. |
| 170 | cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) |
| 171 | stopErr := settle(cleanup) |
| 172 | cancel() |
| 173 | if stopErr != nil { |
| 174 | return nil, nil, settle, RoleProducer{}, errors.Join(err, stopErr) |
| 175 | } |
| 176 | return fail(err) |
| 177 | } |
| 178 | cmd.Process = process |
| 179 | return cmd, nil, settle, producer, nil |
| 180 | } |
| 181 | |
| 182 | func cleanAbs(p string) bool { return filepath.IsAbs(p) && filepath.Clean(p) == p } |
| 183 | |
| 184 | // isolatedArgv is the trampoline's argv for cmd: the unit's view of the |
| 185 | // host's live build, then exactly --socket in the run directory and --db in |
| 186 | // the instance directory, each once. Any other argument is a shape the |
| 187 | // isolated unit does not carry. |
| 188 | func isolatedArgv(cmd *exec.Cmd, host, launchDir, runDir string) ([]string, string, error) { |
| 189 | if len(cmd.Args) == 0 { |
| 190 | return nil, "", hostbox.ErrRoleLaunchShape |
| 191 | } |
| 192 | argv, socket, _, err := IsolatedArgv(host, launchDir, runDir, cmd.Args[1:], cmd.Env) |
| 193 | return argv, socket, err |
| 194 | } |
| 195 | |
| 196 | // PeerInRole verifies that conn's peer runs inside the unit root started |
| 197 | // for cmd: the journaled unit reservation names cmd's process, its cgroup |
| 198 | // still has the journaled identity, and the peer is a member of exactly that |
| 199 | // group. The tenant answering the socket is the trampoline's child, so its |
| 200 | // pid is never cmd's. |
| 201 | func (l RoleLaunch) PeerInRole(ctx context.Context, conn net.Conn, cmd *exec.Cmd) error { |
| 202 | unixConn, ok := conn.(*net.UnixConn) |
| 203 | if l.Root == nil || l.Journal == nil || !ok || cmd == nil || cmd.Process == nil || !filepath.IsAbs(l.CgroupRoot) { |
| 204 | return hostwrite.ErrUnavailable |
| 205 | } |
| 206 | raw, err := unixConn.SyscallConn() |
| 207 | if err != nil { |
| 208 | return err |
| 209 | } |
| 210 | var peer *unix.Ucred |
| 211 | var sockErr error |
| 212 | if err = raw.Control(func(fd uintptr) { peer, sockErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) }); err != nil { |
| 213 | return err |
| 214 | } |
| 215 | if sockErr != nil { |
| 216 | return sockErr |
| 217 | } |
| 218 | if peer == nil || peer.Pid <= 0 { |
| 219 | return hostwrite.ErrUnavailable |
| 220 | } |
| 221 | inventory, err := l.Journal.Read(ctx, l.Target) |
| 222 | if err != nil { |
| 223 | return err |
| 224 | } |
| 225 | for _, r := range inventory.Reservations { |
| 226 | if r.State != hostbox.UnitStarted || r.PID != cmd.Process.Pid { |
| 227 | continue |
| 228 | } |
| 229 | start, err := roleProcessStart(r.PID) |
| 230 | if err != nil { |
| 231 | return err |
| 232 | } |
| 233 | if start != r.ProcessStart { |
| 234 | return hostbox.ErrChanged |
| 235 | } |
| 236 | var stat unix.Stat_t |
| 237 | if err = unix.Stat(filepath.Join(l.CgroupRoot, r.Group.RelativePath), &stat); err != nil { |
| 238 | return err |
| 239 | } |
| 240 | if uint64(stat.Dev) != r.Group.Device || stat.Ino != r.Group.Inode { |
| 241 | return hostbox.ErrChanged |
| 242 | } |
| 243 | membership, err := os.ReadFile("/proc/" + strconv.Itoa(int(peer.Pid)) + "/cgroup") |
| 244 | if err != nil { |
| 245 | return err |
| 246 | } |
| 247 | for _, line := range strings.Split(string(membership), "\n") { |
| 248 | if line == "0::/"+r.Group.RelativePath { |
| 249 | return nil |
| 250 | } |
| 251 | } |
| 252 | return hostbox.ErrChanged |
| 253 | } |
| 254 | return hostbox.ErrChanged |
| 255 | } |
| 256 | |
amadan