| 1 | //go:build linux |
| 2 | |
| 3 | package instanceexec |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "os/signal" |
| 9 | "runtime" |
| 10 | "syscall" |
| 11 | |
| 12 | "golang.org/x/sys/unix" |
| 13 | ) |
| 14 | |
| 15 | // The trampoline stays behind as the namespace's init (#126, found on the |
| 16 | // dev fleet). The templates run with PrivatePIDs=yes, so the unit's main |
| 17 | // process is PID 1 of its own PID namespace, and the kernel gives a |
| 18 | // namespace init no default signal actions: a signal it has no handler |
| 19 | // for is dropped, whoever sends it (only SIGKILL from outside gets |
| 20 | // through). A tenant exec'd into that slot that never asked for SIGTERM, |
| 21 | // which is most Go servers, cannot be stopped by it: Go's runtime tries |
| 22 | // to die by the signal, the kernel drops it, and the runtime exits 2 |
| 23 | // instead; a tenant with no handler at all waits out TimeoutStopSec for |
| 24 | // the SIGKILL. Every hibernate, restart, move and deploy then left the |
| 25 | // unit failed. A flat tenant was never PID 1. |
| 26 | // |
| 27 | // So the trampoline forks the tenant as its child, which is not PID 1 and |
| 28 | // keeps its default actions exactly as on the flat layout, and does the |
| 29 | // two things PID 1 owes the namespace: |
| 30 | // |
| 31 | // - Forward. KillMode=mixed sends the stop's SIGTERM to the main |
| 32 | // process, which is this one; it and the other signals in forwarded |
| 33 | // go on to the tenant, which acts on them as it would have as the |
| 34 | // main process. SIGKILL needs no forwarding: after TimeoutStopSec |
| 35 | // systemd sends it to the whole cgroup, and SIGKILL to a namespace |
| 36 | // init from outside always kills it. |
| 37 | // - Reap. A process whose parent dies is reparented to the namespace's |
| 38 | // init, and stays a zombie until init waits for it. The tenant's own |
| 39 | // children that outlive their parent land here. |
| 40 | // |
| 41 | // When the tenant exits, the init exits with its status: the tenant's |
| 42 | // exit code, or 128+n for a death by signal n (the shell's convention; a |
| 43 | // namespace init cannot die by a signal it sends itself). The run and |
| 44 | // sidecar templates count 129, 130, 141 and 143 as success, the signals |
| 45 | // systemd itself counts as a clean death of a main process. The kernel |
| 46 | // then kills whatever is left in the namespace, as it did when the tenant |
| 47 | // was PID 1. |
| 48 | // |
| 49 | // The init is small on purpose and allocates nothing per signal: one |
| 50 | // buffered channel, one WaitStatus, a kill or a wait4 per signal. |
| 51 | |
| 52 | // forwarded are the signals the init passes on to the tenant. |
| 53 | var forwarded = [...]os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGUSR2} |
| 54 | |
| 55 | // scopedInit is the trampoline's launch: scope, start the tenant under |
| 56 | // the init, and exit with its status. It returns only if the tenant could |
| 57 | // not be started; a scope that cannot be applied refuses the start (the |
| 58 | // probe has already failed readiness on such a box, so no nested wake |
| 59 | // gets here). |
| 60 | func scopedInit(argv0 string, argv, env []string) error { |
| 61 | return runInit(argv0, argv, env, ScopeAbstractUnix) |
| 62 | } |
| 63 | |
| 64 | func runInit(argv0 string, argv, env []string, scope func() error) error { |
| 65 | return runInitChecked(argv0, argv, env, scope, nil) |
| 66 | } |
| 67 | |
| 68 | func runInitChecked(argv0 string, argv, env []string, scope, check func() error) error { |
| 69 | // Before anything can start the tenant: a stop that arrives early is |
| 70 | // queued and forwarded, not dropped by PID 1's missing defaults. |
| 71 | sigs := make(chan os.Signal, 32) |
| 72 | signal.Notify(sigs, append(forwarded[:], syscall.SIGCHLD)...) |
| 73 | |
| 74 | // Landlock restricts a thread, and a fork copies the forking thread's |
| 75 | // domain, which execve keeps: scoping this locked thread and forking |
| 76 | // from it scopes the tenant. The thread stays locked until exit. |
| 77 | runtime.LockOSThread() |
| 78 | if err := scope(); err != nil { |
| 79 | return fmt.Errorf("not starting without the abstract socket scope: %w", err) |
| 80 | } |
| 81 | // PID 1 in production already reaps orphans; this makes the init |
| 82 | // their reaper wherever it runs. |
| 83 | if err := unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); err != nil { |
| 84 | return fmt.Errorf("subreaper: %w", err) |
| 85 | } |
| 86 | // Same argv, env, cwd and stdio as the exec this replaces. Fds beyond |
| 87 | // stdio pass as they did there: anything not close-on-exec stays |
| 88 | // open across fork and exec. Setsid gives the tenant the session of |
| 89 | // its own that systemd gave the main process. |
| 90 | if check != nil { |
| 91 | if err := check(); err != nil { |
| 92 | return err |
| 93 | } |
| 94 | } |
| 95 | tenant, err := syscall.ForkExec(argv0, argv, &syscall.ProcAttr{ |
| 96 | Env: env, |
| 97 | Files: []uintptr{0, 1, 2}, |
| 98 | Sys: &syscall.SysProcAttr{Setsid: true}, |
| 99 | }) |
| 100 | if err != nil { |
| 101 | return err |
| 102 | } |
| 103 | os.Exit(superviseTenant(tenant, sigs)) |
| 104 | return nil |
| 105 | } |
| 106 | |
| 107 | // superviseTenant forwards signals to the tenant and reaps every child |
| 108 | // until the tenant itself is reaped, and returns the init's exit code. |
| 109 | func superviseTenant(tenant int, sigs <-chan os.Signal) int { |
| 110 | var ws syscall.WaitStatus |
| 111 | for s := range sigs { |
| 112 | sig := s.(syscall.Signal) |
| 113 | if sig != syscall.SIGCHLD { |
| 114 | _ = syscall.Kill(tenant, sig) |
| 115 | continue |
| 116 | } |
| 117 | // One SIGCHLD can stand for several exits: drain them all. |
| 118 | for { |
| 119 | pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, nil) |
| 120 | if err == syscall.EINTR { |
| 121 | continue |
| 122 | } |
| 123 | if err != nil || pid <= 0 { |
| 124 | break |
| 125 | } |
| 126 | if pid == tenant { |
| 127 | return initExitCode(ws) |
| 128 | } |
| 129 | } |
| 130 | } |
| 131 | return 1 // unreachable: signal.Notify never closes the channel |
| 132 | } |
| 133 | |
| 134 | // initExitCode maps the tenant's wait status to the init's exit code. |
| 135 | func initExitCode(ws syscall.WaitStatus) int { |
| 136 | if ws.Signaled() { |
| 137 | return 128 + int(ws.Signal()) |
| 138 | } |
| 139 | return ws.ExitStatus() |
| 140 | } |
| 141 | |
amadan