| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "net" |
| 9 | "net/http" |
| 10 | "os" |
| 11 | "os/exec" |
| 12 | "path/filepath" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/activator" |
| 17 | "github.com/carlosframework/platform/internal/cgroup" |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/hostwrite" |
| 22 | "github.com/carlosframework/platform/internal/instancectl" |
| 23 | "github.com/carlosframework/platform/internal/provision" |
| 24 | "golang.org/x/sys/unix" |
| 25 | ) |
| 26 | |
| 27 | // serveClaimRoot runs the production instance-control mux in claim mode with |
| 28 | // h as its claim handler, on a real unix socket. |
| 29 | func serveClaimRoot(t *testing.T, h http.Handler) string { |
| 30 | t.Helper() |
| 31 | return serveClaimRootWith(t, h, nil) |
| 32 | } |
| 33 | |
| 34 | // serveClaimRootWithHeldRelease serves h as the claim root with the moves |
| 35 | // held-release route registered to it too, as the host registers it with |
| 36 | // both flags on (whole-branch fixes, item 6). |
| 37 | func serveClaimRootWithHeldRelease(t *testing.T, h http.Handler) string { |
| 38 | t.Helper() |
| 39 | return serveClaimRootWith(t, h, h) |
| 40 | } |
| 41 | |
| 42 | func serveClaimRootWith(t *testing.T, h, heldRelease http.Handler) string { |
| 43 | t.Helper() |
| 44 | dir, err := os.MkdirTemp("", "cl") |
| 45 | if err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 49 | sock := filepath.Join(dir, "c.sock") |
| 50 | ln, err := net.Listen("unix", sock) |
| 51 | if err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | ctx, cancel := context.WithCancel(context.Background()) |
| 55 | server := &instancectl.Server{ClaimStop: h, ClaimHeldRelease: heldRelease, PeerOK: func(net.Conn) error { return nil }} |
| 56 | done := make(chan struct{}) |
| 57 | go func() { defer close(done); _ = server.Serve(ctx, ln) }() |
| 58 | t.Cleanup(func() { cancel(); <-done }) |
| 59 | return sock |
| 60 | } |
| 61 | |
| 62 | func procSelfStart(t *testing.T) string { |
| 63 | t.Helper() |
| 64 | raw, err := os.ReadFile("/proc/self/stat") |
| 65 | if err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | fields := strings.Fields(string(raw)[strings.LastIndexByte(string(raw), ')')+1:]) |
| 69 | return fields[19] |
| 70 | } |
| 71 | |
| 72 | func TestClaimRoleRootClientThroughTheClaimMux(t *testing.T) { |
| 73 | f := newLaunchFixture(t, servingOp()) |
| 74 | authenticated := 0 |
| 75 | f.root.Authenticate = func(_ context.Context, conn net.Conn) (StopPeer, error) { |
| 76 | if conn == nil { |
| 77 | t.Fatal("no request connection") |
| 78 | } |
| 79 | authenticated++ |
| 80 | return f.peer, nil |
| 81 | } |
| 82 | sock := serveClaimRoot(t, ClaimRootWithLaunch{Launch: f.root}) |
| 83 | client := NewClaimRoleRootClient(sock) |
| 84 | protocol, err := client.Capability(t.Context()) |
| 85 | if err != nil || protocol != hostbox.RoleLaunchProtocol { |
| 86 | t.Fatalf("capability=%q %v", protocol, err) |
| 87 | } |
| 88 | grant, err := client.Launch(t.Context(), f.req) |
| 89 | if err != nil || grant.Check(*f.req.Reservation) != nil { |
| 90 | t.Fatalf("grant=%+v %v", grant, err) |
| 91 | } |
| 92 | if _, err := client.Launch(t.Context(), f.req); !errors.Is(err, hostbox.ErrChanged) { |
| 93 | t.Fatalf("replay over the wire: %v", err) |
| 94 | } |
| 95 | if err := client.Stop(t.Context(), hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStop, Reservation: f.req.Reservation}); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | if authenticated != 4 { |
| 99 | t.Fatalf("authenticated %d of 4 requests", authenticated) |
| 100 | } |
| 101 | // An unauthenticated peer gets nothing, not even the capability. |
| 102 | f.root.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, errors.New("not the edge") } |
| 103 | if _, err := client.Capability(t.Context()); err == nil { |
| 104 | t.Fatal("unauthenticated capability") |
| 105 | } |
| 106 | // A body the protocol cannot carry is refused before authority checks. |
| 107 | f.root.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil } |
| 108 | resp, err := (&http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 109 | return (&net.Dialer{}).DialContext(ctx, "unix", sock) |
| 110 | }}}).Post("http://control/claim/launch", "application/json", strings.NewReader(`{"v":1,"op":"start","argv":["/bin/sh"]}`)) |
| 111 | if err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | resp.Body.Close() |
| 115 | if resp.StatusCode != http.StatusBadRequest { |
| 116 | t.Fatalf("unknown field status=%d", resp.StatusCode) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func TestClaimRoleLaunchEndToEndThroughTheRootProtocol(t *testing.T) { |
| 121 | f := newLaunchFixture(t, servingOp()) |
| 122 | f.root.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil } |
| 123 | base, err := os.MkdirTemp("", "e2e") |
| 124 | if err != nil { |
| 125 | t.Fatal(err) |
| 126 | } |
| 127 | t.Cleanup(func() { os.RemoveAll(base) }) |
| 128 | exe := filepath.Join(base, "app") |
| 129 | if err := os.WriteFile(exe, []byte("build"), 0o555); err != nil { |
| 130 | t.Fatal(err) |
| 131 | } |
| 132 | var st unix.Stat_t |
| 133 | if err := unix.Stat(exe, &st); err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | f.kernel.path, f.kernel.want = exe, hostbox.LaunchFile{Device: uint64(st.Dev), Inode: st.Ino} |
| 137 | f.kernel.observed.Executable = f.kernel.want |
| 138 | f.kernel.observed.Running = f.kernel.want |
| 139 | f.kernel.observed.ProcessStart = procSelfStart(t) |
| 140 | f.units.started.MainPID = os.Getpid() |
| 141 | sock := serveClaimRoot(t, ClaimRootWithLaunch{Launch: f.root}) |
| 142 | work, err := (hostwrite.Config{}).Select(t.Context(), f.target.Host, "spawn") |
| 143 | if err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | f.root.DataRoot, f.root.runRoot = base, filepath.Join(base, "run") |
| 147 | launchDir, runDir := provision.InstanceDir(base, f.target.Host), filepath.Join(f.root.runRoot, provision.InstanceID(f.target.Host)) |
| 148 | if err := os.MkdirAll(launchDir, 0o700); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | launch := activator.RoleLaunch{Work: work, Target: f.target, Root: NewClaimRoleRootClient(sock), LaunchDir: launchDir, RunDir: runDir, LogSocket: filepath.Join(base, "l.sock"), MemoryMax: cgroup.TierDefault.MemoryMax, Executable: func(context.Context, *exec.Cmd) (*os.File, error) { return os.Open(exe) }} |
| 152 | attempt := hostclaim.WakeRetryAttempt{ID: "attempt-1", InvocationID: "inv-1", LaunchInputs: "inputs-1", Session: 13, Box: "box-1", Enrollment: "enrollment-1", BootID: f.peer.BootID, ControllerID: "controller-1"} |
| 153 | build := func() *exec.Cmd { |
| 154 | cmd := exec.Command("/opt/carlos/live/shop.example", "--socket", filepath.Join(runDir, "app.sock"), "--db", filepath.Join(launchDir, "app.db")) |
| 155 | cmd.Env = []string{"CARLOS_ADMIN_TOKEN=test"} |
| 156 | return cmd |
| 157 | } |
| 158 | launch.Spec = &hostbox.LaunchSpec{Identity: f.req.Spec.Identity, Executable: exe, Digest: f.kernel.digest, Args: build().Args[1:], Env: build().Env} |
| 159 | attempt.LaunchInputs, _ = launch.Spec.Hash() |
| 160 | f.claims.set(func(c *hostclaim.Claim) { a := attempt; c.Holder.WakeRetry.Attempt = &a }) |
| 161 | cmd, _, settle, _, err := launch.Start(t.Context(), "main", attempt, build, func() error { return nil }) |
| 162 | if err != nil { |
| 163 | t.Fatal(err) |
| 164 | } |
| 165 | if cmd.Process.Pid != os.Getpid() { |
| 166 | t.Fatalf("pid=%d", cmd.Process.Pid) |
| 167 | } |
| 168 | got, err := f.journal.UnitReservation(t.Context(), f.target, "attempt-1:main") |
| 169 | if err != nil || got.State != hostbox.UnitStarted || got.InvocationID != "inv-1" || got.Inputs != attempt.LaunchInputs { |
| 170 | t.Fatalf("root journal=%+v %v", got, err) |
| 171 | } |
| 172 | if err := settle(t.Context()); err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | if got, err = f.journal.UnitReservation(t.Context(), f.target, "attempt-1:main"); err != nil || got.State != hostbox.UnitSettled { |
| 176 | t.Fatalf("settled=%+v %v", got, err) |
| 177 | } |
| 178 | // The same attempt cannot launch twice, even from the live controller. |
| 179 | if _, _, _, _, err := launch.Start(t.Context(), "main", attempt, build, func() error { return nil }); err == nil { |
| 180 | t.Fatal("spent attempt relaunched") |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | func TestClaimRolePreflightAdmitsOnlyTheCompleteProtocol(t *testing.T) { |
| 185 | var nilAssembly *ClaimRoleAssembly |
| 186 | for name, a := range map[string]*ClaimRoleAssembly{ |
| 187 | "nil assembly": nilAssembly, |
| 188 | "no protocol": {}, |
| 189 | "root refuses": {Root: fakeLaunchRoot{err: activator.ErrClaimIsolationUnavailable}}, |
| 190 | "older protocol": {Root: fakeLaunchRoot{protocol: "isolated-unit-v0"}}, |
| 191 | "unreachable": {Root: fakeLaunchRoot{err: errors.New("dial unix: no such file")}}, |
| 192 | } { |
| 193 | if err := a.Preflight(t.Context(), ClaimLaunchInputs{}); !errors.Is(err, activator.ErrClaimIsolationUnavailable) { |
| 194 | t.Fatalf("%s: %v", name, err) |
| 195 | } |
| 196 | } |
| 197 | if err := (&ClaimRoleAssembly{Root: fakeLaunchRoot{protocol: hostbox.RoleLaunchProtocol}}).Preflight(t.Context(), ClaimLaunchInputs{}); err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | func TestClaimRoleIsolatedPathsBindTheNestedInstance(t *testing.T) { |
| 203 | roots := map[string]string{"data": "/data"} |
| 204 | host := "shop.example" |
| 205 | got, err := claimIsolatedPaths(roots, host, ClaimLaunchInputs{DBPath: "/data/instances/" + iidOf(host) + "/shop.db", Limit: cgroup.TierDefault}) |
| 206 | if err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | if got.LaunchDir != "/data/instances/"+iidOf(host) || got.RunDir != "/run/carlos/instances/"+iidOf(host) || !strings.HasSuffix(got.LogSocket, ".run.sock") || got.MemoryMax != cgroup.TierDefault.MemoryMax { |
| 210 | t.Fatalf("paths=%+v", got) |
| 211 | } |
| 212 | for name, in := range map[string]ClaimLaunchInputs{ |
| 213 | "flat database": {DBPath: "/data/shop.example.db", Limit: cgroup.TierDefault}, |
| 214 | "other instance": {DBPath: "/data/instances/other/shop.db", Limit: cgroup.TierDefault}, |
| 215 | "nested deeper": {DBPath: "/data/instances/" + iidOf(host) + "/x/shop.db", Limit: cgroup.TierDefault}, |
| 216 | "not a tier": {DBPath: "/data/instances/" + iidOf(host) + "/shop.db", Limit: cgroup.Limit{MemoryMax: 7}}, |
| 217 | "relative": {DBPath: "instances/shop.db", Limit: cgroup.TierDefault}, |
| 218 | "unescaped parent": {DBPath: "/data/instances/" + iidOf(host) + "/../x/shop.db", Limit: cgroup.TierDefault}, |
| 219 | } { |
| 220 | if _, err := claimIsolatedPaths(roots, host, in); !errors.Is(err, hostbox.ErrRoleLaunchShape) { |
| 221 | t.Fatalf("%s: %v", name, err) |
| 222 | } |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | func TestProductionClaimLaunchWrapsTheRootHandler(t *testing.T) { |
| 227 | journal, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launch.db")) |
| 228 | if err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | defer journal.Close() |
| 232 | inner := stopKernelFunc(func(context.Context, hostguard.FenceTarget, StopInventory, DrainAcknowledgement) (StopKernelEvidence, error) { |
| 233 | return StopKernelEvidence{}, nil |
| 234 | }) |
| 235 | authenticate := func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, nil } |
| 236 | root := ClaimRootHTTP{Launches: journal, Stop: ClaimStopHTTP{Authenticate: authenticate, Coordinator: ClaimStopCoordinator{Kernel: inner, Guard: hostguard.Guard{DataDir: "/data", Claims: &launchClaims{}}}}} |
| 237 | actor := func(context.Context) (hostbox.Actor, error) { return hostbox.Actor{}, nil } |
| 238 | wrapped, err := claimLaunchFor(root, actor, map[string]string{"opt": "/opt/carlos", "data": "/data"}) |
| 239 | if err != nil { |
| 240 | t.Fatal(err) |
| 241 | } |
| 242 | k, ok := wrapped.Root.Stop.Coordinator.Kernel.(claimUnitStopKernel) |
| 243 | if !ok || k.Journal != journal || k.Inner == nil { |
| 244 | t.Fatalf("stop kernel not wrapped: %T", wrapped.Root.Stop.Coordinator.Kernel) |
| 245 | } |
| 246 | l := wrapped.Launch |
| 247 | units, ok := l.Units.(instancectl.ClaimUnits) |
| 248 | if !ok || units.Systemctl == nil || units.CgroupRoot != "/sys/fs/cgroup" || !strings.HasSuffix(units.ExeRoot, "/exe") || units.Room == nil { |
| 249 | t.Fatalf("units=%+v", l.Units) |
| 250 | } |
| 251 | if kernel, ok := l.Kernel.(linuxClaimLaunchKernel); !ok || kernel.OptRoot != "/opt/carlos" || kernel.CgroupRoot != "/sys/fs/cgroup" || kernel.ProcRoot != "/proc" || kernel.Owner != 0 { |
| 252 | t.Fatalf("kernel=%+v", l.Kernel) |
| 253 | } |
| 254 | if l.Journal != journal || l.Authenticate == nil || l.Actor == nil || l.Guard.DataDir != "/data" { |
| 255 | t.Fatalf("launch root=%+v", l) |
| 256 | } |
| 257 | for name, bad := range map[string]func(*ClaimRootHTTP, *map[string]string){ |
| 258 | "no journal": func(r *ClaimRootHTTP, _ *map[string]string) { r.Launches = nil }, |
| 259 | "no authenticate": func(r *ClaimRootHTTP, _ *map[string]string) { r.Stop.Authenticate = nil }, |
| 260 | "no stop kernel": func(r *ClaimRootHTTP, _ *map[string]string) { r.Stop.Coordinator.Kernel = nil }, |
| 261 | "unbound data root": func(_ *ClaimRootHTTP, m *map[string]string) { (*m)["data"] = "/elsewhere" }, |
| 262 | "no opt root": func(_ *ClaimRootHTTP, m *map[string]string) { *m = map[string]string{"data": "/data"} }, |
| 263 | } { |
| 264 | r := root |
| 265 | m := map[string]string{"opt": "/opt/carlos", "data": "/data"} |
| 266 | bad(&r, &m) |
| 267 | if _, err := claimLaunchFor(r, actor, m); err == nil { |
| 268 | t.Fatalf("%s accepted", name) |
| 269 | } |
| 270 | } |
| 271 | if _, err := WithProductionClaimLaunch(t.Context(), http.NotFoundHandler()); err == nil { |
| 272 | t.Fatal("wrapped a handler that is not the claim root") |
| 273 | } |
| 274 | if _, err := WithProductionClaimLaunch(t.Context(), root); err == nil { |
| 275 | t.Fatal("wrapped without the production assembly") |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | func iidOf(host string) string { return provision.InstanceID(host) } |
| 280 | |
amadan