| 1 | //go:build linux && rootrunner |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | "io" |
| 10 | "net" |
| 11 | "net/http" |
| 12 | "os" |
| 13 | "os/exec" |
| 14 | "os/user" |
| 15 | "path/filepath" |
| 16 | "strconv" |
| 17 | "strings" |
| 18 | "testing" |
| 19 | "time" |
| 20 | |
| 21 | "github.com/carlosframework/platform/internal/activator" |
| 22 | "github.com/carlosframework/platform/internal/hostbox" |
| 23 | "github.com/carlosframework/platform/internal/hostclaim" |
| 24 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 25 | "github.com/carlosframework/platform/internal/hostwrite" |
| 26 | "github.com/carlosframework/platform/internal/instancectl" |
| 27 | "github.com/carlosframework/platform/internal/instanceexec" |
| 28 | "github.com/carlosframework/platform/internal/provision" |
| 29 | "github.com/carlosframework/platform/internal/registry" |
| 30 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 31 | "github.com/carlosframework/platform/internal/rootstate" |
| 32 | "golang.org/x/sys/unix" |
| 33 | ) |
| 34 | |
| 35 | func init() { |
| 36 | if len(os.Args) > 1 { |
| 37 | switch os.Args[1] { |
| 38 | case "instance-exec": |
| 39 | instanceexec.RunExec(os.Args[2:]) |
| 40 | os.Exit(0) |
| 41 | case "instance-exit": |
| 42 | instanceexec.RunExit(os.Args[2:]) |
| 43 | os.Exit(0) |
| 44 | } |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | func TestRootFilesSeedIsolatedTenant(t *testing.T) { |
| 49 | runServingParkPeerNamespace(t, false, false, "seed-isolated") |
| 50 | } |
| 51 | |
| 52 | type seedIsolatedClient struct { |
| 53 | root *ClaimLaunchRoot |
| 54 | peer StopPeer |
| 55 | } |
| 56 | |
| 57 | func (c seedIsolatedClient) Capability(context.Context) (string, error) { |
| 58 | return hostbox.RoleLaunchProtocol, nil |
| 59 | } |
| 60 | func (c seedIsolatedClient) Launch(ctx context.Context, q hostbox.RoleLaunchRequest) (hostbox.RoleLaunchGrant, error) { |
| 61 | out, err := c.root.Serve(ctx, c.peer, q) |
| 62 | if err != nil { |
| 63 | return hostbox.RoleLaunchGrant{}, err |
| 64 | } |
| 65 | return *out.Grant, nil |
| 66 | } |
| 67 | func (c seedIsolatedClient) Stop(ctx context.Context, q hostbox.RoleLaunchRequest) error { |
| 68 | _, err := c.root.Serve(ctx, c.peer, q) |
| 69 | return err |
| 70 | } |
| 71 | |
| 72 | func rootFilesSeedIsolated(t *testing.T, binary string) { |
| 73 | witness := rootFilesSeedIsolatedWitness(t, binary) |
| 74 | runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{witness}) |
| 75 | } |
| 76 | |
| 77 | func rootFilesSeedIsolatedWitness(t *testing.T, binary string, twoHosts ...bool) filesSeedLiveWitness { |
| 78 | hosts := []string{"seed-isolated.bab.oncarlos.com"} |
| 79 | if len(twoHosts) > 0 { |
| 80 | hosts = seedTwoHostNames() |
| 81 | } |
| 82 | for _, host := range hosts { |
| 83 | path := "/run/systemd/system.control/" + instancectl.SliceName(host) + ".d" |
| 84 | if _, err := os.Lstat(path); !os.IsNotExist(err) { |
| 85 | t.Fatal("refuse existing slice properties", path, err) |
| 86 | } |
| 87 | } |
| 88 | for _, dir := range []string{"/data", "/var/lib/carlos-claims", "/opt/carlos", "/run/carlos", "/run/carlos-control", "/etc/carlos"} { |
| 89 | if len(twoHosts) > 1 && (dir == "/data" || dir == "/var/lib/carlos-claims" || dir == "/opt/carlos" || dir == "/etc/carlos") { |
| 90 | if _, err := os.Stat(dir); err != nil { |
| 91 | t.Fatal("missing reboot fixture", dir, err) |
| 92 | } |
| 93 | } else if err := os.Mkdir(dir, 0755); err != nil { |
| 94 | t.Fatal("refuse existing fixture path", dir, err) |
| 95 | } |
| 96 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 97 | if dir == "/data" && strings.HasPrefix(t.Name(), "TestRootRebootFilesSeedTwoHost") { |
| 98 | seedRunnerRebootDisk(t) |
| 99 | } |
| 100 | } |
| 101 | account, err := user.Lookup("carlos") |
| 102 | if err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | uid, _ := strconv.Atoi(account.Uid) |
| 106 | gid, _ := strconv.Atoi(account.Gid) |
| 107 | for _, dir := range []string{"/opt/carlos/versions/fixture", "/run/carlos/instances", "/run/carlos/logs"} { |
| 108 | if err := os.MkdirAll(dir, 0755); err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | } |
| 112 | image, err := os.ReadFile(binary) |
| 113 | if err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | app := "/opt/carlos/versions/fixture/app" |
| 117 | for _, path := range []string{"/opt/carlos/carlos", app} { |
| 118 | if err := os.WriteFile(path, image, 0755); err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | } |
| 122 | raw, err := unitsFS.ReadFile("units/carlos-instance@.service") |
| 123 | if err != nil { |
| 124 | t.Fatal(err) |
| 125 | } |
| 126 | lines := []string{} |
| 127 | for _, line := range strings.Split(string(raw), "\n") { |
| 128 | if strings.HasPrefix(line, "BindsTo=") || strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") { |
| 129 | continue |
| 130 | } |
| 131 | lines = append(lines, line) |
| 132 | } |
| 133 | if os.Getenv("FILES_SEED_ISOLATION_MUTATION") == "1" { |
| 134 | lines = append(lines, "[Service]", "BindReadOnlyPaths=/etc/carlos/fixture-secret:/tmp/exposed-root") |
| 135 | } |
| 136 | unit := "/run/systemd/system/carlos-instance@.service" |
| 137 | file, err := os.OpenFile(unit, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) |
| 138 | if err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | if _, err := file.WriteString(strings.Join(lines, "\n") + "\nIPAddressDeny=any\n"); err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | file.Close() |
| 145 | t.Cleanup(func() { |
| 146 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
| 147 | defer cancel() |
| 148 | for _, host := range hosts { |
| 149 | if out, err := exec.CommandContext(ctx, "systemctl", "stop", instancectl.UnitName(instancectl.RoleRun, host)).CombinedOutput(); err != nil { |
| 150 | t.Errorf("stop isolated tenant: %v %s", err, out) |
| 151 | } |
| 152 | os.RemoveAll("/run/systemd/system/" + instancectl.UnitName(instancectl.RoleRun, host) + ".d") |
| 153 | slice := instancectl.SliceName(host) |
| 154 | if out, err := exec.CommandContext(ctx, "systemctl", "stop", slice).CombinedOutput(); err != nil { |
| 155 | t.Errorf("stop isolated slice: %v %s", err, out) |
| 156 | } |
| 157 | state, err := exec.CommandContext(ctx, "systemctl", "show", slice, "--property=ActiveState", "--value").Output() |
| 158 | if err != nil || strings.TrimSpace(string(state)) != "inactive" { |
| 159 | t.Errorf("isolated slice survived cleanup: %s %v", state, err) |
| 160 | } |
| 161 | exec.CommandContext(ctx, "systemctl", "reset-failed", instancectl.UnitName(instancectl.RoleRun, host)).Run() |
| 162 | path := "/run/systemd/system.control/" + slice + ".d" |
| 163 | if err := os.RemoveAll(path); err != nil { |
| 164 | t.Error("remove fixture slice properties", err) |
| 165 | } |
| 166 | if _, err := os.Lstat(path); !os.IsNotExist(err) { |
| 167 | t.Error("fixture slice properties survived", err) |
| 168 | } |
| 169 | } |
| 170 | for _, slice := range []string{"carlos-instances.slice", "carlos.slice"} { |
| 171 | if out, err := exec.CommandContext(ctx, "systemctl", "stop", slice).CombinedOutput(); err != nil { |
| 172 | t.Errorf("stop isolated parent slice: %v %s", err, out) |
| 173 | } |
| 174 | state, err := exec.CommandContext(ctx, "systemctl", "show", slice, "--property=ActiveState", "--value").Output() |
| 175 | if err != nil || strings.TrimSpace(string(state)) != "inactive" { |
| 176 | t.Errorf("isolated parent slice survived cleanup: %s %v", state, err) |
| 177 | } |
| 178 | } |
| 179 | os.Remove(unit) |
| 180 | if out, err := exec.CommandContext(ctx, "systemctl", "daemon-reload").CombinedOutput(); err != nil { |
| 181 | t.Errorf("reload isolated unit: %v %s", err, out) |
| 182 | } |
| 183 | }) |
| 184 | servingRunnerSystemctl(t, "daemon-reload") |
| 185 | denied := []string{"/etc/carlos/fixture-secret", "/var/lib/carlos-claims/fixture-secret", "/data/instances/foreign.example/fixture-secret"} |
| 186 | for _, path := range denied { |
| 187 | if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { |
| 188 | t.Fatal(err) |
| 189 | } |
| 190 | if err := os.WriteFile(path, []byte("foreign sentinel"), 0644); err != nil { |
| 191 | t.Fatal(err) |
| 192 | } |
| 193 | } |
| 194 | agent := "/sys/fs/cgroup/files-seed-controller" |
| 195 | if err := os.Mkdir(agent, 0755); err != nil { |
| 196 | t.Fatal("refuse existing controller cgroup", err) |
| 197 | } |
| 198 | t.Cleanup(func() { |
| 199 | if err := os.Remove(agent); err != nil { |
| 200 | t.Error(err) |
| 201 | } |
| 202 | }) |
| 203 | witness := rootFilesSeedLiveWitness(t, app) |
| 204 | witness.host = "seed-isolated.bab.oncarlos.com" |
| 205 | configure := witness.configure |
| 206 | witness.configure = func(w *ClaimWakeAssembly) { |
| 207 | configure(w) |
| 208 | owners := w.Inputs.(*claimServingOwners) |
| 209 | selected := owners.Sources.Launch |
| 210 | owners.Sources.Launch = func(ctx context.Context, rt registry.Route) (ClaimServingLaunch, error) { |
| 211 | in, err := selected(ctx, rt) |
| 212 | paths := append([]string(nil), denied...) |
| 213 | if os.Getenv("FILES_SEED_ISOLATION_MUTATION") == "1" { |
| 214 | paths = append(paths, "/tmp/exposed-root") |
| 215 | } |
| 216 | if len(twoHosts) > 0 { |
| 217 | pair := seedTwoHostNames() |
| 218 | other := pair[1] |
| 219 | if rt.Host == other { |
| 220 | other = pair[0] |
| 221 | } |
| 222 | paths = append(paths, provision.NestedDBPath("/data", other), filepath.Join("/data/instances", other, other, "payload")) |
| 223 | in.Env = append(in.Env, "FILES_SEED_EXPECTED=real seeded bytes:"+rt.Host) |
| 224 | } |
| 225 | in.Env = append(in.Env, "FILES_SEED_DENIED="+strings.Join(paths, ":")) |
| 226 | return in, err |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | witness.healthy = true |
| 231 | witness.prepare = func(p *servingPreparation) { |
| 232 | boot, err := execlease.BootID() |
| 233 | if err != nil { |
| 234 | t.Fatal(err) |
| 235 | } |
| 236 | start, err := claimLaunchProcessStart("/proc", os.Getpid()) |
| 237 | if err != nil { |
| 238 | t.Fatal(err) |
| 239 | } |
| 240 | p.local.BootID, p.local.ControllerID, p.local.PID, p.local.ProcessStart = boot, "isolated-controller", os.Getpid(), start |
| 241 | p.f.f.peer = StopPeer{UID: uint32(uid), PID: os.Getpid(), ProcessStart: start, BootID: boot, ControllerID: p.local.ControllerID} |
| 242 | p.wake.Retries.BootID, p.wake.Retries.ControllerID = boot, p.local.ControllerID |
| 243 | p.f.p.root.Stop.Coordinator.CarlosUID = uint32(uid) |
| 244 | prior := claimContainmentAgentRoot |
| 245 | claimContainmentAgentRoot = func(context.Context) (string, error) { return agent, nil } |
| 246 | t.Cleanup(func() { claimContainmentAgentRoot = prior }) |
| 247 | p.wake.Prelaunch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs) (*activator.RoleLaunch, error) { |
| 248 | target := recoveryTuple(o, p.local) |
| 249 | group, err := claimContainmentParent(agent, target) |
| 250 | if err != nil { |
| 251 | return nil, err |
| 252 | } |
| 253 | path := filepath.Join("/sys/fs/cgroup", group) |
| 254 | if err := os.Mkdir(path, 0755); err != nil { |
| 255 | return nil, err |
| 256 | } |
| 257 | t.Cleanup(func() { os.Remove(path) }) |
| 258 | var st unix.Stat_t |
| 259 | if err := unix.Stat(path, &st); err != nil { |
| 260 | return nil, err |
| 261 | } |
| 262 | err = work.With(ctx, in.Identity, func() error { |
| 263 | return p.f.f.journal.Prelaunch(ctx, hostbox.LaunchInventory{Target: target, ControllerPID: p.local.PID, ProcessStart: start, NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: group, Device: uint64(st.Dev), Inode: st.Ino}}}) |
| 264 | }) |
| 265 | return &activator.RoleLaunch{Target: target, Journal: p.f.f.journal}, err |
| 266 | } |
| 267 | state, err := rootstate.Open(filepath.Join(t.TempDir(), "execution"), true, false) |
| 268 | if err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | t.Cleanup(func() { state.Close() }) |
| 272 | census, _ := json.Marshal(rootparticipation.Census{Schema: 1, Box: p.f.actor.Box, Enrollment: p.f.actor.Enrollment, Migration: 1, Revision: 1, LaunchRevision: 1, Complete: true}) |
| 273 | if err := state.Write("census.json", census); err != nil { |
| 274 | t.Fatal(err) |
| 275 | } |
| 276 | units := instancectl.ClaimUnits{Systemctl: func(ctx context.Context, args ...string) (string, error) { |
| 277 | out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() |
| 278 | t.Logf("isolated systemctl %q: %s (%v)", args, out, err) |
| 279 | return string(out), err |
| 280 | }, CgroupRoot: "/sys/fs/cgroup", ExeRoot: "/run/carlos-control/exe", UnitRoot: "/run/systemd/system", DataRoot: "/data"} |
| 281 | root := &ClaimLaunchRoot{Guard: p.f.f.guard, Journal: p.f.f.journal, DataRoot: "/data", ExecLockDir: t.TempDir(), Local: &rev2Local{}, Clock: execlease.SystemClock{}, Actor: func(context.Context) (hostbox.Actor, error) { return p.f.actor, nil }, Units: units, Kernel: linuxClaimLaunchKernel{OptRoot: "/opt/carlos", CgroupRoot: "/sys/fs/cgroup", ProcRoot: "/proc"}, ValidateExecutable: func(string, string, string) error { return nil }, executionState: state} |
| 282 | client := seedIsolatedClient{root: root, peer: p.f.f.peer} |
| 283 | p.wake.Launch = func(ctx context.Context, o hostclaim.Observed, work hostwrite.Work, in ClaimLaunchInputs, pre *activator.RoleLaunch) (*activator.RoleLaunch, error) { |
| 284 | paths, err := claimIsolatedPaths(map[string]string{"data": "/data"}, o.Host, in) |
| 285 | if err != nil { |
| 286 | return nil, err |
| 287 | } |
| 288 | l := *pre |
| 289 | l.Work, l.Identity, l.Root, l.CgroupRoot = work, in.Identity, client, "/sys/fs/cgroup" |
| 290 | l.Spec = &hostbox.LaunchSpec{Identity: in.Identity, Executable: in.Executable, Digest: in.ExecutableSHA256, Args: in.Args, Env: in.Env} |
| 291 | l.LaunchDir, l.RunDir, l.LogSocket, l.MemoryMax = paths.LaunchDir, paths.RunDir, paths.LogSocket, paths.MemoryMax |
| 292 | l.Executable = func(context.Context, *exec.Cmd) (*os.File, error) { return os.Open(app) } |
| 293 | l.Initialize = func(context.Context) error { |
| 294 | if err := filepath.Walk(l.LaunchDir, func(path string, info os.FileInfo, err error) error { |
| 295 | if err != nil { |
| 296 | return err |
| 297 | } |
| 298 | return os.Chown(path, uid, gid) |
| 299 | }); err != nil { |
| 300 | return err |
| 301 | } |
| 302 | if err := os.MkdirAll(l.RunDir, 0750); err != nil { |
| 303 | return err |
| 304 | } |
| 305 | return os.Chown(l.RunDir, uid, gid) |
| 306 | } |
| 307 | return &l, nil |
| 308 | } |
| 309 | if len(p.wake.pending) != 0 { |
| 310 | t.Fatal("fixture rebuilt active wake assembly") |
| 311 | } |
| 312 | p.wake.pending = nil |
| 313 | p.runtime, err = p.wake.Build(t.Context(), p.source, p.writer) |
| 314 | if err != nil { |
| 315 | t.Fatal(err) |
| 316 | } |
| 317 | } |
| 318 | witness.start = func(t *testing.T, p *servingPreparation, o hostclaim.Observed, start activator.ClaimStart) func() { |
| 319 | launch, err := start.PrepareLaunch(t.Context()) |
| 320 | if err != nil { |
| 321 | t.Fatal("prepare isolated launch", err) |
| 322 | } |
| 323 | cmd, _, settle, _, err := launch.Start(t.Context(), "main", start.CurrentAttempt(), start.Build, func() error { return nil }) |
| 324 | if err != nil { |
| 325 | if body, readErr := os.ReadFile(filepath.Join(filepath.Dir(p.f.db), o.Host, "payload.isolation-violation")); readErr == nil { |
| 326 | t.Fatalf("isolation mutation detected: %s", body) |
| 327 | } |
| 328 | out, _ := exec.Command("journalctl", "-u", instancectl.UnitName(instancectl.RoleRun, o.Host), "--no-pager", "-n", "30").CombinedOutput() |
| 329 | t.Fatalf("isolated launch: %v\n%s", err, out) |
| 330 | } |
| 331 | stopped := false |
| 332 | stop := func() { |
| 333 | if !stopped { |
| 334 | stopped = true |
| 335 | if err := settle(context.Background()); err != nil { |
| 336 | t.Error(err) |
| 337 | } |
| 338 | } |
| 339 | } |
| 340 | t.Cleanup(stop) |
| 341 | ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) |
| 342 | defer cancel() |
| 343 | if err := start.Ready(ctx, cmd); err != nil { |
| 344 | if body, readErr := os.ReadFile(filepath.Join(filepath.Dir(p.f.db), o.Host, "payload.isolation-violation")); readErr == nil { |
| 345 | t.Fatalf("isolation mutation detected: %s", body) |
| 346 | } |
| 347 | t.Fatal("isolated readiness", err) |
| 348 | } |
| 349 | wrong, _ := os.FindProcess(os.Getpid()) |
| 350 | if err := start.Healthy(ctx, 0, &exec.Cmd{Process: wrong}); err == nil { |
| 351 | t.Fatal("controller accepted as healthy tenant") |
| 352 | } |
| 353 | if err := start.Healthy(ctx, 0, cmd); err != nil { |
| 354 | t.Fatal("isolated exact health", err) |
| 355 | } |
| 356 | transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { |
| 357 | return (&net.Dialer{}).DialContext(ctx, "unix", provision.NestedSocket(o.Host)) |
| 358 | }} |
| 359 | defer transport.CloseIdleConnections() |
| 360 | response, err := (&http.Client{Transport: transport, Timeout: time.Second}).Get("http://tenant/") |
| 361 | if err != nil { |
| 362 | t.Fatal(err) |
| 363 | } |
| 364 | body, err := io.ReadAll(response.Body) |
| 365 | response.Body.Close() |
| 366 | expected := "real seeded bytes" |
| 367 | if len(twoHosts) > 0 { |
| 368 | expected += ":" + o.Host |
| 369 | } |
| 370 | if err != nil || response.StatusCode != 200 || string(body) != expected { |
| 371 | t.Fatal("isolated payload", string(body), err) |
| 372 | } |
| 373 | for _, path := range denied { |
| 374 | if body, err := os.ReadFile(path); err != nil || string(body) != "foreign sentinel" { |
| 375 | t.Fatal("foreign sentinel changed", path, err) |
| 376 | } |
| 377 | } |
| 378 | t.Log(fmt.Sprintf("healthy isolated wake: host=%s session=%d pid=%d", o.Host, o.Claim.Holder.Session, cmd.Process.Pid)) |
| 379 | return stop |
| 380 | } |
| 381 | witness.serve = func(t *testing.T, p *servingPreparation, o hostclaim.Observed, start activator.ClaimStart) { |
| 382 | witness.start(t, p, o, start)() |
| 383 | } |
| 384 | return witness |
| 385 | } |
| 386 | |
amadan