| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/cgroup" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | ) |
| 18 | |
| 19 | // The attempt's chosen bytes are the bytes that run (spec §7.5 step 3, |
| 20 | // §4.2): verifyStaged admits a regular file whose sha256 matches, and |
| 21 | // refuses everything it cannot compare. |
| 22 | func TestVerifyStaged(t *testing.T) { |
| 23 | dir := t.TempDir() |
| 24 | bin := filepath.Join(dir, "app") |
| 25 | if err := os.WriteFile(bin, []byte("build v2"), 0o755); err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | sum := sha256.Sum256([]byte("build v2")) |
| 29 | good := hex.EncodeToString(sum[:]) |
| 30 | if err := verifyStaged(bin, good); err != nil { |
| 31 | t.Fatalf("matching digest refused: %v", err) |
| 32 | } |
| 33 | if err := verifyStaged(bin, strings.ToUpper(good)); err != nil { |
| 34 | t.Fatalf("upper-case hex refused: %v", err) |
| 35 | } |
| 36 | link := filepath.Join(dir, "live") |
| 37 | if err := os.Symlink(bin, link); err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | for name, tc := range map[string]struct{ path, digest string }{ |
| 41 | "mismatch": {bin, strings.Repeat("0", 64)}, |
| 42 | "empty digest": {bin, ""}, |
| 43 | "short digest": {bin, "abc"}, |
| 44 | "relative path": {"app", good}, |
| 45 | "through a symlink": {link, good}, |
| 46 | "missing": {filepath.Join(dir, "gone"), good}, |
| 47 | "directory": {dir, good}, |
| 48 | } { |
| 49 | if err := verifyStaged(tc.path, tc.digest); err == nil { |
| 50 | t.Errorf("%s: accepted", name) |
| 51 | } |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | // recordingGenExec is an exec backend that can run staged generations. |
| 56 | type recordingGenExec struct { |
| 57 | *fakeBackend |
| 58 | started int |
| 59 | } |
| 60 | |
| 61 | func (r *recordingGenExec) StartStaged(ctx context.Context, _ registry.Route, g Gen) (Gen, Proc, error) { |
| 62 | if err := runExecGate(ctx); err != nil { |
| 63 | return g, nil, err |
| 64 | } |
| 65 | r.started++ |
| 66 | return g, &fakeProc{}, nil |
| 67 | } |
| 68 | func (r *recordingGenExec) GenTier(registry.Route, string) cgroup.Limit { return cgroup.TierDefault } |
| 69 | func (r *recordingGenExec) GenReady(registry.Route, Gen, Proc) {} |
| 70 | func (r *recordingGenExec) RaiseGenParent(context.Context, registry.Route, cgroup.Limit, cgroup.Limit) error { |
| 71 | return nil |
| 72 | } |
| 73 | func (r *recordingGenExec) RestoreGenParent(context.Context, registry.Route, cgroup.Limit) error { |
| 74 | return nil |
| 75 | } |
| 76 | func (r *recordingGenExec) GenSwitched(registry.Route, Gen, Proc) error { return nil } |
| 77 | |
| 78 | // Swaps are for exec rows only (spec §2 decision 9): a unit-backed row is |
| 79 | // refused, and an exec half that cannot run staged generations refuses |
| 80 | // rather than falling back to anything. |
| 81 | func TestSplitBackendSwapsAreExecOnly(t *testing.T) { |
| 82 | ctx := context.Background() |
| 83 | exec := &recordingGenExec{fakeBackend: &fakeBackend{}} |
| 84 | s := &SplitBackend{Exec: exec, Unit: &UnitBackend{}} |
| 85 | unitRt := registry.Route{Host: "u.example.com", Backing: registry.BackingUnit} |
| 86 | if _, _, err := s.StartStaged(ctx, unitRt, Gen{N: 1}); !errors.Is(err, errUnitNoGens) { |
| 87 | t.Fatalf("unit row StartStaged: %v, want errUnitNoGens", err) |
| 88 | } |
| 89 | if err := s.RaiseGenParent(ctx, unitRt, cgroup.TierDefault, cgroup.TierDefault); !errors.Is(err, errUnitNoGens) { |
| 90 | t.Fatalf("unit row RaiseGenParent: %v", err) |
| 91 | } |
| 92 | execRt := registry.Route{Host: "e.example.com"} |
| 93 | g, _, err := s.StartStaged(ctx, execRt, Gen{N: 7, StagedPath: "/opt/carlos/versions/v2/app"}) |
| 94 | if err != nil || g.N != 7 || exec.started != 1 { |
| 95 | t.Fatalf("exec row StartStaged = %+v, %v (started %d)", g, err, exec.started) |
| 96 | } |
| 97 | plain := &SplitBackend{Exec: &fakeBackend{}, Unit: &UnitBackend{}} |
| 98 | if _, _, err := plain.StartStaged(ctx, execRt, Gen{N: 1}); !errors.Is(err, errSwapNoGens) { |
| 99 | t.Fatalf("exec half without staged generations: %v, want errSwapNoGens", err) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // A flat staged generation execs the attempt's staged path, not the live |
| 104 | // link, on its own generation socket (spec §7.5 step 3). The fake binary |
| 105 | // never binds. What it recorded as argv[0] is the proof. |
| 106 | func TestExecBackendStartStagedExecsTheStagedPath(t *testing.T) { |
| 107 | b := testExecBackend(t) |
| 108 | b.RunDirOverride = t.TempDir() |
| 109 | host := "staged.example.com" |
| 110 | stageLive(t, b, host, "v1") // the live link names v1 throughout |
| 111 | marker := filepath.Join(t.TempDir(), "argv0") |
| 112 | v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| 113 | if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | if err := os.WriteFile(v2, []byte("#!/bin/sh\nprintf '%s' \"$0\" > '"+marker+"'\n"), 0o755); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | rt := registry.Route{Host: host, Kind: "instance", DBPath: filepath.Join(b.DataDir, host+".db"), |
| 120 | Socket: filepath.Join(b.RunDirOverride, host+".sock")} // flat under the override |
| 121 | _, proc, err := b.StartStaged(context.Background(), rt, Gen{N: 5, Build: "v2", StagedPath: v2}) |
| 122 | if err != nil { |
| 123 | t.Fatalf("StartStaged: %v", err) |
| 124 | } |
| 125 | defer proc.Stop(context.Background()) //nolint:errcheck |
| 126 | if got := waitForFile(t, marker, 2*time.Second); got != v2 { |
| 127 | t.Fatalf("the generation execd %q, want the staged %q", got, v2) |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | // StartStaged is a spawn, not a health check (spec §8.2): it hands back the |
| 132 | // running process at once, so the caller's own deadline governs and a build |
| 133 | // that never answers is the caller's to record and kill. This runs the real |
| 134 | // flat spawn with the real 30 s spawnHealthTimeout. A StartStaged that still |
| 135 | // waits for health inside blocks for those 30 s and then returns a nil |
| 136 | // handle, which a planned wake reads as "nothing executed". |
| 137 | func TestExecBackendStartStagedReturnsTheProcessAtOnce(t *testing.T) { |
| 138 | b := testExecBackend(t) |
| 139 | b.RunDirOverride = t.TempDir() |
| 140 | host := "silent.example.com" |
| 141 | stageLive(t, b, host, "v1") |
| 142 | v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| 143 | if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | if err := os.WriteFile(v2, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil { // never binds |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | rt := registry.Route{Host: host, Kind: "instance", DBPath: filepath.Join(b.DataDir, host+".db"), |
| 150 | Socket: filepath.Join(b.RunDirOverride, host+".sock")} |
| 151 | type out struct { |
| 152 | g Gen |
| 153 | proc Proc |
| 154 | err error |
| 155 | } |
| 156 | done := make(chan out, 1) |
| 157 | go func() { |
| 158 | g, p, err := b.StartStaged(context.Background(), rt, Gen{N: 6, Build: "v2", StagedPath: v2}) |
| 159 | done <- out{g, p, err} |
| 160 | }() |
| 161 | var o out |
| 162 | select { |
| 163 | case o = <-done: |
| 164 | case <-time.After(5 * time.Second): |
| 165 | t.Fatal("StartStaged waited for health: it must return the running process at once") |
| 166 | } |
| 167 | if o.err != nil || o.proc == nil { |
| 168 | t.Fatalf("StartStaged = %v, %v: want the running process", o.proc, o.err) |
| 169 | } |
| 170 | defer o.proc.Stop(context.Background()) //nolint:errcheck |
| 171 | if o.proc.Exited() { |
| 172 | t.Fatal("the returned process has already exited") |
| 173 | } |
| 174 | if o.g.Socket != b.flatGenSocket(host, 6) { |
| 175 | t.Fatalf("Socket = %q, want generation 6's", o.g.Socket) |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | // A foreign row keeps its canonical socket (plan 1): a cold, pinned or hold |
| 180 | // start execs the staged build there, one generation at a time. Refusing it |
| 181 | // would break every planned wake of a foreign row once the flag is on. |
| 182 | func TestExecBackendStartStagedRunsAForeignRowOnItsCanonicalSocket(t *testing.T) { |
| 183 | b := testExecBackend(t) |
| 184 | host := "f.example.com" |
| 185 | marker := filepath.Join(t.TempDir(), "argv0") |
| 186 | v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| 187 | if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| 188 | t.Fatal(err) |
| 189 | } |
| 190 | if err := os.WriteFile(v2, []byte("#!/bin/sh\nprintf '%s' \"$0\" > '"+marker+"'\n"), 0o755); err != nil { |
| 191 | t.Fatal(err) |
| 192 | } |
| 193 | rt := registry.Route{Host: host, Kind: "instance", DBPath: filepath.Join(b.DataDir, host+".db"), |
| 194 | Socket: filepath.Join(t.TempDir(), "odd.sock")} // hand-set: LayoutForeign |
| 195 | if l, _ := b.Layout(rt); l != LayoutForeign { |
| 196 | t.Fatalf("fixture layout = %v, want LayoutForeign", l) |
| 197 | } |
| 198 | g, proc, err := b.StartStaged(context.Background(), rt, Gen{N: 1, Build: "v2", StagedPath: v2}) |
| 199 | if err != nil { |
| 200 | t.Fatalf("foreign row: %v", err) |
| 201 | } |
| 202 | defer proc.Stop(context.Background()) //nolint:errcheck |
| 203 | if g.Socket != rt.Socket || g.Control != rt.Socket+".control" { |
| 204 | t.Fatalf("foreign generation sockets %q %q, want the canonical %q", g.Socket, g.Control, rt.Socket) |
| 205 | } |
| 206 | if got := waitForFile(t, marker, 2*time.Second); got != v2 { |
| 207 | t.Fatalf("the generation execd %q, want the staged %q", got, v2) |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | // A link left at a foreign row's canonical path by an agent that ran with |
| 212 | // generations on would fail the app's bind, so a staged start there clears |
| 213 | // it first, as StartGen does. |
| 214 | func TestExecBackendStartStagedClearsALeftoverCanonicalLink(t *testing.T) { |
| 215 | b := testExecBackend(t) |
| 216 | host := "l.example.com" |
| 217 | v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| 218 | if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| 219 | t.Fatal(err) |
| 220 | } |
| 221 | if err := os.WriteFile(v2, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | rt := registry.Route{Host: host, Kind: "instance", DBPath: filepath.Join(b.DataDir, host+".db"), |
| 225 | Socket: filepath.Join(t.TempDir(), "odd.sock")} |
| 226 | if err := os.Symlink("/nonexistent/g/old.sock", rt.Socket); err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | _, proc, err := b.StartStaged(context.Background(), rt, Gen{N: 2, Build: "v2", StagedPath: v2}) |
| 230 | if err != nil { |
| 231 | t.Fatalf("foreign row: %v", err) |
| 232 | } |
| 233 | defer proc.Stop(context.Background()) //nolint:errcheck |
| 234 | if _, err := os.Lstat(rt.Socket); !errors.Is(err, os.ErrNotExist) { |
| 235 | t.Fatalf("the leftover link at the canonical path survived: %v", err) |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | // Each artifact's own tier (spec §7.3): a staged child gets g.Tier, not the |
| 240 | // live link's tier, and the overlap parent is the incumbent's tier plus the |
| 241 | // successor's. Here the live link (b.Tier) says 512 MiB throughout, while |
| 242 | // the incumbent declares children (1 GiB) and the successor does not. Using |
| 243 | // b.tier(rt) for both terms gives a 1 GiB parent, and an incumbent that |
| 244 | // fills its own 1 GiB is OOM-killed by the successor's first allocation. |
| 245 | func TestExecBackendUsesEachArtifactsTier(t *testing.T) { |
| 246 | const host = "t.example.com" |
| 247 | b, parent := delegatedBackend(t, host) |
| 248 | cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| 249 | b.GenContainment = true |
| 250 | b.Tier = func(registry.Route) cgroup.Limit { return cgroup.TierDefault } |
| 251 | b.TierOf = func(_ registry.Route, v string) cgroup.Limit { |
| 252 | if v == "v1" { |
| 253 | return cgroup.TierChildren |
| 254 | } |
| 255 | return cgroup.TierDefault |
| 256 | } |
| 257 | rt := registry.Route{Host: host} |
| 258 | if got := b.GenTier(rt, "v1"); got != cgroup.TierChildren { |
| 259 | t.Fatalf("GenTier(v1) = %v, want the children tier", got) |
| 260 | } |
| 261 | if err := b.RaiseGenParent(context.Background(), rt, b.GenTier(rt, "v1"), b.GenTier(rt, "v2")); err != nil { |
| 262 | t.Fatal(err) |
| 263 | } |
| 264 | if got, err := os.ReadFile(filepath.Join(parent, "memory.max")); err != nil || string(got) != "1610612736" { |
| 265 | t.Fatalf("overlap parent memory.max = %q, %v; want 1 GiB + 512 MiB", got, err) |
| 266 | } |
| 267 | cmd, settle, err := b.spawnInCgroupAtTier(rt, "g9", true, cgroup.TierChildren, func() *exec.Cmd { return exec.Command("/bin/true") }, nil) |
| 268 | if err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | _ = cmd.Wait() |
| 272 | settle() |
| 273 | if got, err := os.ReadFile(filepath.Join(parent, "g9", "memory.max")); err != nil || string(got) != "1073741824" { |
| 274 | t.Fatalf("g9 memory.max = %q, %v; want its own artifact's 1 GiB", got, err) |
| 275 | } |
| 276 | if err := b.RestoreGenParent(context.Background(), rt, cgroup.TierChildren); err != nil { |
| 277 | t.Fatal(err) |
| 278 | } |
| 279 | if got, _ := os.ReadFile(filepath.Join(parent, "memory.max")); string(got) != "1073741824" { |
| 280 | t.Fatalf("restored parent memory.max = %q, want the current generation's 1 GiB", got) |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | // The switch points the canonical path at the new generation's socket |
| 285 | // (spec §7.1, rev 15), for dialers outside the agent (#103). |
| 286 | func TestExecBackendGenSwitchedRepointsTheCanonicalLink(t *testing.T) { |
| 287 | b := testExecBackend(t) |
| 288 | dir := t.TempDir() |
| 289 | rt := registry.Route{Host: "c.example.com", Socket: filepath.Join(dir, "c.example.com.sock")} |
| 290 | for _, n := range []uint64{1, 2} { |
| 291 | g := Gen{N: n, Socket: filepath.Join(dir, "g", "c."+strings.Repeat("x", int(n))+".sock")} |
| 292 | if err := b.GenSwitched(rt, g, &fakeProc{}); err != nil { |
| 293 | t.Fatal(err) |
| 294 | } |
| 295 | if got, err := os.Readlink(rt.Socket); err != nil || got != g.Socket { |
| 296 | t.Fatalf("after switch %d the canonical path names %q (%v), want %q", n, got, err, g.Socket) |
| 297 | } |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | // Without plan 2's per-generation containment there is no parent to raise, |
| 302 | // and the swap is refused before anything spawns. |
| 303 | func TestExecBackendRaiseNeedsContainment(t *testing.T) { |
| 304 | b := testExecBackend(t) |
| 305 | if err := b.RaiseGenParent(context.Background(), registry.Route{Host: "h.example.com"}, cgroup.TierDefault, cgroup.TierDefault); !errors.Is(err, errNoGenContainment) { |
| 306 | t.Fatalf("RaiseGenParent without GenContainment: %v", err) |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | var ( |
| 311 | _ SwapBackend = (*ExecBackend)(nil) |
| 312 | _ SwapBackend = (*SplitBackend)(nil) |
| 313 | ) |
| 314 | |
amadan