| 1 | //go:build linux |
| 2 | |
| 3 | package maintenanceexec |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/base64" |
| 9 | "encoding/hex" |
| 10 | "encoding/json" |
| 11 | "errors" |
| 12 | "io" |
| 13 | "os" |
| 14 | "os/exec" |
| 15 | "path/filepath" |
| 16 | "reflect" |
| 17 | "strconv" |
| 18 | "strings" |
| 19 | "sync" |
| 20 | "syscall" |
| 21 | "testing" |
| 22 | "time" |
| 23 | |
| 24 | "github.com/carlosframework/platform/internal/maintenance" |
| 25 | "golang.org/x/sys/unix" |
| 26 | ) |
| 27 | |
| 28 | // This fixture exercises actual exec and pipe behavior. Its process-group |
| 29 | // cleanup is a test substitute, not native cgroup containment evidence. |
| 30 | type fixtureGroup struct { |
| 31 | cmd *exec.Cmd |
| 32 | starts, kills int |
| 33 | startErr, killErr error |
| 34 | before func() |
| 35 | } |
| 36 | |
| 37 | func (g *fixtureGroup) Start(c *exec.Cmd) error { |
| 38 | g.starts++ |
| 39 | g.cmd = c |
| 40 | if g.before != nil { |
| 41 | g.before() |
| 42 | } |
| 43 | if g.startErr != nil { |
| 44 | return g.startErr |
| 45 | } |
| 46 | c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} |
| 47 | return c.Start() |
| 48 | } |
| 49 | func (g *fixtureGroup) Populated() (bool, error) { |
| 50 | if g.cmd == nil || g.cmd.Process == nil { |
| 51 | return false, nil |
| 52 | } |
| 53 | entries, err := os.ReadDir("/proc") |
| 54 | if err != nil { |
| 55 | return false, err |
| 56 | } |
| 57 | for _, entry := range entries { |
| 58 | if _, err := strconv.Atoi(entry.Name()); err != nil { |
| 59 | continue |
| 60 | } |
| 61 | raw, err := os.ReadFile("/proc/" + entry.Name() + "/stat") |
| 62 | if err != nil { |
| 63 | continue |
| 64 | } |
| 65 | fields := strings.Fields(string(raw)[strings.LastIndexByte(string(raw), ')')+1:]) |
| 66 | if len(fields) >= 3 && fields[2] == strconv.Itoa(g.cmd.Process.Pid) && fields[0] != "Z" { |
| 67 | return true, nil |
| 68 | } |
| 69 | } |
| 70 | return false, nil |
| 71 | } |
| 72 | func (g *fixtureGroup) KillAndEmpty(ctx context.Context) error { |
| 73 | if ctx.Err() != nil { |
| 74 | return ctx.Err() |
| 75 | } |
| 76 | g.kills++ |
| 77 | if g.cmd != nil && g.cmd.Process != nil { |
| 78 | unix.Kill(-g.cmd.Process.Pid, unix.SIGKILL) |
| 79 | } |
| 80 | return g.killErr |
| 81 | } |
| 82 | |
| 83 | type fixtureLease struct { |
| 84 | mu sync.Mutex |
| 85 | until time.Time |
| 86 | lost chan struct{} |
| 87 | err error |
| 88 | } |
| 89 | |
| 90 | func (l *fixtureLease) SafeUntil() (time.Time, error) { |
| 91 | l.mu.Lock() |
| 92 | defer l.mu.Unlock() |
| 93 | return l.until, l.err |
| 94 | } |
| 95 | func (l *fixtureLease) Lost() <-chan struct{} { return l.lost } |
| 96 | func (l *fixtureLease) lose() { l.mu.Lock(); defer l.mu.Unlock(); l.err = ErrRefused; close(l.lost) } |
| 97 | |
| 98 | func TestChildFixture(t *testing.T) { |
| 99 | mode := os.Getenv("MAINTENANCE_EXEC_FIXTURE") |
| 100 | if mode == "" { |
| 101 | return |
| 102 | } |
| 103 | if len(os.Environ()) != 2 || os.Getenv("GODEBUG") != "containermaxprocs=0,updatemaxprocs=0" { |
| 104 | os.Exit(15) |
| 105 | } |
| 106 | if cwd, err := os.Getwd(); err != nil || cwd != "/" { |
| 107 | os.Exit(16) |
| 108 | } |
| 109 | stderr, err := os.Stderr.Stat() |
| 110 | null, nullErr := os.Stat(os.DevNull) |
| 111 | if err != nil || nullErr != nil || !os.SameFile(stderr, null) { |
| 112 | os.Exit(17) |
| 113 | } |
| 114 | entries, err := os.ReadDir("/proc/self/fd") |
| 115 | if err != nil { |
| 116 | os.Exit(18) |
| 117 | } |
| 118 | for _, entry := range entries { |
| 119 | fd, err := strconv.Atoi(entry.Name()) |
| 120 | if err != nil || fd <= 4 { |
| 121 | continue |
| 122 | } |
| 123 | target, err := os.Readlink("/proc/self/fd/" + entry.Name()) |
| 124 | if errors.Is(err, os.ErrNotExist) || strings.HasPrefix(target, "anon_inode:") { |
| 125 | continue |
| 126 | } |
| 127 | os.Exit(19) |
| 128 | } |
| 129 | if mode == "blocked" { |
| 130 | time.Sleep(time.Minute) |
| 131 | return |
| 132 | } |
| 133 | if mode == "overflow" { |
| 134 | os.Stdout.Write([]byte(strings.Repeat("x", maintenance.MaxResultBytes+1))) |
| 135 | time.Sleep(time.Minute) |
| 136 | return |
| 137 | } |
| 138 | if mode == "nonzero" { |
| 139 | os.Exit(7) |
| 140 | } |
| 141 | if mode == "descendant" { |
| 142 | child := exec.Command("/proc/self/exe", "-test.run=^TestChildFixture$") |
| 143 | child.Env = []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=blocked"} |
| 144 | child.Dir = "/" |
| 145 | child.Stdout = os.Stdout |
| 146 | if child.Start() != nil { |
| 147 | os.Exit(14) |
| 148 | } |
| 149 | } |
| 150 | c, err := maintenance.ReadLaunch(os.NewFile(3, "context")) |
| 151 | if err != nil { |
| 152 | os.Exit(8) |
| 153 | } |
| 154 | pin := os.NewFile(4, "artifact") |
| 155 | info, err := pin.Stat() |
| 156 | if err != nil { |
| 157 | os.Exit(9) |
| 158 | } |
| 159 | self, err := os.Stat("/proc/self/exe") |
| 160 | if err != nil || !os.SameFile(info, self) { |
| 161 | os.Exit(10) |
| 162 | } |
| 163 | flags, err := unix.FcntlInt(pin.Fd(), unix.F_GETFL, 0) |
| 164 | if err != nil || flags&unix.O_ACCMODE != unix.O_RDONLY { |
| 165 | os.Exit(11) |
| 166 | } |
| 167 | raw, err := io.ReadAll(os.Stdin) |
| 168 | if err != nil { |
| 169 | os.Exit(12) |
| 170 | } |
| 171 | sum := sha256.Sum256(raw) |
| 172 | if hex.EncodeToString(sum[:]) != c.InputSHA256 { |
| 173 | os.Exit(13) |
| 174 | } |
| 175 | json.NewEncoder(os.Stdout).Encode(map[string]any{"input_bytes": len(raw), "bound_image": true}) |
| 176 | os.Exit(0) |
| 177 | } |
| 178 | func executionFixture(t *testing.T) (*os.File, maintenance.LaunchContext, maintenance.Input, *fixtureGroup, *fixtureLease) { |
| 179 | t.Helper() |
| 180 | t.Setenv("MAINTENANCE_PARENT_ONLY", "must-not-reach-child") |
| 181 | src, err := os.Open("/proc/self/exe") |
| 182 | if err != nil { |
| 183 | t.Fatal(err) |
| 184 | } |
| 185 | defer src.Close() |
| 186 | path := filepath.Join(t.TempDir(), "image") |
| 187 | dst, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0700) |
| 188 | if err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | hash := sha256.New() |
| 192 | if _, err := io.Copy(io.MultiWriter(dst, hash), src); err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | if err := dst.Close(); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | pin, err := os.Open(path) |
| 199 | if err != nil { |
| 200 | t.Fatal(err) |
| 201 | } |
| 202 | t.Cleanup(func() { pin.Close() }) |
| 203 | input := maintenance.Input{Arguments: []string{"-test.run=^TestChildFixture$"}, Stdin: []byte(`{"test":"` + strings.Repeat("a", 60000) + `"}`)} |
| 204 | sum := sha256.Sum256(input.Stdin) |
| 205 | id := base64.RawURLEncoding.EncodeToString(make([]byte, 32)) |
| 206 | digest := strings.Repeat("a", 64) |
| 207 | c := maintenance.LaunchContext{Version: 1, Account: "bds", App: "home", Host: "home.bds.oncarlos.com", RequestID: id, AttemptID: id, RequestDigest: digest, RecordIdentity: digest, ArtifactDigest: hex.EncodeToString(hash.Sum(nil)), EnvironmentEnvelopeDigest: digest, GuardGeneration: 1, InputSHA256: hex.EncodeToString(sum[:]), DatabasePath: "/synthetic/home.db", DatabaseDevice: 1, DatabaseInode: 2, ExecutionDeadline: time.Now().Add(8 * time.Second).UTC().Format(time.RFC3339Nano)} |
| 208 | return pin, c, input, &fixtureGroup{}, &fixtureLease{until: time.Now().Add(10 * time.Second), lost: make(chan struct{})} |
| 209 | } |
| 210 | func TestPinnedImageAndExactPrivateInput(t *testing.T) { |
| 211 | pin, c, input, g, l := executionFixture(t) |
| 212 | g.before = func() { |
| 213 | if err := os.Rename(pin.Name(), pin.Name()+".old"); err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | if err := os.WriteFile(pin.Name(), []byte("not an executable"), 0700); err != nil { |
| 217 | t.Fatal(err) |
| 218 | } |
| 219 | } |
| 220 | out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=success"}) |
| 221 | if err != nil { |
| 222 | t.Fatal(err, g.cmd.ProcessState) |
| 223 | } |
| 224 | var result struct { |
| 225 | InputBytes int `json:"input_bytes"` |
| 226 | BoundImage bool `json:"bound_image"` |
| 227 | } |
| 228 | if json.Unmarshal(out, &result) != nil || result.InputBytes != len(input.Stdin) || !result.BoundImage { |
| 229 | t.Fatal("wrong child result") |
| 230 | } |
| 231 | if g.starts != 1 || g.kills != 1 { |
| 232 | t.Fatal("wrong containment lifecycle", g.starts, g.kills) |
| 233 | } |
| 234 | } |
| 235 | func TestBlockedIOOverflowFailureAndLeaseLoss(t *testing.T) { |
| 236 | for _, mode := range []string{"blocked", "overflow", "nonzero", "lease-loss"} { |
| 237 | t.Run(mode, func(t *testing.T) { |
| 238 | pin, c, input, g, l := executionFixture(t) |
| 239 | ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) |
| 240 | defer cancel() |
| 241 | fixtureMode := mode |
| 242 | if mode == "lease-loss" { |
| 243 | fixtureMode = "blocked" |
| 244 | g.before = func() { go func() { time.Sleep(50 * time.Millisecond); l.lose() }() } |
| 245 | } |
| 246 | started := time.Now() |
| 247 | out, err := run(ctx, g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=" + fixtureMode}) |
| 248 | if err == nil || len(out) != 0 || g.starts != 1 || g.kills == 0 { |
| 249 | t.Fatal("failure became success", err, g.starts, g.kills) |
| 250 | } |
| 251 | if time.Since(started) > 3*time.Second { |
| 252 | t.Fatal("supervisor blocked") |
| 253 | } |
| 254 | }) |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | func TestDescendantHoldingOutputPipe(t *testing.T) { |
| 259 | pin, c, input, g, l := executionFixture(t) |
| 260 | ctx, cancel := context.WithTimeout(t.Context(), 4*time.Second) |
| 261 | defer cancel() |
| 262 | started := time.Now() |
| 263 | out, err := run(ctx, g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=descendant"}) |
| 264 | if err != ErrRefused || len(out) != 0 || g.kills != 1 { |
| 265 | t.Fatal("surviving descendant was accepted", err, g.kills) |
| 266 | } |
| 267 | if time.Since(started) > 3*time.Second { |
| 268 | t.Fatal("waited for descendant instead of terminating it") |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | func TestExpiredLeaseStillTerminatesProcess(t *testing.T) { |
| 273 | pin, c, input, g, l := executionFixture(t) |
| 274 | l.until = time.Now().Add(2200 * time.Millisecond) |
| 275 | g.before = func() { time.Sleep(time.Until(l.until) + 10*time.Millisecond) } |
| 276 | out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=blocked"}) |
| 277 | if err != ErrUncertain || len(out) != 0 || g.kills == 0 { |
| 278 | t.Fatal("expired lease skipped termination", err, g.kills) |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | func descriptorSnapshot(t *testing.T) map[string]string { |
| 283 | t.Helper() |
| 284 | entries, err := os.ReadDir("/proc/self/fd") |
| 285 | if err != nil { |
| 286 | t.Fatal(err) |
| 287 | } |
| 288 | out := map[string]string{} |
| 289 | for _, entry := range entries { |
| 290 | target, err := os.Readlink("/proc/self/fd/" + entry.Name()) |
| 291 | if errors.Is(err, os.ErrNotExist) || strings.HasPrefix(target, "anon_inode:") { |
| 292 | continue |
| 293 | } |
| 294 | if err != nil { |
| 295 | t.Fatal(err) |
| 296 | } |
| 297 | out[entry.Name()] = target |
| 298 | } |
| 299 | return out |
| 300 | } |
| 301 | |
| 302 | func TestParentDescriptorsClose(t *testing.T) { |
| 303 | for _, mode := range []string{"success", "blocked", "overflow", "nonzero", "descendant"} { |
| 304 | t.Run(mode, func(t *testing.T) { |
| 305 | pin, c, input, g, l := executionFixture(t) |
| 306 | before := descriptorSnapshot(t) |
| 307 | ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) |
| 308 | defer cancel() |
| 309 | _, err := run(ctx, g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=" + mode}) |
| 310 | if (err == nil) != (mode == "success") { |
| 311 | t.Fatal("wrong result", err) |
| 312 | } |
| 313 | if !reflect.DeepEqual(before, descriptorSnapshot(t)) { |
| 314 | t.Fatal("parent descriptor set changed after execution") |
| 315 | } |
| 316 | }) |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | func TestInputRemainsBoundAfterPreflight(t *testing.T) { |
| 321 | pin, c, input, g, l := executionFixture(t) |
| 322 | g.before = func() { input.Stdin[10] = 'b' } |
| 323 | out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=success"}) |
| 324 | if err != nil || len(out) == 0 { |
| 325 | t.Fatal("child received input changed after validation", err) |
| 326 | } |
| 327 | } |
| 328 | func TestPreflightRefusesWithoutSpawn(t *testing.T) { |
| 329 | for _, mode := range []string{"input", "artifact", "deadline", "checker", "lease"} { |
| 330 | t.Run(mode, func(t *testing.T) { |
| 331 | pin, c, input, g, l := executionFixture(t) |
| 332 | switch mode { |
| 333 | case "input": |
| 334 | input.Stdin = []byte(`{}`) |
| 335 | case "artifact": |
| 336 | c.ArtifactDigest = strings.Repeat("b", 64) |
| 337 | case "deadline": |
| 338 | c.ExecutionDeadline = time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano) |
| 339 | case "checker": |
| 340 | c.Version = 2 |
| 341 | case "lease": |
| 342 | l.lose() |
| 343 | } |
| 344 | if _, err := run(t.Context(), g, l, pin, c, input, nil); err == nil || g.starts != 0 { |
| 345 | t.Fatal("invalid launch spawned") |
| 346 | } |
| 347 | }) |
| 348 | } |
| 349 | } |
| 350 | func TestFailedStartAndUncertainCleanup(t *testing.T) { |
| 351 | pin, c, input, g, l := executionFixture(t) |
| 352 | g.startErr = errors.New("synthetic private start detail") |
| 353 | if _, err := run(t.Context(), g, l, pin, c, input, nil); err != ErrRefused || g.starts != 1 || g.kills != 1 { |
| 354 | t.Fatal("start failure lost cleanup", err) |
| 355 | } |
| 356 | pin, c, input, g, l = executionFixture(t) |
| 357 | g.killErr = errors.New("synthetic private cleanup detail") |
| 358 | out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=success"}) |
| 359 | if err != ErrUncertain || len(out) != 0 { |
| 360 | t.Fatal("uncertain cleanup accepted", err) |
| 361 | } |
| 362 | } |
| 363 | |
amadan