amadan

carlos / platform Public

Refuse surviving maintenance descendants and terminate after lease expiry

Paul Campbell pushed by paul@keymail.dev 717bdd87b311df6a990a9399f5d87bc87f08bd6e parent 01b82bd
8 files changed, +242 −25
  • implementation/maintenance-worker-progress.json +12 −3
  • internal/cgroup/strict_linux.go +13 −0
  • internal/cgroup/strict_linux_test.go +25 −0
  • internal/cgroup/strict_other.go +1 −0
  • internal/maintenanceexec/errors.go +6 −0
  • internal/maintenanceexec/run_linux.go +29 −17
  • internal/maintenanceexec/run_linux_test.go +140 −5
  • internal/maintenanceexec/run_other.go +16 −0

CI passed — run details

diff --git a/implementation/maintenance-worker-progress.json b/implementation/maintenance-worker-progress.json
index d4dc9767c..23c22af22 100644
--- a/implementation/maintenance-worker-progress.json
+++ b/implementation/maintenance-worker-progress.json
@@ -1,10 +1,10 @@
{
"scope": "Complete supported maintenance dispatch needed by Oficina provisioning; no live invocation until all approved launch gates are implemented and reviewed",
"base": "006e7825",
- "current_slice": "Strict typed private launch-context codec for approved mutable v1 and checkpoint-checker v2",
+ "current_slice": "Strict private launch codec and inert Linux subprocess supervisor; no worker mounting",
"authority": "Decoded context is metadata only. It cannot construct a retained lease, executor, lifecycle claim, approval or trusted invocation.",
"remaining": [
- "Verified pinned-artifact launch and cancellable bounded pipe supervisor using strict cgroup handles",
+ "Native cgroup/lease integration evidence for reviewed subprocess supervisor",
"Retained worker lease/executor and closed-checkpoint preparation proof",
"Home startup adapter binding signed input, image, descriptors and opened database",
"Owner-reviewed prepare/approval/status/result/retry/resume CLI and console workflow",
@@ -12,5 +12,14 @@
"Integrated native tests and Fable review before worker mounting",
"Preview owner-approved provisioning, restore drill, and public gateway cutover"
],
- "verification": ["Required closed fields and exact numeric ranges", "Canonical timestamps, identifiers and binding digests", "UTF-8 and surrogate validity", "8KiB hard limit and overflow detection", "Nested v2 checkpoint evidence bindings", "No command mounting or execution authority from decoded bytes"]
+ "verification": [
+ "Required closed fields and exact numeric ranges",
+ "Canonical timestamps, identifiers and binding digests",
+ "UTF-8 and surrogate validity",
+ "8KiB hard limit and overflow detection",
+ "Nested v2 checkpoint evidence bindings",
+ "No command mounting or execution authority from decoded bytes",
+ "Focused race tests for descendants, expired-lease termination, exact private input and descriptor/environment isolation",
+ "Fable approved codec at 01b82bde; supervisor refusal preserved and fixes awaiting re-review"
+ ]
}
diff --git a/internal/cgroup/strict_linux.go b/internal/cgroup/strict_linux.go
index 7d365038e..4b2557507 100644
--- a/internal/cgroup/strict_linux.go
+++ b/internal/cgroup/strict_linux.go
@@ -266,6 +266,19 @@ func (p *Prepared) Start(cmd *exec.Cmd) error {
return nil
}
+// Populated checks the retained group after the direct child has been reaped.
+func (p *Prepared) Populated() (bool, error) {
+ if p == nil || p.strictPreparedState == nil {
+ return false, ErrStrictState
+ }
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if e := p.identityOK(); e != nil {
+ return false, e
+ }
+ return strictPopulation(p.events)
+}
+
// KillAndEmpty proves kernel population is zero before the caller's deadline.
// Failure retains the pinned cleanup handle and permits another bounded attempt.
func (p *Prepared) KillAndEmpty(ctx context.Context) error {
diff --git a/internal/cgroup/strict_linux_test.go b/internal/cgroup/strict_linux_test.go
index 0d6124a94..03f687a55 100644
--- a/internal/cgroup/strict_linux_test.go
+++ b/internal/cgroup/strict_linux_test.go
@@ -65,6 +65,31 @@ func strictContext(t *testing.T, d time.Duration) context.Context {
t.Cleanup(cancel)
return ctx
}
+
+func TestStrictPopulationUsesRetainedIdentity(t *testing.T) {
+ p := strictFixture(t)
+ for _, populated := range []bool{false, true} {
+ value := "populated 0\nfrozen 0\n"
+ if populated {
+ value = "populated 1\nfrozen 0\n"
+ }
+ if e := strictWrite(p.events, value); e != nil {
+ t.Fatal(e)
+ }
+ if got, e := p.Populated(); e != nil || got != populated {
+ t.Fatal("incorrect population", got, e)
+ }
+ }
+ if e := os.Rename(p.dir.Name(), p.dir.Name()+".old"); e != nil {
+ t.Fatal(e)
+ }
+ if _, e := p.Populated(); e == nil {
+ t.Fatal("replaced cgroup accepted")
+ }
+ if _, e := (&Prepared{}).Populated(); e == nil {
+ t.Fatal("empty handle accepted")
+ }
+}
func TestStrictRejectsUnavailableManagerHostAndTier(t *testing.T) {
for _, m := range []*Manager{nil, {root: t.TempDir()}} {
for _, host := range []string{"ok.example", "../outside", "Upper.example", "", "a/b"} {
diff --git a/internal/cgroup/strict_other.go b/internal/cgroup/strict_other.go
index 0a1b0ed76..b5c10bd57 100644
--- a/internal/cgroup/strict_other.go
+++ b/internal/cgroup/strict_other.go
@@ -13,5 +13,6 @@ func (m *Manager) PrepareStrict(host string, limit Limit) (*Prepared, error) {
return nil, ErrStrictUnsupported
}
func (p *Prepared) Start(cmd *exec.Cmd) error { return ErrStrictUnsupported }
+func (p *Prepared) Populated() (bool, error) { return false, ErrStrictUnsupported }
func (p *Prepared) KillAndEmpty(ctx context.Context) error { return ErrStrictUnsupported }
func (p *Prepared) Close() error { return ErrStrictUnsupported }
diff --git a/internal/maintenanceexec/errors.go b/internal/maintenanceexec/errors.go
new file mode 100644
index 000000000..818cdcf84
--- /dev/null
+++ b/internal/maintenanceexec/errors.go
@@ -0,0 +1,6 @@
+package maintenanceexec
+
+import "errors"
+
+var ErrRefused = errors.New("maintenance execution: refused")
+var ErrUncertain = errors.New("maintenance execution: process cleanup uncertain")
diff --git a/internal/maintenanceexec/run_linux.go b/internal/maintenanceexec/run_linux.go
index b37aa4e82..252662fe6 100644
--- a/internal/maintenanceexec/run_linux.go
+++ b/internal/maintenanceexec/run_linux.go
@@ -8,7 +8,6 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
- "errors"
"io"
"os"
"os/exec"
@@ -20,13 +19,11 @@ import (
"golang.org/x/sys/unix"
)
-var ErrRefused = errors.New("maintenance execution: refused")
-var ErrUncertain = errors.New("maintenance execution: process cleanup uncertain")
-
const stopMargin = 2 * time.Second
type containment interface {
Start(*exec.Cmd) error
+ Populated() (bool, error)
KillAndEmpty(context.Context) error
}
type ownership interface {
@@ -46,9 +43,12 @@ func Run(ctx context.Context, group *cgroup.Prepared, held *lease.Held, artifact
}
func run(ctx context.Context, group containment, held ownership, artifact *os.File, launch maintenance.LaunchContext, input maintenance.Input, environment []string) (result []byte, err error) {
- if ctx == nil || ctx.Err() != nil || artifact == nil || launch.Version != 1 {
+ if ctx == nil || ctx.Err() != nil || artifact == nil || launch.Version != 1 || len(input.Arguments) > maintenance.MaxArguments || len(input.Stdin) > maintenance.MaxInputBytes {
return nil, ErrRefused
}
+ input.Arguments = append([]string(nil), input.Arguments...)
+ input.Stdin = append([]byte(nil), input.Stdin...)
+ defer clear(input.Stdin)
raw, e := maintenance.EncodeLaunch(launch)
if e != nil {
return nil, ErrRefused
@@ -102,9 +102,7 @@ func run(ctx context.Context, group containment, held ownership, artifact *os.Fi
return nil, ErrRefused
}
if e = group.Start(cmd); e != nil {
- stopCtx, cancel := context.WithDeadline(context.Background(), until)
- stopErr := group.KillAndEmpty(stopCtx)
- cancel()
+ stopErr := stop(group, until)
if stopErr != nil {
return nil, ErrUncertain
}
@@ -119,9 +117,7 @@ func run(ctx context.Context, group containment, held ownership, artifact *os.Fi
return
}
pipes.close()
- stopCtx, cancel := context.WithDeadline(context.Background(), until)
- defer cancel()
- if group.KillAndEmpty(stopCtx) != nil {
+ if stop(group, until) != nil {
err = ErrUncertain
result = nil
}
@@ -174,9 +170,13 @@ func run(ctx context.Context, group containment, held ownership, artifact *os.Fi
}
}
if failed || waited {
- stopCtx, cancel := context.WithDeadline(context.Background(), until)
- e := group.KillAndEmpty(stopCtx)
- cancel()
+ if waited {
+ populated, e := group.Populated()
+ if e != nil || populated {
+ failed = true
+ }
+ }
+ e := stop(group, until)
if e != nil {
clear(result)
return nil, ErrUncertain
@@ -206,9 +206,7 @@ func run(ctx context.Context, group containment, held ownership, artifact *os.Fi
}
}
if !cleanupDone {
- stopCtx, cancel := context.WithDeadline(context.Background(), until)
- e := group.KillAndEmpty(stopCtx)
- cancel()
+ e := stop(group, until)
if e != nil {
clear(result)
return nil, ErrUncertain
@@ -225,6 +223,20 @@ func run(ctx context.Context, group containment, held ownership, artifact *os.Fi
return result, nil
}
+func stop(group containment, until time.Time) error {
+ deadline := time.Now().Add(stopMargin)
+ if until.After(deadline) {
+ deadline = until
+ }
+ ctx, cancel := context.WithDeadline(context.Background(), deadline)
+ defer cancel()
+ err := group.KillAndEmpty(ctx)
+ if err != nil || !time.Now().Before(until) {
+ return ErrUncertain
+ }
+ return nil
+}
+
type pipeEvent struct {
kind string
data []byte
diff --git a/internal/maintenanceexec/run_linux_test.go b/internal/maintenanceexec/run_linux_test.go
index 67ef4e60c..9b65cf118 100644
--- a/internal/maintenanceexec/run_linux_test.go
+++ b/internal/maintenanceexec/run_linux_test.go
@@ -13,6 +13,8 @@ import (
"os"
"os/exec"
"path/filepath"
+ "reflect"
+ "strconv"
"strings"
"sync"
"syscall"
@@ -44,7 +46,33 @@ func (g *fixtureGroup) Start(c *exec.Cmd) error {
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
return c.Start()
}
-func (g *fixtureGroup) KillAndEmpty(context.Context) error {
+func (g *fixtureGroup) Populated() (bool, error) {
+ if g.cmd == nil || g.cmd.Process == nil {
+ return false, nil
+ }
+ entries, err := os.ReadDir("/proc")
+ if err != nil {
+ return false, err
+ }
+ for _, entry := range entries {
+ if _, err := strconv.Atoi(entry.Name()); err != nil {
+ continue
+ }
+ raw, err := os.ReadFile("/proc/" + entry.Name() + "/stat")
+ if err != nil {
+ continue
+ }
+ fields := strings.Fields(string(raw)[strings.LastIndexByte(string(raw), ')')+1:])
+ if len(fields) >= 3 && fields[2] == strconv.Itoa(g.cmd.Process.Pid) && fields[0] != "Z" {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+func (g *fixtureGroup) KillAndEmpty(ctx context.Context) error {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
g.kills++
if g.cmd != nil && g.cmd.Process != nil {
unix.Kill(-g.cmd.Process.Pid, unix.SIGKILL)
@@ -72,6 +100,32 @@ func TestChildFixture(t *testing.T) {
if mode == "" {
return
}
+ if len(os.Environ()) != 2 || os.Getenv("GODEBUG") != "containermaxprocs=0,updatemaxprocs=0" {
+ os.Exit(15)
+ }
+ if cwd, err := os.Getwd(); err != nil || cwd != "/" {
+ os.Exit(16)
+ }
+ stderr, err := os.Stderr.Stat()
+ null, nullErr := os.Stat(os.DevNull)
+ if err != nil || nullErr != nil || !os.SameFile(stderr, null) {
+ os.Exit(17)
+ }
+ entries, err := os.ReadDir("/proc/self/fd")
+ if err != nil {
+ os.Exit(18)
+ }
+ for _, entry := range entries {
+ fd, err := strconv.Atoi(entry.Name())
+ if err != nil || fd <= 4 {
+ continue
+ }
+ target, err := os.Readlink("/proc/self/fd/" + entry.Name())
+ if errors.Is(err, os.ErrNotExist) || strings.HasPrefix(target, "anon_inode:") {
+ continue
+ }
+ os.Exit(19)
+ }
if mode == "blocked" {
time.Sleep(time.Minute)
return
@@ -84,6 +138,15 @@ func TestChildFixture(t *testing.T) {
if mode == "nonzero" {
os.Exit(7)
}
+ if mode == "descendant" {
+ child := exec.Command("/proc/self/exe", "-test.run=^TestChildFixture$")
+ child.Env = []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=blocked"}
+ child.Dir = "/"
+ child.Stdout = os.Stdout
+ if child.Start() != nil {
+ os.Exit(14)
+ }
+ }
c, err := maintenance.ReadLaunch(os.NewFile(3, "context"))
if err != nil {
os.Exit(8)
@@ -114,6 +177,7 @@ func TestChildFixture(t *testing.T) {
}
func executionFixture(t *testing.T) (*os.File, maintenance.LaunchContext, maintenance.Input, *fixtureGroup, *fixtureLease) {
t.Helper()
+ t.Setenv("MAINTENANCE_PARENT_ONLY", "must-not-reach-child")
src, err := os.Open("/proc/self/exe")
if err != nil {
t.Fatal(err)
@@ -153,9 +217,9 @@ func TestPinnedImageAndExactPrivateInput(t *testing.T) {
t.Fatal(err)
}
}
- out, err := run(t.Context(), g, l, pin, c, input, []string{"MAINTENANCE_EXEC_FIXTURE=success"})
+ out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=success"})
if err != nil {
- t.Fatal(err)
+ t.Fatal(err, g.cmd.ProcessState)
}
var result struct {
InputBytes int `json:"input_bytes"`
@@ -180,7 +244,7 @@ func TestBlockedIOOverflowFailureAndLeaseLoss(t *testing.T) {
g.before = func() { go func() { time.Sleep(50 * time.Millisecond); l.lose() }() }
}
started := time.Now()
- out, err := run(ctx, g, l, pin, c, input, []string{"MAINTENANCE_EXEC_FIXTURE=" + fixtureMode})
+ out, err := run(ctx, g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=" + fixtureMode})
if err == nil || len(out) != 0 || g.starts != 1 || g.kills == 0 {
t.Fatal("failure became success", err, g.starts, g.kills)
}
@@ -190,6 +254,77 @@ func TestBlockedIOOverflowFailureAndLeaseLoss(t *testing.T) {
})
}
}
+
+func TestDescendantHoldingOutputPipe(t *testing.T) {
+ pin, c, input, g, l := executionFixture(t)
+ ctx, cancel := context.WithTimeout(t.Context(), 4*time.Second)
+ defer cancel()
+ started := time.Now()
+ out, err := run(ctx, g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=descendant"})
+ if err != ErrRefused || len(out) != 0 || g.kills != 1 {
+ t.Fatal("surviving descendant was accepted", err, g.kills)
+ }
+ if time.Since(started) > 3*time.Second {
+ t.Fatal("waited for descendant instead of terminating it")
+ }
+}
+
+func TestExpiredLeaseStillTerminatesProcess(t *testing.T) {
+ pin, c, input, g, l := executionFixture(t)
+ l.until = time.Now().Add(2200 * time.Millisecond)
+ g.before = func() { time.Sleep(time.Until(l.until) + 10*time.Millisecond) }
+ out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=blocked"})
+ if err != ErrUncertain || len(out) != 0 || g.kills == 0 {
+ t.Fatal("expired lease skipped termination", err, g.kills)
+ }
+}
+
+func descriptorSnapshot(t *testing.T) map[string]string {
+ t.Helper()
+ entries, err := os.ReadDir("/proc/self/fd")
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := map[string]string{}
+ for _, entry := range entries {
+ target, err := os.Readlink("/proc/self/fd/" + entry.Name())
+ if errors.Is(err, os.ErrNotExist) || strings.HasPrefix(target, "anon_inode:") {
+ continue
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ out[entry.Name()] = target
+ }
+ return out
+}
+
+func TestParentDescriptorsClose(t *testing.T) {
+ for _, mode := range []string{"success", "blocked", "overflow", "nonzero", "descendant"} {
+ t.Run(mode, func(t *testing.T) {
+ pin, c, input, g, l := executionFixture(t)
+ before := descriptorSnapshot(t)
+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
+ defer cancel()
+ _, err := run(ctx, g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=" + mode})
+ if (err == nil) != (mode == "success") {
+ t.Fatal("wrong result", err)
+ }
+ if !reflect.DeepEqual(before, descriptorSnapshot(t)) {
+ t.Fatal("parent descriptor set changed after execution")
+ }
+ })
+ }
+}
+
+func TestInputRemainsBoundAfterPreflight(t *testing.T) {
+ pin, c, input, g, l := executionFixture(t)
+ g.before = func() { input.Stdin[10] = 'b' }
+ out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=success"})
+ if err != nil || len(out) == 0 {
+ t.Fatal("child received input changed after validation", err)
+ }
+}
func TestPreflightRefusesWithoutSpawn(t *testing.T) {
for _, mode := range []string{"input", "artifact", "deadline", "checker", "lease"} {
t.Run(mode, func(t *testing.T) {
@@ -220,7 +355,7 @@ func TestFailedStartAndUncertainCleanup(t *testing.T) {
}
pin, c, input, g, l = executionFixture(t)
g.killErr = errors.New("synthetic private cleanup detail")
- out, err := run(t.Context(), g, l, pin, c, input, []string{"MAINTENANCE_EXEC_FIXTURE=success"})
+ out, err := run(t.Context(), g, l, pin, c, input, []string{"GODEBUG=containermaxprocs=0,updatemaxprocs=0", "MAINTENANCE_EXEC_FIXTURE=success"})
if err != ErrUncertain || len(out) != 0 {
t.Fatal("uncertain cleanup accepted", err)
}
diff --git a/internal/maintenanceexec/run_other.go b/internal/maintenanceexec/run_other.go
new file mode 100644
index 000000000..0aa531438
--- /dev/null
+++ b/internal/maintenanceexec/run_other.go
@@ -0,0 +1,16 @@
+//go:build !linux
+
+package maintenanceexec
+
+import (
+ "context"
+ "os"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/maintenance"
+)
+
+func Run(context.Context, *cgroup.Prepared, *lease.Held, *os.File, maintenance.LaunchContext, maintenance.Input, []string) ([]byte, error) {
+ return nil, ErrRefused
+}