amadan

carlos / platform Public

zdd plan 3 task 9: nested generations mount the staged build; root validates it first

A nested generation's (and legacy sidecar's) executable mount is the
attempt's staged build. instance-control validates the path before it lays
out or starts anything: exactly release.VersionPath for the host's own
account and app (registry row, read-only), opened beneath the version store
with no symlink, sha256 equal to the attempt's digest. A refusal answers
403, the client maps it to ErrStagedExe, and verdictOf reads it as an
abandonment. A nested hold swap is allowed now. instancectl imports
release and registry directly (no import cycle).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude b7bd32027709e666ecbd0755588bbe99c1d31de7 parent 3ff569b
18 files changed, +995 −49
  • internal/activator/backend_exec.go +70 −13
  • internal/activator/containment.go +12 −3
  • internal/activator/containment_test.go +7 −0
  • internal/activator/control.go +2 −0
  • internal/activator/genbackend.go +32 −13
  • internal/activator/swap.go +5 −2
  • internal/activator/swap_hold_test.go +11 −3
  • internal/activator/swap_nested_test.go +236 −0
  • internal/activator/swap_overlap_test.go +6 −2
  • internal/activator/swap_sidecar_test.go +3 −1
  • internal/activator/swaphold.go +4 −3
  • internal/activator/unitproc_gen_test.go +3 −2
  • internal/instancectl/client.go +34 −3
  • internal/instancectl/gen.go +6 −1
  • internal/instancectl/server.go +43 −3
  • internal/instancectl/stagedexe.go +159 −0
  • internal/instancectl/stagedexe_linux_test.go +248 −0
  • internal/instancectl/stagedexe_test.go +114 −0
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index de6d5644d..3a479b949 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -1572,16 +1572,21 @@ func (b *ExecBackend) normalizeSocketMode(rt registry.Route, sock string, proc P
}
}
-// startUnitAt is Start's nested body (design §6): the same admin token, env
-// composition and health gate as startChild, but the process itself is
-// started by the root instance-control channel from a launch file rather
-// than exec'd directly. The agent has no privilege to enter the tenant's
-// namespace, only the control service does.
+// spawnUnitFrom is the nested spawn with the executable and the tier named,
+// and nothing after root has started the unit: it returns the moment the
+// unit runs, as spawnChildFrom does for a flat host. A nil exe is the live
+// link, which is every wake (startUnitAt adds today's health wait). A staged
+// exe is a swap's successor or a pinned wake (zero-downtime spec §7.5 step
+// 3): root validates the path and mounts it in place of the live link, so
+// whatever happens to the live link meanwhile changes nothing, and the
+// caller owns the health deadline and the termination. tier is that
+// artifact's own tier; zero is the live tier (b.tier).
//
// n is the generation this spawn is (StartGen's n; 0 from Start). Under
// GenContainment it selects the carlos-instance-gen@<id>:<n> unit and its
-// generation-keyed files (unitGen); otherwise it is ignored.
-func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock string, n uint64) (Proc, string, error) {
+// generation-keyed files (unitGen); otherwise it is ignored. A failure
+// whose kill could not prove the unit empty wraps errGenUnproven.
+func (b *ExecBackend) spawnUnitFrom(ctx context.Context, rt registry.Route, sock string, n uint64, exe *instancectl.StagedExe, tier cgroup.Limit) (Proc, string, error) {
if err := b.nestedGate(rt); err != nil {
return nil, "", err
}
@@ -1589,8 +1594,12 @@ func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock s
if err != nil {
return nil, "", err
}
- if err := b.ensureLiveLink(ctx, rt.Host); err != nil {
- return nil, "", err
+ if exe == nil {
+ // A staged exe is mounted in place of the live link, which then
+ // need not exist (as spawnChildFrom does for a flat host).
+ if err := b.ensureLiveLink(ctx, rt.Host); err != nil {
+ return nil, "", err
+ }
}
p, ctl, logs, fileEnv, err := b.prepareAt(rt, instancectl.RoleRun, sock, gen)
if err != nil {
@@ -1612,7 +1621,12 @@ func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock s
// direction as startChild: a concurrent repoint at worst records the
// older version (one extra convergence cycle), never the newer.
spawnVer, _ := b.LiveVersion(rt.Host)
- inv, err := b.startRole(ctx, rt, instancectl.RoleRun, gen)
+ inv, err := b.startRole(ctx, rt, instancectl.RoleRun, gen, exe, tier)
+ if errors.Is(err, instancectl.ErrStagedExe) {
+ // Root refused the staged build before it laid out or started
+ // anything, so there is nothing to kill and nothing unproven.
+ return fail(fmt.Errorf("%s: start unit: %w", rt.Host, err))
+ }
if gen != 0 && (err != nil || inv == "") {
if err == nil {
err = fmt.Errorf("%s exited before it was observed", unitName(instancectl.RoleRun, rt.Host, gen))
@@ -1633,7 +1647,11 @@ func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock s
// running, so there is nothing to stop; the wake failed.
return fail(fmt.Errorf("%s: %s exited before it was observed", rt.Host, unitName(instancectl.RoleRun, rt.Host, gen)))
}
- log.Printf("activator: %s: started %s memory.max=%d", rt.Host, unitName(instancectl.RoleRun, rt.Host, gen), b.tier(rt).MemoryMax)
+ started := tier
+ if started == (cgroup.Limit{}) {
+ started = b.tier(rt)
+ }
+ log.Printf("activator: %s: started %s memory.max=%d", rt.Host, unitName(instancectl.RoleRun, rt.Host, gen), started.MemoryMax)
up := b.newUnitProcFor(rt, instancectl.RoleRun, gen, p, inv, ctl, logs)
if b.Cache != nil {
// A fresh process for this host is exactly the moment its cached
@@ -1642,6 +1660,21 @@ func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock s
log.Printf("activator: purged %d cached response(s) for %s on spawn", n, rt.Host)
}
}
+ return up, spawnVer, nil
+}
+
+// startUnitAt is Start's nested body (design §6): the same admin token, env
+// composition and health gate as startChild, but the process itself is
+// started by the root instance-control channel from a launch file rather
+// than exec'd directly. The agent has no privilege to enter the tenant's
+// namespace, only the control service does. It is spawnUnitFrom on the live
+// link, then the health wait.
+func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock string, n uint64) (Proc, string, error) {
+ p, spawnVer, err := b.spawnUnitFrom(ctx, rt, sock, n, nil, cgroup.Limit{})
+ if err != nil {
+ return p, "", err
+ }
+ up := p.(*unitProc)
if err := waitHealthyCtx(ctx, sock, spawnHealthTimeout); err != nil {
// Set before the Stop call below, closing the same race Stop's own
// internal set leaves open by a few instructions: observe runs
@@ -1665,7 +1698,7 @@ func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock s
// stopped before the lease goes (P2-13). Under GenContainment
// the error also says so (errGenUnproven), so that cleanup
// asks for the proof with generations off as well.
- if gen != 0 {
+ if up.gen != 0 {
return up, "", fmt.Errorf("%w; %w: stop after failed health wait: %v", err, errGenUnproven, serr)
}
return up, "", err
@@ -2081,6 +2114,14 @@ func (b *ExecBackend) StartWorkerStaged(ctx context.Context, rt registry.Route,
// single layer, read straight from its own file, never merged with the
// instance's two-layer config.
func (b *ExecBackend) startSidecarUnit(ctx context.Context, rt registry.Route) (Proc, error) {
+ return b.startSidecarUnitFrom(ctx, rt, nil, cgroup.Limit{})
+}
+
+// startSidecarUnitFrom is startSidecarUnit with the executable and the tier
+// named. A nil exe is the live link (today's start). A staged exe is the
+// serving generation's captured build (zero-downtime spec §7.5), which root
+// validates before it lays out the mount, and tier that generation's own.
+func (b *ExecBackend) startSidecarUnitFrom(ctx context.Context, rt registry.Route, exe *instancectl.StagedExe, tier cgroup.Limit) (Proc, error) {
if err := b.nestedGate(rt); err != nil {
return nil, err
}
@@ -2093,7 +2134,23 @@ func (b *ExecBackend) startSidecarUnit(ctx context.Context, rt registry.Route) (
logs.Close()
return nil, err
}
- inv, err := b.Control.Start(ctx, rt.Host, instancectl.RoleSidecar, b.tier(rt).MemoryMax)
+ var inv string
+ if exe != nil {
+ // The serving generation's captured staged build, which root
+ // validates before it lays out the mount, at that generation's own
+ // tier, never the live link's.
+ gc, gerr := b.genControl()
+ if gerr != nil {
+ logs.Close()
+ return nil, gerr
+ }
+ if tier == (cgroup.Limit{}) {
+ tier = b.tier(rt)
+ }
+ inv, err = gc.StartSidecarExe(ctx, rt.Host, tier.MemoryMax, *exe)
+ } else {
+ inv, err = b.Control.Start(ctx, rt.Host, instancectl.RoleSidecar, b.tier(rt).MemoryMax)
+ }
if err != nil {
logs.Close()
return nil, fmt.Errorf("%s: start sidecar unit: %w", rt.Host, err)
diff --git a/internal/activator/containment.go b/internal/activator/containment.go
index 6d0c0569f..d3a526e70 100644
--- a/internal/activator/containment.go
+++ b/internal/activator/containment.go
@@ -140,8 +140,10 @@ func (b *ExecBackend) unitGen(n uint64) (uint64, error) {
}
// startRole starts one role's unit and returns its invocation: the
-// original template through Control, a generation through GenControl.
-func (b *ExecBackend) startRole(ctx context.Context, rt registry.Route, role instancectl.Role, gen uint64) (string, error) {
+// original template through Control, a generation through GenControl. exe
+// names a staged build for a generation (nil: the live link, plan 2's
+// StartGen), and tier is the generation's own (zero: b.tier(rt)).
+func (b *ExecBackend) startRole(ctx context.Context, rt registry.Route, role instancectl.Role, gen uint64, exe *instancectl.StagedExe, tier cgroup.Limit) (string, error) {
if gen == 0 {
return b.Control.Start(ctx, rt.Host, role, b.tier(rt).MemoryMax)
}
@@ -149,7 +151,14 @@ func (b *ExecBackend) startRole(ctx context.Context, rt registry.Route, role ins
if err != nil {
return "", err
}
- return gc.StartGen(ctx, rt.Host, gen, b.tier(rt).MemoryMax)
+ if tier == (cgroup.Limit{}) {
+ tier = b.tier(rt)
+ }
+ if exe != nil {
+ // The staged artifact's own tier, not the live link's.
+ return gc.StartGenExe(ctx, rt.Host, gen, tier.MemoryMax, *exe)
+ }
+ return gc.StartGen(ctx, rt.Host, gen, tier.MemoryMax)
}
// ParentLimit sets rt's generation parent to l: carlos-exec@<host> for a
diff --git a/internal/activator/containment_test.go b/internal/activator/containment_test.go
index a66c53e47..f3c0b9a37 100644
--- a/internal/activator/containment_test.go
+++ b/internal/activator/containment_test.go
@@ -24,6 +24,13 @@ func (f fakeGenControl) StartGen(_ context.Context, host string, n uint64, max i
}
return f.inv, f.startErr
}
+func (f fakeGenControl) StartGenExe(ctx context.Context, host string, n uint64, max int64, _ instancectl.StagedExe) (string, error) {
+ return f.StartGen(ctx, host, n, max)
+}
+func (f fakeGenControl) StartSidecarExe(_ context.Context, host string, max int64, _ instancectl.StagedExe) (string, error) {
+ f.rec(fmt.Sprintf("startsidecarexe %s %d", host, max))
+ return f.inv, f.startErr
+}
func (f fakeGenControl) StopGen(_ context.Context, host string, n uint64, kill bool) error {
f.rec(fmt.Sprintf("stopgen %s %d kill=%v", host, n, kill))
if f.stopHook != nil {
diff --git a/internal/activator/control.go b/internal/activator/control.go
index 51d6fd9c8..823b8d9ba 100644
--- a/internal/activator/control.go
+++ b/internal/activator/control.go
@@ -33,6 +33,8 @@ var _ Control = (*instancectl.Client)(nil)
// GenContainment is on.
type GenControl interface {
StartGen(ctx context.Context, host string, n uint64, memoryMax int64) (string, error)
+ StartGenExe(ctx context.Context, host string, n uint64, memoryMax int64, x instancectl.StagedExe) (string, error)
+ StartSidecarExe(ctx context.Context, host string, memoryMax int64, x instancectl.StagedExe) (string, error)
StopGen(ctx context.Context, host string, n uint64, kill bool) error
ResetGen(ctx context.Context, host string, n uint64) error
Parent(ctx context.Context, host string, memoryMax int64) error
diff --git a/internal/activator/genbackend.go b/internal/activator/genbackend.go
index df1e2657c..8e656ea8e 100644
--- a/internal/activator/genbackend.go
+++ b/internal/activator/genbackend.go
@@ -12,10 +12,9 @@ import (
)
var (
- errSwapNoGens = errors.New("swap: backend cannot run staged generations")
- errUnitNoGens = errors.New("swap: unit-backed routes keep stop-then-start")
- errSwapLayout = errors.New("swap: this row's layout has no generation sockets")
- errNestedStaged = errors.New("swap: nested staged generations need Task 9")
+ errSwapNoGens = errors.New("swap: backend cannot run staged generations")
+ errUnitNoGens = errors.New("swap: unit-backed routes keep stop-then-start")
+ errSwapLayout = errors.New("swap: this row's layout has no generation sockets")
)
// SwapBackend is what a swap needs from a backend beyond Backend and plan
@@ -88,7 +87,19 @@ func (b *ExecBackend) StartStaged(ctx context.Context, rt registry.Route, g Gen)
g.Socket, g.Control = rt.Socket, rt.Socket+".control"
return g, proc, nil
case LayoutNested:
- return g, nil, errNestedStaged
+ sock := nestedGenSocketIn(p.runDir, g.N)
+ // A start with no staged path (a cold wake) is on the live link, as
+ // the flat arm is.
+ var exe *instancectl.StagedExe
+ if g.StagedPath != "" {
+ exe = &instancectl.StagedExe{Path: g.StagedPath, Version: g.Build, Digest: g.Digest}
+ }
+ proc, _, err := b.spawnUnitFrom(ctx, rt, sock, g.N, exe, g.Tier)
+ if err != nil {
+ return g, proc, err
+ }
+ g.Socket, g.Control = sock, sock+".control"
+ return g, proc, nil
case LayoutFenced, LayoutConflict:
return g, nil, fmt.Errorf("%s: refusing to spawn: %s", rt.Host, p.reason)
}
@@ -209,7 +220,7 @@ func (b *SplitBackend) GenSwitched(rt registry.Route, g Gen, p Proc) error {
return gb.GenSwitched(rt, g, p)
}
-var errNestedSidecarGen = errors.New("nested sidecar: a generation's sidecar needs Task 9's staged executable")
+var errNestedSidecarGen = errors.New("nested sidecar: the generation has no staged executable")
// SidecarGenStarter starts a legacy co-spawned sidecar from a generation's
// captured staged path (spec §7.5: "Every legacy co-spawned sidecar runs
@@ -219,17 +230,25 @@ type SidecarGenStarter interface {
}
// StartSidecarGen execs g.StagedPath for a flat (or foreign) host. A
-// nested host's sidecar unit binds what root lays out for it, which is the
-// live link until Task 9 lets the agent name the generation's staged
-// executable. Until then the nested arm refuses. It never reads the live
-// link to decide: a repoint can land between that read and the start, and
-// after a fallback the live link names a build the instance is not serving.
-// The supervisor retries with its backoff.
+// nested host's sidecar unit binds what root lays out for it, so the agent
+// names the generation's staged executable and root validates it. A
+// generation with none (it was woken on the live link) refuses. It never
+// reads the live link to decide: a repoint can land between that read and
+// the start, and after a fallback the live link names a build the instance
+// is not serving. The supervisor retries with its backoff.
func (b *ExecBackend) StartSidecarGen(ctx context.Context, rt registry.Route, g Gen) (Proc, error) {
p := b.pathsFor(rt)
switch p.layout {
case LayoutNested:
- return nil, errNestedSidecarGen
+ // The unit binds what root lays out, so root is told the
+ // generation's staged build and validates it (Task 9). Nothing here
+ // reads the live link: a fallback or a repoint cannot change what
+ // this sidecar runs.
+ if g.StagedPath == "" {
+ return nil, errNestedSidecarGen
+ }
+ exe := instancectl.StagedExe{Path: g.StagedPath, Version: g.Build, Digest: g.Digest}
+ return b.startSidecarUnitFrom(ctx, rt, &exe, g.Tier)
case LayoutFenced, LayoutConflict:
return nil, fmt.Errorf("%s: refusing to start sidecar: %s", rt.Host, p.reason)
}
diff --git a/internal/activator/swap.go b/internal/activator/swap.go
index c0090be84..bd46a8f6d 100644
--- a/internal/activator/swap.go
+++ b/internal/activator/swap.go
@@ -15,6 +15,7 @@ import (
"github.com/carlosframework/platform/internal/hostbox"
"github.com/carlosframework/platform/internal/hostguard"
"github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
"github.com/carlosframework/platform/internal/registry"
)
@@ -153,7 +154,9 @@ const (
// It wraps hostbox.ErrUnsupported, which is not otherwise classified,
// so it is named here;
// - a row that cannot run a staged generation at all: errUnitNoGens,
-// errNestedStaged, errSwapNoGens.
+// errSwapNoGens;
+// - root refusing the nested staged executable (instancectl.ErrStagedExe,
+// Task 9): it validates before it mounts or starts anything.
//
// errSwapStopped (the barrier's cancel, from memberCurrentLocked) is
// NoOwnership too: the entry is stopping, so nothing may spawn for it.
@@ -175,7 +178,7 @@ func verdictOf(err error) gateVerdict {
errors.Is(err, hostbox.ErrChanged), errors.Is(err, errClaimsAdmission),
errors.Is(err, ErrStrictRequired), errors.Is(err, ErrStrictGroupInUse), errors.Is(err, ErrStrictCleanupPending),
errors.Is(err, ErrClaimIsolationUnavailable),
- errors.Is(err, errUnitNoGens), errors.Is(err, errNestedStaged), errors.Is(err, errSwapNoGens):
+ errors.Is(err, errUnitNoGens), errors.Is(err, errSwapNoGens), errors.Is(err, instancectl.ErrStagedExe):
return gateNoOwnership
}
return gateProceed
diff --git a/internal/activator/swap_hold_test.go b/internal/activator/swap_hold_test.go
index 71cf9afbb..740987f0a 100644
--- a/internal/activator/swap_hold_test.go
+++ b/internal/activator/swap_hold_test.go
@@ -484,9 +484,6 @@ func TestSwapHoldRefusesARowThatCannotRunAStagedBuild(t *testing.T) {
f.a.entries[f.host].route.Backing = registry.BackingUnit
f.a.mu.Unlock()
}, errSwapKind},
- {"nested", func(f *swapFixture) {
- f.b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
- }, errSwapLayout},
{"fenced layout", func(f *swapFixture) {
f.b.layoutFn = func(registry.Route) (Layout, string) { return LayoutFenced, "a migration journal" }
}, errSwapLayout},
@@ -699,3 +696,14 @@ func TestSwapOverlapOnAForeignRowRunsAsAHold(t *testing.T) {
t.Fatal("the old process outlived the hold")
}
}
+
+// A nested row can hold-swap now that root mounts the staged build (Task 9):
+// the refusal Task 6 made for it is gone, and the hold runs to SwapDone.
+func TestSwapHoldAcceptsANestedRow(t *testing.T) {
+ f := newSwapFixture(t)
+ f.b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
+ res := f.a.Swap(context.Background(), f.attempt(ModeHold), f.newest)
+ if res.Outcome != SwapDone || res.Gen.StagedPath != f.staged {
+ t.Fatalf("Swap = %+v, want a done hold on the staged build", res)
+ }
+}
diff --git a/internal/activator/swap_nested_test.go b/internal/activator/swap_nested_test.go
new file mode 100644
index 000000000..98b8f0cd9
--- /dev/null
+++ b/internal/activator/swap_nested_test.go
@@ -0,0 +1,236 @@
+package activator
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// exeControl is a root control channel with the generation verbs. Its plain
+// StartGen refuses, so a test sees which start was used.
+type exeControl struct {
+ mu sync.Mutex
+ got []instancectl.StagedExe
+ sidecarExes []instancectl.StagedExe
+ plainStarts []instancectl.Role // Start calls: the live-link exe layout
+ tiers []int64 // StartGenExe's memoryMax, per call
+ sidecarTiers []int64 // StartSidecarExe's memoryMax, per call
+ exeErr error // what StartGenExe answers, when set
+ stopGens int // StopGen calls
+ stopGenErr error // what StopGen answers
+}
+
+func (c *exeControl) Start(_ context.Context, _ string, role instancectl.Role, _ int64) (string, error) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.plainStarts = append(c.plainStarts, role)
+ return "inv", nil
+}
+func (c *exeControl) StartSidecarExe(_ context.Context, _ string, memoryMax int64, x instancectl.StagedExe) (string, error) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.sidecarExes = append(c.sidecarExes, x)
+ c.sidecarTiers = append(c.sidecarTiers, memoryMax)
+ return "inv-sc", nil
+}
+func (c *exeControl) Seed(context.Context, string, int64) (instancectl.SeedResponse, error) {
+ return instancectl.SeedResponse{}, nil
+}
+func (c *exeControl) Stop(context.Context, string, instancectl.Role) error { return nil }
+func (c *exeControl) Reset(context.Context, string, instancectl.Role) error { return nil }
+func (c *exeControl) Sweep(context.Context) (instancectl.SweepResponse, error) {
+ return instancectl.SweepResponse{}, nil
+}
+func (c *exeControl) Ready(context.Context) error { return nil }
+func (c *exeControl) StartGen(context.Context, string, uint64, int64) (string, error) {
+ return "", errors.New("test: the live-link generation start was used")
+}
+func (c *exeControl) StopGen(context.Context, string, uint64, bool) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.stopGens++
+ return c.stopGenErr
+}
+func (c *exeControl) ResetGen(context.Context, string, uint64) error { return nil }
+func (c *exeControl) Parent(context.Context, string, int64) error { return nil }
+func (c *exeControl) StartGenExe(_ context.Context, _ string, _ uint64, memoryMax int64, x instancectl.StagedExe) (string, error) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.got = append(c.got, x)
+ c.tiers = append(c.tiers, memoryMax)
+ if c.exeErr != nil {
+ return "", c.exeErr
+ }
+ return "inv-7", nil
+}
+
+// A nested staged generation names its executable to root. Without one,
+// the generation start is plan 2's, on the live link.
+func TestStartRoleNamesTheStagedExe(t *testing.T) {
+ b := testExecBackend(t)
+ b.GenContainment = true
+ ctl := &exeControl{}
+ b.Control = ctl
+ rt := registry.Route{Host: "n.example.com"}
+ x := instancectl.StagedExe{Path: "/opt/carlos/versions/v2/blog", Version: "v2", Digest: strings.Repeat("a", 64)}
+ inv, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7, &x, cgroup.Limit{})
+ if err != nil || inv != "inv-7" || len(ctl.got) != 1 || ctl.got[0] != x {
+ t.Fatalf("startRole = %q, %v; root was sent %+v", inv, err, ctl.got)
+ }
+ if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7, nil, cgroup.Limit{}); err == nil || !strings.Contains(err.Error(), "live-link") {
+ t.Fatalf("no staged exe: %v, want plan 2's start", err)
+ }
+}
+
+// A nested legacy sidecar is started on the serving generation's captured
+// staged executable, which root validates (spec §7.5). The live link names
+// v1 while the generation serves v2, as after a fallback, and the sidecar
+// still starts, on v2. Task 7's interim nested arm refused it
+// (errNestedSidecarGen), and today's StartSidecar would bind the live v1.
+func TestStartSidecarGenNestedNamesTheStagedExe(t *testing.T) {
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ b.GenContainment = true
+ ctl := &exeControl{}
+ b.Control = ctl
+ stageLive(t, b, rt.Host, "v1")
+ if err := os.MkdirAll(filepath.Dir(b.pathsFor(rt).sidecarEnv), 0o750); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(b.pathsFor(rt).sidecarEnv, []byte("X=1\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ // The generation's own tier: 1 GiB, while the live link's (b.tier) is
+ // the default 512 MiB. After a fallback the two differ.
+ g := Gen{N: 4, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("b", 64), Tier: cgroup.TierChildren}
+ p, err := b.StartSidecarGen(context.Background(), rt, g)
+ if err != nil {
+ t.Fatalf("StartSidecarGen: %v", err)
+ }
+ defer p.Stop(context.Background()) //nolint:errcheck
+ want := instancectl.StagedExe{Path: g.StagedPath, Version: "v2", Digest: g.Digest}
+ ctl.mu.Lock()
+ defer ctl.mu.Unlock()
+ if len(ctl.sidecarExes) != 1 || ctl.sidecarExes[0] != want {
+ t.Fatalf("root was sent %+v for the sidecar, want %+v", ctl.sidecarExes, want)
+ }
+ if len(ctl.plainStarts) != 0 {
+ t.Fatalf("a live-link start ran too: %v", ctl.plainStarts)
+ }
+ if ctl.sidecarTiers[0] != cgroup.TierChildren.MemoryMax {
+ t.Fatalf("root was sent memory_max %d, want the generation's own %d", ctl.sidecarTiers[0], cgroup.TierChildren.MemoryMax)
+ }
+}
+
+// Root refusing the staged path abandons the swap (the one classifier,
+// verdictOf, reads instancectl.ErrStagedExe as NoOwnership). It is not a
+// failure of the build, and the incumbent keeps serving.
+func TestSwapAbandonsWhenRootRefusesTheStagedExe(t *testing.T) {
+ f := newSwapFixture(t)
+ f.b.genErr = fmt.Errorf("%w: not blog's staged v2", instancectl.ErrStagedExe)
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, "staged executable refused") {
+ t.Fatalf("Swap = %+v, want abandoned as a root refusal", res)
+ }
+ if got := f.appSocket(t); got != f.incumbent.Socket {
+ t.Fatalf("AppSocket = %q, want the incumbent", got)
+ }
+}
+
+// A nested staged generation is returned as soon as root has started its
+// unit, with no health wait inside (spec §8.2): the socket here is never
+// bound, and the real 30 s spawnHealthTimeout is left alone. root is sent
+// the generation's own tier.
+func TestStartStagedNestedReturnsTheUnitAtOnce(t *testing.T) {
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ b.GenContainment = true
+ ctl := &exeControl{}
+ b.Control = ctl
+ g := Gen{N: 7, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("c", 64), Tier: cgroup.TierChildren}
+ done := make(chan error, 1)
+ var proc Proc
+ go func() {
+ var err error
+ _, proc, err = b.StartStaged(context.Background(), rt, g)
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ if err != nil || proc == nil {
+ t.Fatalf("StartStaged = %v, %v: want the started unit", proc, err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("StartStaged waited for health: it must return the started unit at once")
+ }
+ defer proc.Stop(context.Background()) //nolint:errcheck
+ ctl.mu.Lock()
+ defer ctl.mu.Unlock()
+ if len(ctl.got) != 1 || ctl.tiers[0] != cgroup.TierChildren.MemoryMax {
+ t.Fatalf("root was sent %+v at %v, want one start at the generation's 1 GiB", ctl.got, ctl.tiers)
+ }
+}
+
+// Root refusing the staged build started nothing, so the agent kills
+// nothing and holds nothing unproven. The refusal reaches the swap as
+// ErrStagedExe alone (verdictOf: abandonment), with the generation number
+// the caller keeps.
+func TestStartStagedNestedRootRefusalIsNotUnproven(t *testing.T) {
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ b.GenContainment = true
+ ctl := &exeControl{exeErr: fmt.Errorf("%w: not blog's staged v2", instancectl.ErrStagedExe)}
+ b.Control = ctl
+ g := Gen{N: 7, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("c", 64), Tier: cgroup.TierChildren}
+ got, proc, err := b.StartStaged(context.Background(), rt, g)
+ if !errors.Is(err, instancectl.ErrStagedExe) || errors.Is(err, errGenUnproven) || proc != nil || got.N != 7 {
+ t.Fatalf("StartStaged = %+v, %v, %v", got, proc, err)
+ }
+ if ctl.stopGens != 0 {
+ t.Fatalf("the agent killed %d generation(s) root never started", ctl.stopGens)
+ }
+ if verdictOf(err) != gateNoOwnership {
+ t.Fatalf("verdictOf(%v) = %d, want NoOwnership", err, verdictOf(err))
+ }
+}
+
+// A nested successor whose start failed and whose kill is unproven says so
+// (errGenUnproven) and returns its generation number, so the member's stop
+// can prove it (stopMember) and the operation set is not released.
+func TestStartStagedNestedUnprovenKeepsTheGenerationNumber(t *testing.T) {
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ b.GenContainment = true
+ ctl := &exeControl{exeErr: errors.New("502 from root"), stopGenErr: errors.New("processes remain")}
+ b.Control = ctl
+ g := Gen{N: 7, Build: "v2", StagedPath: "/opt/carlos/versions/v2/blog", Digest: strings.Repeat("c", 64), Tier: cgroup.TierChildren}
+ got, proc, err := b.StartStaged(context.Background(), rt, g)
+ if !errors.Is(err, errGenUnproven) || proc != nil || got.N != 7 {
+ t.Fatalf("StartStaged = %+v, %v, %v: want errGenUnproven, no handle, generation 7", got, proc, err)
+ }
+ if ctl.stopGens != 1 {
+ t.Fatalf("StopGen calls = %d, want the one kill", ctl.stopGens)
+ }
+ if verdictOf(err) != gateProceed {
+ t.Fatalf("an unproven start must stay a failure, verdictOf = %d", verdictOf(err))
+ }
+}
+
+// A nested start with no staged path (a cold wake) is on the live link, as
+// the flat arm is, and names no executable to root.
+func TestStartStagedNestedWithoutAStagedPathUsesTheLiveLink(t *testing.T) {
+ b, rt := nestedBackend(t, &fakeControl{}, &fakeUnit{})
+ b.GenContainment = true
+ ctl := &exeControl{}
+ b.Control = ctl
+ _, _, err := b.StartStaged(context.Background(), rt, Gen{N: 7})
+ if err == nil || !strings.Contains(err.Error(), "live-link") || len(ctl.got) != 0 {
+ t.Fatalf("StartStaged = %v; root was sent %+v: want plan 2's live-link start", err, ctl.got)
+ }
+}
diff --git a/internal/activator/swap_overlap_test.go b/internal/activator/swap_overlap_test.go
index 07a35cb85..ec3f91a6b 100644
--- a/internal/activator/swap_overlap_test.go
+++ b/internal/activator/swap_overlap_test.go
@@ -11,6 +11,7 @@ import (
"github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/hostguard"
"github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
"github.com/carlosframework/platform/internal/registry"
)
@@ -273,7 +274,9 @@ func TestClassifyStep(t *testing.T) {
{"a closed claims launch", live, fmt.Errorf("launch: %w", ErrClaimIsolationUnavailable), stepStale},
// A row that cannot run a staged generation at all.
{"a unit row", live, errUnitNoGens, stepStale},
- {"a nested row before Task 9", live, errNestedStaged, stepStale},
+ // Root refused the nested staged executable before it mounted or
+ // started anything (Task 9): an abandonment, not a failed build.
+ {"root refused the staged exe", live, fmt.Errorf("start unit: %w: not blog's staged v2", instancectl.ErrStagedExe), stepStale},
{"a backend without generations", live, errSwapNoGens, stepStale},
{"a health deadline", live, fmt.Errorf("health: %w", context.DeadlineExceeded), stepFailed},
{"HTTP 500", live, errors.New("GET / answered 500"), stepFailed},
@@ -323,7 +326,8 @@ func TestVerdictOf(t *testing.T) {
{claimAdmission(ErrClaimIsolationUnavailable, false), gateNoOwnership},
// A row that cannot run a staged generation: nothing staged spawns.
{errUnitNoGens, gateNoOwnership},
- {errNestedStaged, gateNoOwnership},
+ {instancectl.ErrStagedExe, gateNoOwnership},
+ {fmt.Errorf("start unit: %w: x", instancectl.ErrStagedExe), gateNoOwnership},
{errSwapNoGens, gateNoOwnership},
} {
if got := verdictOf(tc.err); got != tc.want {
diff --git a/internal/activator/swap_sidecar_test.go b/internal/activator/swap_sidecar_test.go
index 2b4c79cc0..a40779728 100644
--- a/internal/activator/swap_sidecar_test.go
+++ b/internal/activator/swap_sidecar_test.go
@@ -644,8 +644,10 @@ func TestExecBackendStartSidecarGen(t *testing.T) {
rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.NestedDBPath(b.DataDir, host),
Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")}
for _, build := range []string{"v1", "v2"} { // the live build and another
+ // A generation with no staged executable (woken on the live
+ // link) has nothing to name to root, so it still refuses.
if _, err := b.StartSidecarGen(context.Background(), rt, Gen{N: 3, Build: build}); !errors.Is(err, errNestedSidecarGen) {
- t.Fatalf("serving %s: %v, want errNestedSidecarGen until Task 9", build, err)
+ t.Fatalf("serving %s with no staged path: %v, want errNestedSidecarGen", build, err)
}
}
})
diff --git a/internal/activator/swaphold.go b/internal/activator/swaphold.go
index e20917caa..340980cba 100644
--- a/internal/activator/swaphold.go
+++ b/internal/activator/swaphold.go
@@ -108,10 +108,11 @@ func (a *Service) swapHold(ctx context.Context, at Attempt, newest func(string)
if e.worker || e.route.Kind != "instance" || e.route.Backing == registry.BackingUnit {
return errSwapKind
}
- // The layouts that can run a staged generation: flat, and foreign on
- // its canonical socket. Nested waits for Task 9's executable mount.
+ // The layouts that can run a staged generation: flat, foreign on its
+ // canonical socket, and nested (root mounts the staged build,
+ // Task 9).
if c, ok := a.backend.(Classifier); ok {
- if l, _ := c.Layout(e.route); l != LayoutFlat && l != LayoutForeign {
+ if l, _ := c.Layout(e.route); l != LayoutFlat && l != LayoutForeign && l != LayoutNested {
return errSwapLayout
}
}
diff --git a/internal/activator/unitproc_gen_test.go b/internal/activator/unitproc_gen_test.go
index 552a09c7b..ac0854730 100644
--- a/internal/activator/unitproc_gen_test.go
+++ b/internal/activator/unitproc_gen_test.go
@@ -11,6 +11,7 @@ import (
"testing"
"time"
+ "github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/instancectl"
"github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
@@ -144,10 +145,10 @@ func TestStartRoleDispatchesByGeneration(t *testing.T) {
ctl := &fakeControl{inv: "inv"}
b, rt := nestedBackend(t, ctl, &fakeUnit{})
b.Control = fakeGenControl{ctl}
- if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 0); err != nil {
+ if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 0, nil, cgroup.Limit{}); err != nil {
t.Fatal(err)
}
- if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7); err != nil {
+ if _, err := b.startRole(context.Background(), rt, instancectl.RoleRun, 7, nil, cgroup.Limit{}); err != nil {
t.Fatal(err)
}
if want := "start h.example.com run\nstartgen h.example.com 7 536870912"; ctl.joined() != want {
diff --git a/internal/instancectl/client.go b/internal/instancectl/client.go
index 9d495efd5..3c2a9678b 100644
--- a/internal/instancectl/client.go
+++ b/internal/instancectl/client.go
@@ -181,11 +181,42 @@ func genValues(host string, n uint64) url.Values {
// tier and returns the invocation id the server read after the start. A
// failure whose cleanup could not prove the unit gone wraps ErrGenUnproven.
func (c *Client) StartGen(ctx context.Context, host string, n uint64, memoryMax int64) (string, error) {
- q := genValues(host, n)
+ return c.startUnitQuery(ctx, genValues(host, n), memoryMax, nil)
+}
+
+// StartGenExe is StartGen with the generation's executable named: the
+// staged build root validates and mounts in place of the live link. A 403
+// is root's refusal, reported as ErrStagedExe.
+func (c *Client) StartGenExe(ctx context.Context, host string, n uint64, memoryMax int64, x StagedExe) (string, error) {
+ return c.startUnitQuery(ctx, genValues(host, n), memoryMax, &x)
+}
+
+// StartSidecarExe is Start for the legacy sidecar role with its executable
+// named: the serving generation's staged build, which root validates and
+// mounts in place of the live link (zero-downtime spec §7.5). A 403 is
+// root's refusal, reported as ErrStagedExe.
+func (c *Client) StartSidecarExe(ctx context.Context, host string, memoryMax int64, x StagedExe) (string, error) {
+ return c.startUnitQuery(ctx, hostRole(host, RoleSidecar), memoryMax, &x)
+}
+
+// startUnitQuery is /start for a generation or the sidecar, optionally with
+// a staged executable. One place maps root's two distinct refusals: 403 is
+// ErrStagedExe (nothing was mounted), and the generation verbs' unproven
+// status is ErrGenUnproven.
+func (c *Client) startUnitQuery(ctx context.Context, q url.Values, memoryMax int64, x *StagedExe) (string, error) {
q.Set("memory_max", strconv.FormatInt(memoryMax, 10))
+ if x != nil {
+ q.Set("exe", x.Path)
+ q.Set("version", x.Version)
+ q.Set("digest", x.Digest)
+ }
var out StartResponse
- if status, _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/start", q, &out); err != nil {
- if status == statusGenUnproven {
+ status, _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/start", q, &out)
+ if err != nil {
+ switch {
+ case status == http.StatusForbidden && x != nil:
+ return "", fmt.Errorf("%w: %v", ErrStagedExe, err)
+ case status == statusGenUnproven && q.Has("gen"):
return "", fmt.Errorf("%w: %w", ErrGenUnproven, err)
}
return "", err
diff --git a/internal/instancectl/gen.go b/internal/instancectl/gen.go
index 0bdcd5315..a870565f6 100644
--- a/internal/instancectl/gen.go
+++ b/internal/instancectl/gen.go
@@ -125,8 +125,13 @@ func (s *Server) startGen(w http.ResponseWriter, r *http.Request, req request) {
defer s.markGenStarting(unit, false)
// A refusal for the disk writes nothing. An exe layout that fails
// half-way may have made its directory, so that failure cleans up too.
+ // A staged executable is validated before anything is laid out
+ // (zero-downtime plan 3, Task 9).
+ if s.refuseStagedExe(w, req, unit) {
+ return
+ }
if s.refuseStart(w, req.host, unit, func() error {
- err := s.ensureGenExe(req.host, req.gen)
+ err := s.ensureGenExeAt(req.host, req.gen, req.exe)
if err != nil {
s.removeGenFiles(unit)
}
diff --git a/internal/instancectl/server.go b/internal/instancectl/server.go
index 9d91bffcf..3cd196991 100644
--- a/internal/instancectl/server.go
+++ b/internal/instancectl/server.go
@@ -57,6 +57,12 @@ type Server struct {
// ExeRoot is where ensureExe lays out each instance's view of its own
// build (#214 item 2); empty skips it, for tests of other verbs.
ExeRoot string
+ // OptDir is the version store's root (/opt/carlos), and HostApp reads a
+ // host's account and app from the registry. Together they are what a
+ // nested generation's staged executable is validated against
+ // (validateStagedExe). Either one unset refuses every staged start.
+ OptDir string
+ HostApp func(host string) (acct, app string, err error)
// DropInDir is where generation units' runtime drop-ins go
// (/run/systemd/system). Empty refuses every generation start: the
// template alone runs nothing (zero-downtime plan 2 D1).
@@ -147,6 +153,8 @@ func runWithClaim(args []string, claim http.Handler) {
ProbeEvery: 5 * time.Minute,
Logf: log.Printf,
ExeRoot: filepath.Join(controlsock.Dir, "exe"),
+ OptDir: "/opt/carlos",
+ HostApp: registryHostApp(filepath.Join(dataDir(), "registry.db")),
DropInDir: "/run/systemd/system",
ResolvConf: "/etc/resolv.conf",
ResolvedConf: ResolvedConf,
@@ -281,6 +289,7 @@ type request struct {
tier cgroup.Limit
gen uint64 // a generation of the run role (zero-downtime spec §7.6); 0 is none
kill bool
+ exe StagedExe // a staged build to mount (zero-downtime plan 3); Path "" is none
}
// verbShape is what a verb's query may carry.
@@ -289,6 +298,7 @@ type verbShape struct {
tier bool
gen bool
kill bool
+ exe bool // a start verb: a generation's, or the legacy sidecar's, staged build
}
func parseRequest(r *http.Request, v verbShape) (request, error) {
@@ -344,6 +354,22 @@ func parseRequest(r *http.Request, v verbShape) (request, error) {
}
req.kill = true
}
+ if q.Has("exe") || q.Has("version") || q.Has("digest") {
+ if !v.exe {
+ return request{}, errors.New("this verb takes no staged executable")
+ }
+ if x := q.Get("exe"); x != "" {
+ // A generation start, or the legacy sidecar (zero-downtime spec
+ // §7.5: it runs its instance's serving artifact). The original
+ // template's run unit keeps the live link.
+ if req.gen == 0 && req.role != RoleSidecar {
+ return request{}, errors.New("exe is only for a generation start or a sidecar")
+ }
+ req.exe = StagedExe{Path: x, Version: q.Get("version"), Digest: q.Get("digest")}
+ } else {
+ return request{}, errors.New("version and digest need an exe")
+ }
+ }
return req, nil
}
@@ -368,7 +394,7 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
}
func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar}, tier: true, gen: true})
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar}, tier: true, gen: true, exe: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@@ -379,10 +405,24 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
}
ctx := r.Context()
unit := UnitName(req.role, req.host)
- if s.refuseStart(w, req.host, unit, func() error { return s.ensureHostExe(req.host) }) {
+ // A sidecar naming its generation's staged build is validated before
+ // anything is laid out, under the run unit's rule (validateStagedExe).
+ if s.refuseStagedExe(w, req, unit) {
return
}
- if err := s.setTier(ctx, req.host, req.tier); err != nil {
+ if s.refuseStart(w, req.host, unit, func() error { return s.ensureHostExeAt(req.host, req.exe) }) {
+ return
+ }
+ // A sidecar start that names a staged build comes from a box with
+ // generations on, where the host slice is the generation parent (spec
+ // §7.3). A swap may have raised it to two tiers, and lowering it here
+ // would put both generations under one tier's limit. Its restore is the
+ // swap owner's, so a populated slice is left alone, exactly as startGen
+ // does; a cold slice still gets the tier. A start without exe (flag
+ // off) sets the tier as today.
+ if req.exe.Path != "" && s.slicePopulated(req.host) {
+ s.logf("instance-control: %s: slice populated, leaving its limit to the generation owner", req.host)
+ } else if err := s.setTier(ctx, req.host, req.tier); err != nil {
http.Error(w, "set-property: "+err.Error(), http.StatusBadGateway)
return
}
diff --git a/internal/instancectl/stagedexe.go b/internal/instancectl/stagedexe.go
new file mode 100644
index 000000000..334e43f6f
--- /dev/null
+++ b/internal/instancectl/stagedexe.go
@@ -0,0 +1,159 @@
+package instancectl
+
+import (
+ "crypto/sha256"
+ "database/sql"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/instancefs"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+
+ _ "modernc.org/sqlite"
+)
+
+// StagedExe is a nested generation's executable as the agent names it: the
+// staged build of the attempt, its version and the sha256 the attempt
+// verified (zero-downtime spec §7.5 step 3).
+type StagedExe struct {
+ Path string
+ Version string
+ Digest string
+}
+
+// ErrStagedExe is root refusing to mount a staged executable it cannot
+// vouch for. The agent abandons the swap; nothing is mounted.
+var ErrStagedExe = errors.New("instancectl: staged executable refused")
+
+// validateStagedExe is root's check before it mounts a staged build into a
+// tenant's namespace. Root mounts only what is provably that host's own
+// app's staged build:
+// - The path must be exactly release.VersionPath for the host's account
+// and app, as its registry row records them, at the attempt's version.
+// Another app's build, another account's, or any other path is refused.
+// - The file is opened beneath the version store with every component
+// resolved without a symlink (instancefs.OpenBeneath).
+// - Its sha256 must be the attempt's digest.
+//
+// The version store is written by root alone, so what is checked here is
+// what the bind will find.
+func (s *Server) validateStagedExe(host string, x StagedExe) error {
+ if s.HostApp == nil || s.OptDir == "" {
+ return fmt.Errorf("%w: this service has no registry view", ErrStagedExe)
+ }
+ if !release.ValidVersion(x.Version) || len(x.Digest) != 2*sha256.Size {
+ return fmt.Errorf("%w: version %q, digest %q", ErrStagedExe, x.Version, x.Digest)
+ }
+ acct, app, err := s.HostApp(host)
+ if err != nil || app == "" {
+ return fmt.Errorf("%w: %s has no app on this box (%v)", ErrStagedExe, host, err)
+ }
+ want := release.VersionPath(s.OptDir, release.RouteAccount(registry.Route{Account: acct}), x.Version, app)
+ if x.Path != want {
+ return fmt.Errorf("%w: %q is not %s's staged %s (%q)", ErrStagedExe, x.Path, host, x.Version, want)
+ }
+ versions := filepath.Join(s.OptDir, "versions")
+ rel, err := filepath.Rel(versions, want)
+ if err != nil {
+ return fmt.Errorf("%w: %v", ErrStagedExe, err)
+ }
+ f, err := instancefs.OpenBeneath(versions, rel, os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%w: %v", ErrStagedExe, err)
+ }
+ defer f.Close()
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return fmt.Errorf("%w: %v", ErrStagedExe, err)
+ }
+ if got := hex.EncodeToString(h.Sum(nil)); got != strings.ToLower(x.Digest) {
+ return fmt.Errorf("%w: %s has digest %s, the attempt verified %s", ErrStagedExe, want, got, x.Digest)
+ }
+ return nil
+}
+
+// refuseStagedExe answers 403 for a start whose staged executable root
+// cannot vouch for, and reports whether it did. A request that names none
+// is not refused here. It is the one place both start paths (a generation's
+// and the legacy sidecar's) validate, before anything is laid out.
+func (s *Server) refuseStagedExe(w http.ResponseWriter, req request, unit string) bool {
+ if req.exe.Path == "" {
+ return false
+ }
+ if err := s.validateStagedExe(req.host, req.exe); err != nil {
+ s.logf("instance-control: %s: not starting %s: %v", req.host, unit, err)
+ http.Error(w, "not starting "+unit+": "+err.Error(), http.StatusForbidden)
+ return true
+ }
+ return false
+}
+
+// ensureGenExeAt lays out generation n's executable mount. It points at a
+// validated staged build when x names one, and at the live link otherwise
+// (plan 2's ensureGenExe).
+func (s *Server) ensureGenExeAt(host string, n uint64, x StagedExe) error {
+ if x.Path == "" {
+ return s.ensureGenExe(host, n)
+ }
+ return ensureExe(s.ExeRoot, GenKey(host, n), x.Path, host)
+}
+
+// ensureHostExeAt lays out the host's own executable mount (the one the
+// original templates bind: <ExeRoot>/<id>/app). It points at a validated
+// staged build when x names one, which only a legacy sidecar's start does,
+// and at the live link otherwise (ensureHostExe).
+func (s *Server) ensureHostExeAt(host string, x StagedExe) error {
+ if x.Path == "" {
+ return s.ensureHostExe(host)
+ }
+ return ensureExe(s.ExeRoot, provision.InstanceID(host), x.Path, host)
+}
+
+// dataDir is the agent's data directory: CARLOS_DATA_DIR, else /data (the
+// agent's own resolver in host/agent.go, repeated as instanceexec does).
+func dataDir() string {
+ if d := os.Getenv("CARLOS_DATA_DIR"); d != "" {
+ return d
+ }
+ return "/data"
+}
+
+// registryHostApp reads a host's account and app from the agent's registry
+// at path, read-only. Root must never leave root-owned files beside the
+// agent's WAL database, so the read is only attempted while the agent has
+// the registry open, that is, while its -wal and -shm files exist. SQLite
+// would otherwise create them (mode=ro does not stop it making the wal-index
+// of a WAL database), owned by root and unusable by the agent. With either
+// missing the lookup is refused and the start with it. A refused lookup is
+// an abandoned swap, never a failed build.
+func registryHostApp(path string) func(host string) (string, string, error) {
+ return func(host string) (string, string, error) {
+ for _, p := range []string{path, path + "-wal", path + "-shm"} {
+ fi, err := os.Lstat(p)
+ if err != nil {
+ return "", "", fmt.Errorf("registry %s is not open by the agent (%v)", path, err)
+ }
+ if !fi.Mode().IsRegular() {
+ return "", "", fmt.Errorf("registry %s: %s is not a regular file", path, p)
+ }
+ }
+ u := url.URL{Scheme: "file", Path: path, RawQuery: "mode=ro&_pragma=busy_timeout(5000)"}
+ db, err := sql.Open("sqlite", u.String())
+ if err != nil {
+ return "", "", err
+ }
+ defer db.Close()
+ var acct, app string
+ err = db.QueryRow(`SELECT account, app FROM routes WHERE host = ?`, host).Scan(&acct, &app)
+ return acct, app, err
+ }
+}
diff --git a/internal/instancectl/stagedexe_linux_test.go b/internal/instancectl/stagedexe_linux_test.go
new file mode 100644
index 000000000..80e0111ca
--- /dev/null
+++ b/internal/instancectl/stagedexe_linux_test.go
@@ -0,0 +1,248 @@
+//go:build linux
+
+package instancectl
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func writeStaged(t *testing.T, path, body string) string {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256([]byte(body))
+ return hex.EncodeToString(sum[:])
+}
+
+// Root mounts only the host's own app's staged build, reached without a
+// symlink, whose bytes are the attempt's (zero-downtime plan 3, Task 9).
+func TestValidateStagedExe(t *testing.T) {
+ opt := t.TempDir()
+ host := "blog.example.com"
+ s := &Server{OptDir: opt, HostApp: func(h string) (string, string, error) {
+ if h != host {
+ return "", "", errors.New("no such route")
+ }
+ return release.OpsAccount, "blog", nil
+ }}
+ path := release.VersionPath(opt, release.OpsAccount, "v2", "blog")
+ digest := writeStaged(t, path, "build v2")
+ if err := s.validateStagedExe(host, StagedExe{Path: path, Version: "v2", Digest: digest}); err != nil {
+ t.Fatalf("the host's own staged build was refused: %v", err)
+ }
+ shop := release.VersionPath(opt, release.OpsAccount, "v2", "shop")
+ shopDigest := writeStaged(t, shop, "build v2")
+ // v3 of blog reached through a symlinked version directory.
+ if err := os.Symlink(filepath.Join(opt, "versions", "v2"), filepath.Join(opt, "versions", "v3")); err != nil {
+ t.Fatal(err)
+ }
+ linked := release.VersionPath(opt, release.OpsAccount, "v3", "blog")
+ for name, tc := range map[string]struct {
+ host string
+ x StagedExe
+ }{
+ "another app's build": {host, StagedExe{Path: shop, Version: "v2", Digest: shopDigest}},
+ "another version": {host, StagedExe{Path: path, Version: "v3", Digest: digest}},
+ "a path outside": {host, StagedExe{Path: "/etc/passwd", Version: "v2", Digest: digest}},
+ "bytes that differ": {host, StagedExe{Path: path, Version: "v2", Digest: strings.Repeat("0", 64)}},
+ "a symlinked component": {host, StagedExe{Path: linked, Version: "v3", Digest: digest}},
+ "a bad version": {host, StagedExe{Path: path, Version: "../v2", Digest: digest}},
+ "an unknown host": {"other.example.com", StagedExe{Path: path, Version: "v2", Digest: digest}},
+ } {
+ if err := s.validateStagedExe(tc.host, tc.x); !errors.Is(err, ErrStagedExe) {
+ t.Errorf("%s: %v, want ErrStagedExe", name, err)
+ }
+ }
+ if err := (&Server{OptDir: opt}).validateStagedExe(host, StagedExe{Path: path, Version: "v2", Digest: digest}); !errors.Is(err, ErrStagedExe) {
+ t.Errorf("no registry view: %v, want ErrStagedExe", err)
+ }
+}
+
+// ctlCalled reports whether any recorded systemctl call starts with prefix.
+func ctlCalled(c *fakeCtl, prefix string) bool {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ for _, l := range c.calls {
+ if strings.HasPrefix(l, prefix) {
+ return true
+ }
+ }
+ return false
+}
+
+func (f *fakeCtl) reset() {
+ f.mu.Lock()
+ f.calls = nil
+ f.mu.Unlock()
+}
+
+// The nested legacy sidecar runs the serving generation's staged build
+// (spec §7.5), validated by root under the run unit's rule before its
+// executable mount is pointed at it. The live link names another build
+// throughout and makes no difference. A refused path answers 403 and
+// leaves the mount alone. Before this task a sidecar /start could not name
+// an executable at all, so the sidecar bound the live link.
+func TestSidecarStartMountsTheValidatedStagedExe(t *testing.T) {
+ opt, exeRoot, cgroupRoot := t.TempDir(), t.TempDir(), t.TempDir()
+ staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog")
+ digest := writeStaged(t, staged, "build v2")
+ ctl := &fakeCtl{}
+ _, sock := serveOnSocket(t, ctl, func(s *Server) {
+ s.DropInDir, s.ExeRoot, s.OptDir, s.CgroupRoot = t.TempDir(), exeRoot, opt, cgroupRoot
+ s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil }
+ })
+ c := NewClient(sock)
+ app := filepath.Join(exeRoot, provision.InstanceID(genHost), "app")
+ if _, err := c.StartSidecarExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax,
+ StagedExe{Path: staged, Version: "v2", Digest: digest}); err != nil {
+ t.Fatalf("the host's own staged build was refused for its sidecar: %v", err)
+ }
+ if got, err := os.Readlink(app); err != nil || got != staged {
+ t.Fatalf("the sidecar's exe = %q (%v), want the staged %q", got, err, staged)
+ }
+ for name, x := range map[string]StagedExe{
+ "bytes that differ": {Path: staged, Version: "v2", Digest: strings.Repeat("0", 64)},
+ "a path outside": {Path: "/etc/passwd", Version: "v2", Digest: digest},
+ } {
+ if _, err := c.StartSidecarExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, x); !errors.Is(err, ErrStagedExe) {
+ t.Errorf("%s: %v, want ErrStagedExe", name, err)
+ }
+ if got, _ := os.Readlink(app); got != staged {
+ t.Errorf("%s: a refused start moved the mount to %q", name, got)
+ }
+ }
+ // The slice above was cold, so its tier was set. With a generation
+ // running (populated), a staged sidecar start leaves the slice's limit
+ // alone: during an overlap it is the raised parent of both generations
+ // (spec §7.3), and a start that set one tier would put them both under
+ // it. A flag-off start (no exe) sets the tier as today.
+ if !ctlCalled(ctl, "set-property --runtime "+SliceName(genHost)) {
+ t.Fatal("a cold slice got no tier")
+ }
+ events := filepath.Join(cgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
+ if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ ctl.reset()
+ if _, err := c.StartSidecarExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax,
+ StagedExe{Path: staged, Version: "v2", Digest: digest}); err != nil {
+ t.Fatal(err)
+ }
+ if ctlCalled(ctl, "set-property") {
+ t.Fatal("a staged sidecar start rewrote the populated generation parent's limit")
+ }
+ if _, err := c.Start(context.Background(), genHost, RoleSidecar, cgroup.TierDefault.MemoryMax); err != nil {
+ t.Fatal(err)
+ }
+ if !ctlCalled(ctl, "set-property --runtime "+SliceName(genHost)) {
+ t.Fatal("a flag-off sidecar start no longer sets the tier")
+ }
+ // A run-role start on the original template names no executable.
+ q := hostRole(genHost, RoleRun)
+ q.Set("memory_max", strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10))
+ q.Set("exe", staged)
+ q.Set("version", "v2")
+ q.Set("digest", digest)
+ if code, _, _ := c.do(context.Background(), DeadlineControl, http.MethodPost, "/start", q, nil); code != http.StatusBadRequest {
+ t.Errorf("a run-role start with exe answered %d, want 400", code)
+ }
+}
+
+// stagedGenServer is genServer with a version store holding the host's own
+// staged v2 build. It returns the staged exe root has to accept.
+func stagedGenServer(t *testing.T, ctl *fakeCtl) (*Server, string, StagedExe) {
+ t.Helper()
+ opt := t.TempDir()
+ staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog")
+ digest := writeStaged(t, staged, "build v2")
+ dropins, exe := t.TempDir(), t.TempDir()
+ s, sock := serveOnSocket(t, ctl, func(s *Server) {
+ s.DropInDir, s.ExeRoot, s.OptDir = dropins, exe, opt
+ s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil }
+ })
+ return s, sock, StagedExe{Path: staged, Version: "v2", Digest: digest}
+}
+
+// A validated staged generation start mounts the staged build. A refused one
+// answers ErrStagedExe having touched neither systemd nor the disk.
+func TestGenStartExeMountsTheValidatedBuildAndRefusalTouchesNothing(t *testing.T) {
+ ctl := genCtl()
+ s, sock, x := stagedGenServer(t, ctl)
+ c := NewClient(sock)
+ bad := x
+ bad.Digest = strings.Repeat("0", 64)
+ if _, err := c.StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) {
+ t.Fatalf("StartGenExe(bad digest) = %v, want ErrStagedExe", err)
+ }
+ if ctl.joined() != "" {
+ t.Fatalf("a refused staged start reached systemctl:\n%s", ctl.joined())
+ }
+ if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
+ t.Fatal("an executable mount was laid out for a refused start")
+ }
+ if _, err := c.StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax, x); err != nil {
+ t.Fatalf("StartGenExe: %v", err)
+ }
+ if got, err := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 7), "app")); err != nil || got != x.Path {
+ t.Fatalf("generation 7's exe = %q (%v), want %q", got, err, x.Path)
+ }
+}
+
+// The staged start keeps plan 2's answers: a cleanup that cannot prove the
+// unit gone is ErrGenUnproven and not ErrStagedExe, and its files stay.
+func TestGenStartExeKeepsTheUnprovenAnswer(t *testing.T) {
+ ctl := genCtl()
+ ctl.fail = map[string]error{"show -p InvocationID": errors.New("bus timeout")}
+ s, sock, x := stagedGenServer(t, ctl)
+ s.cgroupWaitBudget = 100 * time.Millisecond
+ populateGenCgroup(t, s)
+ _, err := NewClient(sock).StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax, x)
+ if !errors.Is(err, ErrGenUnproven) || errors.Is(err, ErrStagedExe) {
+ t.Fatalf("StartGenExe = %v, want ErrGenUnproven only", err)
+ }
+ if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err != nil {
+ t.Fatalf("the exe layout of an unproven generation was removed: %v", err)
+ }
+}
+
+// exe, version and digest belong to the start verbs only.
+func TestStagedExeParametersRefusedWhereTheyDoNotBelong(t *testing.T) {
+ ctl := genCtl()
+ _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
+ for _, path := range []string{
+ "/stop?host=" + genHost + "&role=run&gen=7&exe=/x",
+ "/stop?host=" + genHost + "&role=sidecar&exe=/x",
+ "/reset?host=" + genHost + "&role=sidecar&version=v2",
+ "/seed?host=" + genHost + "&memory_max=536870912&exe=/x",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&exe=/x&version=v2&digest=d",
+ "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=7&version=v2",
+ } {
+ if resp := post(t, c, path); resp.StatusCode != 400 {
+ t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
+ }
+ }
+ if ctl.joined() != "" {
+ t.Errorf("systemctl was called:\n%s", ctl.joined())
+ }
+}
diff --git a/internal/instancectl/stagedexe_test.go b/internal/instancectl/stagedexe_test.go
new file mode 100644
index 000000000..16e6be9ca
--- /dev/null
+++ b/internal/instancectl/stagedexe_test.go
@@ -0,0 +1,114 @@
+package instancectl
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The executable mount names the staged build when one is given, and the
+// live link (plan 2's layout) when not.
+func TestEnsureGenExeAtTargetsTheStagedPath(t *testing.T) {
+ s := &Server{ExeRoot: t.TempDir()}
+ staged := "/opt/carlos/versions/v2/blog"
+ if err := s.ensureGenExeAt(genHost, 7, StagedExe{Path: staged}); err != nil {
+ t.Fatal(err)
+ }
+ if got, err := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 7), "app")); err != nil || got != staged {
+ t.Fatalf("generation 7's exe = %q (%v), want %q", got, err, staged)
+ }
+ if err := s.ensureGenExeAt(genHost, 8, StagedExe{}); err != nil {
+ t.Fatal(err)
+ }
+ if got, _ := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 8), "app")); got != "/opt/carlos/live/"+genHost {
+ t.Fatalf("generation 8's exe = %q, want the live link", got)
+ }
+}
+
+// A refused path answers 403 before anything is laid out, and the client
+// reports ErrStagedExe, so the swap is abandoned rather than failed.
+func TestGenStartRefusesAStagedExeRootCannotVouchFor(t *testing.T) {
+ exeRoot := t.TempDir()
+ _, sock := serveOnSocket(t, &fakeCtl{}, func(s *Server) {
+ s.DropInDir, s.ExeRoot, s.OptDir = t.TempDir(), exeRoot, t.TempDir()
+ s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil }
+ })
+ _, err := NewClient(sock).StartGenExe(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax,
+ StagedExe{Path: "/etc/passwd", Version: "v2", Digest: strings.Repeat("0", 64)})
+ if !errors.Is(err, ErrStagedExe) {
+ t.Fatalf("StartGenExe = %v, want ErrStagedExe", err)
+ }
+ if _, err := os.Stat(filepath.Join(exeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
+ t.Fatal("an executable mount was laid out for a refused path")
+ }
+}
+
+// Root reads the host's account and app from the registry read-only, and
+// only while the agent has it open, so it never leaves root-owned WAL files
+// the agent cannot use.
+func TestRegistryHostAppReadsTheRow(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "registry.db")
+ // The agent's own handle stays open for the whole test, as it does in
+ // production: that is what keeps the -wal and -shm files in place.
+ reg, err := registry.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ db, err := sql.Open("sqlite", path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(`INSERT INTO routes (host, kind, account, app, created_at) VALUES ('blog.example.com', 'instance', 'k7xq', 'blog', 1)`); err != nil {
+ t.Fatal(err)
+ }
+ db.Close()
+ acct, app, err := registryHostApp(path)("blog.example.com")
+ if err != nil || acct != "k7xq" || app != "blog" {
+ t.Fatalf("registryHostApp = %q %q %v", acct, app, err)
+ }
+ if _, _, err := registryHostApp(path)("nobody.example.com"); err == nil {
+ t.Fatal("an unknown host was answered")
+ }
+}
+
+// With the agent's registry closed there are no -wal or -shm files, and
+// opening a WAL database would create them, owned by whoever opened it (root
+// here). The lookup refuses instead and leaves the directory as it found it.
+func TestRegistryHostAppCreatesNothingBesideTheDatabase(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "registry.db")
+ reg, err := registry.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ reg.Close()
+ before, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := registryHostApp(path)("blog.example.com"); err == nil {
+ t.Fatal("a registry the agent does not hold open was read")
+ }
+ after, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(after) != len(before) {
+ t.Fatalf("the lookup changed %s: %d entries before, %d after", dir, len(before), len(after))
+ }
+ if _, _, err := registryHostApp(filepath.Join(dir, "nope.db"))("blog.example.com"); err == nil {
+ t.Fatal("a missing registry was read")
+ }
+ if _, err := os.Stat(filepath.Join(dir, "nope.db")); !os.IsNotExist(err) {
+ t.Fatal("the lookup created a registry file")
+ }
+}