amadan
settings-4 task 5b: the exec backend runs one recorded staged identity, flat and nested, for the run and seed units
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
9 files changed,
+901
−56
internal/activator/backend_exec.go+131 −21internal/activator/containment.go+36 −3internal/activator/control.go+15 −0internal/activator/identity.go+66 −0internal/activator/identity_test.go+447 −0internal/instancectl/client.go+39 −9internal/instancectl/server.go+29 −12internal/instancectl/stagedexe.go+6 −4internal/instancectl/stagedexe_linux_test.go+132 −7
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go| index 34592be3c..cd4aa9b93 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -780,11 +780,30 @@ func (b *ExecBackend) Seed(ctx context.Context, rt registry.Route) error { |
| p := b.pathsFor(rt) |
| switch p.layout { |
| case LayoutNested: |
| - return b.seedUnit(ctx, rt) |
| + return b.seedUnit(ctx, rt, nil) |
| case LayoutFenced, LayoutConflict: |
| return fmt.Errorf("%s: refusing to seed: %s", rt.Host, p.reason) |
| } |
| - return b.seedChild(ctx, rt) |
| + return b.seedChild(ctx, rt, b.liveLink(rt.Host)) |
| +} |
| + |
| +// SeedExe is Seed on the identity a wake recorded (settings spec §4.2): |
| +// id.StagedPath is checked against id.Digest first, and a copy that |
| +// changed seeds nothing (never the live link instead). A flat or foreign |
| +// row execs that file by path. A nested row asks root to run the seed unit |
| +// on it, which root validates and binds in place of the live link. |
| +func (b *ExecBackend) SeedExe(ctx context.Context, rt registry.Route, id release.StagedIdentity) error { |
| + if err := release.VerifyStaged(id); err != nil { |
| + return fmt.Errorf("%s: not seeded with %s: %w", rt.Host, id.Version, err) |
| + } |
| + p := b.pathsFor(rt) |
| + switch p.layout { |
| + case LayoutNested: |
| + return b.seedUnit(ctx, rt, stagedExeOf(id)) |
| + case LayoutFenced, LayoutConflict: |
| + return fmt.Errorf("%s: refusing to seed: %s", rt.Host, p.reason) |
| + } |
| + return b.seedChild(ctx, rt, id.StagedPath) |
| } |
| // seedArgv parses rt.Seed's JSON argv array and appends the --db flag, the |
| @@ -815,7 +834,10 @@ func removeSeedDB(dbPath string) { |
| } |
| } |
| -func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route) error { |
| +// seedChild is Seed's flat body. exe is the file the seed execs: the live |
| +// link (Seed), which is materialized first if it is missing, or a staged |
| +// path SeedExe verified, which is exec'd by path whatever the link names. |
| +func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route, exe string) error { |
| host := rt.Host |
| paths := b.pathsFor(rt) |
| argv, err := seedArgv(rt, paths.dbPath) |
| @@ -825,8 +847,10 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route) error { |
| if argv == nil { |
| return nil // defensive: doWake only calls us for a non-empty seed |
| } |
| - if err := b.ensureLiveLink(ctx, host); err != nil { |
| - return err |
| + if exe == b.liveLink(host) { |
| + if err := b.ensureLiveLink(ctx, host); err != nil { |
| + return err |
| + } |
| } |
| // Mint before the env is read, exactly as Start does — a seed is the FIRST |
| // thing a fresh instance runs, so this is usually where the token comes |
| @@ -850,7 +874,7 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route) error { |
| ctx, cancel := context.WithTimeout(ctx, seedTimeout) |
| defer cancel() |
| var stderr capBuffer |
| - log.Printf("activator: seeding %s: %s %s", host, b.liveLink(host), strings.Join(argv, " ")) |
| + log.Printf("activator: seeding %s: %s %s", host, exe, strings.Join(argv, " ")) |
| // The seed shares the instance's carlos-exec@<host> cgroup: first-boot |
| // cost is that instance's work. No CleanupHost here — the instance spawn |
| // reuses the dir moments later, and its exit path (or the next agent |
| @@ -860,7 +884,7 @@ func (b *ExecBackend) seedChild(ctx context.Context, rt registry.Route) error { |
| return perr |
| } |
| cmd, settle, err := b.spawnInCgroupAt(rt, child, false, func() *exec.Cmd { |
| - cmd := exec.CommandContext(ctx, b.liveLink(host), argv...) |
| + cmd := exec.CommandContext(ctx, exe, argv...) |
| // The control-channel vars are agent-owned in BOTH directions here too |
| // (Start's discipline, see withoutVars), and here they are always |
| // STRIPPED: the seed gets no control listener of its own, so fileEnv is |
| @@ -988,13 +1012,26 @@ func (c *capBuffer) String() string { |
| // this call itself caused, run through the root instance-control channel |
| // instead of a direct exec (design §6.2). Argv and env come from the same |
| // seedArgv and withoutVars calls seedChild uses, so the two cannot drift. |
| -func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route) error { |
| +// |
| +// A nil exe seeds on the live link (Seed). A staged exe is the build the |
| +// wake recorded (SeedExe): root validates it and binds the seed unit's app |
| +// mount to it, so the live link need not exist and a repoint of it |
| +// changes nothing. A control channel that cannot name it refuses the seed. |
| +func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route, exe *instancectl.StagedExe) error { |
| host := rt.Host |
| if err := b.nestedGate(rt); err != nil { |
| return err |
| } |
| - if err := b.ensureLiveLink(ctx, host); err != nil { |
| - return err |
| + var xc StagedExeControl |
| + if exe == nil { |
| + if err := b.ensureLiveLink(ctx, host); err != nil { |
| + return err |
| + } |
| + } else { |
| + var err error |
| + if xc, err = b.stagedExeControl(); err != nil { |
| + return fmt.Errorf("%s: not seeded with %s: %w", host, exe.Version, err) |
| + } |
| } |
| p, _, logs, fileEnv, err := b.prepare(rt, instancectl.RoleSeed) |
| if err != nil { |
| @@ -1028,8 +1065,19 @@ func (b *ExecBackend) seedUnit(ctx context.Context, rt registry.Route) error { |
| // client's own timeout but harmless, and keeps this call visibly |
| // bounded without second-guessing how long a seed may take. |
| sctx, cancel := context.WithTimeout(ctx, instancectl.DeadlineSeed) |
| - res, err := b.Control.Seed(sctx, host, b.tier(rt).MemoryMax) |
| + var res instancectl.SeedResponse |
| + if exe != nil { |
| + res, err = xc.SeedExe(sctx, host, b.tier(rt).MemoryMax, *exe) |
| + } else { |
| + res, err = b.Control.Seed(sctx, host, b.tier(rt).MemoryMax) |
| + } |
| cancel() |
| + if errors.Is(err, instancectl.ErrStagedExe) { |
| + // Root refused the recorded build before it laid out or started |
| + // anything (403), so no seed ran: nothing to stop, and the |
| + // database is not the seed's work. |
| + return fmt.Errorf("seed for %s: %w", host, err) |
| + } |
| if err != nil { |
| return b.seedUnitFailed(ctx, rt, dbPath, seedOwnsDB, fmt.Errorf("seed for %s: %w", host, err)) |
| } |
| @@ -1193,6 +1241,24 @@ func (b *ExecBackend) ensureLiveLink(ctx context.Context, host string) error { |
| return fmt.Errorf("no live binary for %s (%s missing and no Materialize hook)", host, bin) |
| } |
| +// LiveIdentity is the one file a cold wake will run (settings spec §4.2): |
| +// the live link, materialized first if it is missing (ensureLiveLink, so |
| +// the Materialize hook and its root-stage relay run exactly as a wake's |
| +// spawn would run them), then resolved ONCE into the file at the end of |
| +// its chain and that file's digest, as StartWorkerStaged does. Version is |
| +// "" for a link outside the version store; the caller decides whether that |
| +// may run (Service.recordBuild refuses it while a build state is kept). |
| +func (b *ExecBackend) LiveIdentity(ctx context.Context, rt registry.Route) (release.StagedIdentity, error) { |
| + if err := b.ensureLiveLink(ctx, rt.Host); err != nil { |
| + return release.StagedIdentity{}, err |
| + } |
| + id, err := release.ResolveLive(b.OptDir, rt.Host) |
| + if err != nil { |
| + return release.StagedIdentity{}, fmt.Errorf("%s: %w", rt.Host, err) |
| + } |
| + return id, nil |
| +} |
| + |
| // Start dispatches a wake by layout (design §8.1): a nested row starts as a |
| // unit through the root instance-control channel, a fenced or conflicting |
| // row refuses outright, and a flat or foreign row runs today's exec child. |
| @@ -1200,11 +1266,11 @@ func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, strin |
| p := b.pathsFor(rt) |
| switch p.layout { |
| case LayoutNested: |
| - return b.startUnitAt(ctx, rt, rt.Socket, 0) |
| + return b.startUnitAt(ctx, rt, rt.Socket, 0, nil) |
| case LayoutFenced, LayoutConflict: |
| return nil, "", fmt.Errorf("%s: refusing to wake: %s", rt.Host, p.reason) |
| } |
| - return b.startChildAt(ctx, rt, rt.Socket, 0) |
| + return b.startChildAt(ctx, rt, rt.Socket, 0, "") |
| } |
| // StartGen is Start for generation n (zero-downtime spec §7.1). perGen false |
| @@ -1220,6 +1286,39 @@ func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, strin |
| // whose counter started at 1 like this one's). A sibling generation's path |
| // is never touched. |
| func (b *ExecBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, perGen bool) (Proc, Gen, error) { |
| + return b.startGenFrom(ctx, rt, n, perGen, nil) |
| +} |
| + |
| +// StartGenExe is StartGen on the identity a wake recorded (settings spec |
| +// §4.2): the same generation, socket and tier, but the process runs |
| +// id.StagedPath, checked against id.Digest first. A copy that changed is a |
| +// failed start, and nothing runs (never the live link instead). A flat or |
| +// foreign row execs the file by path. A nested row asks root to bind it in |
| +// place of the live link: the original run unit while GenContainment is |
| +// off (generation 0), generation n's unit while it is on. The handle names |
| +// id's build, path, digest and demand, so a sidecar can bind the same file. |
| +func (b *ExecBackend) StartGenExe(ctx context.Context, rt registry.Route, n uint64, perGen bool, id release.StagedIdentity) (Proc, Gen, error) { |
| + if err := release.VerifyStaged(id); err != nil { |
| + return nil, Gen{}, fmt.Errorf("%s: not started on %s: %w", rt.Host, id.Version, err) |
| + } |
| + proc, g, err := b.startGenFrom(ctx, rt, n, perGen, &id) |
| + if err != nil { |
| + return proc, g, err |
| + } |
| + g.Build, g.StagedPath, g.Digest, g.Demand = id.Version, id.StagedPath, id.Digest, id.Demand |
| + return proc, g, nil |
| +} |
| + |
| +// startGenFrom is StartGen's body. A nil id is the live link (StartGen); a |
| +// non-nil one is the verified file StartGenExe names. |
| +func (b *ExecBackend) startGenFrom(ctx context.Context, rt registry.Route, n uint64, perGen bool, id *release.StagedIdentity) (Proc, Gen, error) { |
| + var ( |
| + flatExe string |
| + unitExe *instancectl.StagedExe |
| + ) |
| + if id != nil { |
| + flatExe, unitExe = id.StagedPath, stagedExeOf(*id) |
| + } |
| p := b.pathsFor(rt) |
| sock := rt.Socket |
| if perGen { |
| @@ -1238,7 +1337,7 @@ func (b *ExecBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, |
| switch p.layout { |
| case LayoutNested: |
| // prepareAt removes the stale file at sock for the nested layout. |
| - proc, ver, err = b.startUnitAt(ctx, rt, sock, n) |
| + proc, ver, err = b.startUnitAt(ctx, rt, sock, n, unitExe) |
| case LayoutFenced, LayoutConflict: |
| return nil, Gen{}, fmt.Errorf("%s: refusing to wake: %s", rt.Host, p.reason) |
| default: |
| @@ -1254,7 +1353,7 @@ func (b *ExecBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, |
| return nil, Gen{}, fmt.Errorf("%s: generation socket: %w", rt.Host, err) |
| } |
| } |
| - proc, ver, err = b.startChildAt(ctx, rt, sock, n) |
| + proc, ver, err = b.startChildAt(ctx, rt, sock, n, flatExe) |
| } |
| g := Gen{N: n, Socket: sock, Control: sock + ".control"} |
| if err != nil { |
| @@ -1295,11 +1394,21 @@ func clearOwnGenSocket(sock string) error { |
| // generation socket (StartGen). n is the generation this spawn is (StartGen's |
| // n; 0 from Start). It only places the spawn's cgroup, and only under |
| // GenContainment (placeFor). |
| -func (b *ExecBackend) startChildAt(ctx context.Context, rt registry.Route, sock string, n uint64) (Proc, string, error) { |
| - p, spawnVer, err := b.spawnChildFrom(ctx, rt, sock, n, "", cgroup.Limit{}) |
| +// |
| +// exe is "" for the live link, or the staged file a wake recorded |
| +// (StartGenExe). spawnChildFrom leaves a staged spawn's cache purge to the |
| +// swap's switch, but a wake's staged spawn is the host's only process, so |
| +// its purge happens here, at the same moment, before the health wait. |
| +func (b *ExecBackend) startChildAt(ctx context.Context, rt registry.Route, sock string, n uint64, exe string) (Proc, string, error) { |
| + p, spawnVer, err := b.spawnChildFrom(ctx, rt, sock, n, exe, cgroup.Limit{}) |
| if err != nil { |
| return p, "", err |
| } |
| + if exe != "" && b.Cache != nil { |
| + if purged := b.Cache.PurgeHost(rt.Host); purged > 0 { |
| + log.Printf("activator: purged %d cached response(s) for %s on spawn", purged, rt.Host) |
| + } |
| + } |
| if err := waitHealthyCtx(ctx, sock, spawnHealthTimeout); err != nil { |
| stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout) |
| defer cancel() |
| @@ -1742,10 +1851,11 @@ func (b *ExecBackend) spawnUnitFrom(ctx context.Context, rt registry.Route, sock |
| // composition and health gate as startChild, but the process itself is |
| // started by the root instance-control channel from a launch file rather |
| // than exec'd directly. The agent has no privilege to enter the tenant's |
| -// namespace, only the control service does. It is spawnUnitFrom on the live |
| -// link, then the health wait. |
| -func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock string, n uint64) (Proc, string, error) { |
| - p, spawnVer, err := b.spawnUnitFrom(ctx, rt, sock, n, nil, cgroup.Limit{}) |
| +// namespace, only the control service does. It is spawnUnitFrom, then the |
| +// health wait. exe is nil for the live link, or the staged file a wake |
| +// recorded (StartGenExe), which root binds in place of the link. |
| +func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock string, n uint64, exe *instancectl.StagedExe) (Proc, string, error) { |
| + p, spawnVer, err := b.spawnUnitFrom(ctx, rt, sock, n, exe, cgroup.Limit{}) |
| if err != nil { |
| return p, "", err |
| } |
diff --git a/internal/activator/containment.go b/internal/activator/containment.go| index fdae43c47..b948de49f 100644 |
| --- a/internal/activator/containment.go |
| +++ b/internal/activator/containment.go |
| @@ -164,6 +164,24 @@ func (b *ExecBackend) genControl() (GenControl, error) { |
| return gc, nil |
| } |
| +// errNoStagedExeControl refuses a start or seed of an original template |
| +// that names a staged build when the control channel cannot name one. |
| +// Binding the live link instead would run a build nobody recorded. |
| +var errNoStagedExeControl = errors.New("activator: the instance control channel cannot name an executable for this unit") |
| + |
| +// stagedExeControl is Control's original-template executable half, or |
| +// errNoStagedExeControl. |
| +func (b *ExecBackend) stagedExeControl() (StagedExeControl, error) { |
| + if b.Control == nil { |
| + return nil, errNoStagedExeControl |
| + } |
| + xc, ok := b.Control.(StagedExeControl) |
| + if !ok { |
| + return nil, errNoStagedExeControl |
| + } |
| + return xc, nil |
| +} |
| + |
| // unitGen is the generation a nested run unit is started as: 0 (the |
| // original template) while GenContainment is off, n otherwise. n of 0 |
| // with containment on is refused. |
| @@ -179,11 +197,26 @@ func (b *ExecBackend) unitGen(n uint64) (uint64, error) { |
| // startRole starts one role's unit and returns its invocation: the |
| // original template through Control, a generation through GenControl. exe |
| -// names a staged build for a generation (nil: the live link, plan 2's |
| -// StartGen), and tier is the generation's own (zero: b.tier(rt)). |
| +// names a staged build (nil: the live link, plan 2's StartGen), and tier |
| +// is that build's own (zero: b.tier(rt)). |
| +// |
| +// At generation 0 a named exe is the build a cold wake recorded (settings |
| +// spec §4.2), started through StagedExeControl. A channel without it |
| +// refuses the start: dropping exe would bind the live link, a build |
| +// nobody recorded. |
| func (b *ExecBackend) startRole(ctx context.Context, rt registry.Route, role instancectl.Role, gen uint64, exe *instancectl.StagedExe, tier cgroup.Limit) (string, error) { |
| if gen == 0 { |
| - return b.Control.Start(ctx, rt.Host, role, b.tier(rt).MemoryMax) |
| + if exe == nil { |
| + return b.Control.Start(ctx, rt.Host, role, b.tier(rt).MemoryMax) |
| + } |
| + xc, err := b.stagedExeControl() |
| + if err != nil { |
| + return "", fmt.Errorf("%s: not started on %s: %w", rt.Host, exe.Version, err) |
| + } |
| + if tier == (cgroup.Limit{}) { |
| + tier = b.tier(rt) |
| + } |
| + return xc.StartExe(ctx, rt.Host, role, tier.MemoryMax, *exe) |
| } |
| gc, err := b.genControl() |
| if err != nil { |
diff --git a/internal/activator/control.go b/internal/activator/control.go| index 823b8d9ba..48a4b79a9 100644 |
| --- a/internal/activator/control.go |
| +++ b/internal/activator/control.go |
| @@ -42,6 +42,21 @@ type GenControl interface { |
| var _ GenControl = (*instancectl.Client)(nil) |
| +// StagedExeControl is the original templates' half of naming an |
| +// executable (settings spec §4.2): the run unit at generation 0 and the |
| +// seed unit, started on a staged build root validates and binds in place |
| +// of the live link. A cold wake runs the build it recorded through it, |
| +// with generations on or off. It is separate from Control and GenControl |
| +// so a Control without it (a fake, an older client) still satisfies both; |
| +// the backend asks for it only when it has a file to name, and refuses |
| +// the start without it (stagedExeControl), never binding the link instead. |
| +type StagedExeControl interface { |
| + StartExe(ctx context.Context, host string, role instancectl.Role, memoryMax int64, x instancectl.StagedExe) (string, error) |
| + SeedExe(ctx context.Context, host string, memoryMax int64, x instancectl.StagedExe) (instancectl.SeedResponse, error) |
| +} |
| + |
| +var _ StagedExeControl = (*instancectl.Client)(nil) |
| + |
| // UnitState answers `systemctl show -p ActiveState,InvocationID <unit>`, |
| // which needs no privilege. An unloaded unit answers inactive with an |
| // empty invocation. |
diff --git a/internal/activator/identity.go b/internal/activator/identity.go| new file mode 100644 |
| index 000000000..d400d0e4d |
| --- /dev/null |
| +++ b/internal/activator/identity.go |
| @@ -0,0 +1,66 @@ |
| +package activator |
| + |
| +import ( |
| + "context" |
| + |
| + "github.com/carlosframework/platform/internal/instancectl" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// identityBackend is the exec backend's identity-pinned spawn and seed |
| +// (settings spec §4.2, Astra I1 #5). A cold wake captures ONE staged |
| +// identity (LiveIdentity), records it, and then seeds and spawns exactly |
| +// that file by path, verified against its digest. Neither call execs |
| +// live/<host>, so root repointing the link between the record and the exec |
| +// cannot make a different build touch the instance's data. |
| +type identityBackend interface { |
| + // LiveIdentity materializes host's live link if it is missing (the |
| + // Materialize hook and its root-stage relay), then resolves it ONCE |
| + // into the file it names and that file's digest. Version is "" for a |
| + // link outside the version store; the caller decides what that means. |
| + LiveIdentity(ctx context.Context, rt registry.Route) (release.StagedIdentity, error) |
| + // StartGenExe is StartGen (generation n, perGen choosing the socket) |
| + // with the executable named: id.StagedPath, verified against id.Digest |
| + // first. A copy that changed is a failed start, never a fallback to the |
| + // link. The Gen it returns carries id's build, path, digest and demand, |
| + // so a later sidecar binds the same file. |
| + StartGenExe(ctx context.Context, rt registry.Route, n uint64, perGen bool, id release.StagedIdentity) (Proc, Gen, error) |
| + // SeedExe is Seed with the executable named, verified the same way. |
| + SeedExe(ctx context.Context, rt registry.Route, id release.StagedIdentity) error |
| +} |
| + |
| +var _ identityBackend = (*ExecBackend)(nil) |
| + |
| +// identityPicker is a composed backend that answers per route. |
| +type identityPicker interface { |
| + identityFor(rt registry.Route) identityBackend |
| +} |
| + |
| +// identityFor is the backend that can pin an executable for rt, or nil |
| +// (rt then keeps today's Start and Seed, and nothing is recorded for it). |
| +// Production's SplitBackend serves unit rows too, so a composed backend |
| +// answers per route, never by a type assertion on itself alone. |
| +func identityFor(b Backend, rt registry.Route) identityBackend { |
| + if p, ok := b.(identityPicker); ok { |
| + return p.identityFor(rt) |
| + } |
| + ib, _ := b.(identityBackend) |
| + return ib |
| +} |
| + |
| +// identityFor is the exec half for an exec row and nil for a unit row: |
| +// systemd execs a unit row's live link itself, so nothing here can pin its |
| +// file. SplitBackend does not implement identityBackend itself. |
| +func (b *SplitBackend) identityFor(rt registry.Route) identityBackend { |
| + if rt.Backing == registry.BackingUnit { |
| + return nil |
| + } |
| + ib, _ := b.pick(rt).(identityBackend) |
| + return ib |
| +} |
| + |
| +// stagedExeOf is id as root's control channel names an executable. |
| +func stagedExeOf(id release.StagedIdentity) *instancectl.StagedExe { |
| + return &instancectl.StagedExe{Path: id.StagedPath, Version: id.Version, Digest: id.Digest} |
| +} |
diff --git a/internal/activator/identity_test.go b/internal/activator/identity_test.go| new file mode 100644 |
| index 000000000..1b07a5f95 |
| --- /dev/null |
| +++ b/internal/activator/identity_test.go |
| @@ -0,0 +1,447 @@ |
| +package activator |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/edge" |
| + "github.com/carlosframework/platform/internal/instancectl" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// stageVersion writes <OptDir>/versions/<ver>/app with body and points |
| +// host's live link at it. Each version's body differs, so each has its own |
| +// digest. |
| +func stageVersion(t *testing.T, b *ExecBackend, host, ver, body string) string { |
| + t.Helper() |
| + bin := filepath.Join(b.OptDir, "versions", ver, "app") |
| + if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(bin, []byte("#!/bin/sh\n# "+ver+"\n"+body), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + repointLive(t, b, host, bin) |
| + return bin |
| +} |
| + |
| +// repointLive is root's repoint: live/<host> names target from now on. |
| +func repointLive(t *testing.T, b *ExecBackend, host, target string) { |
| + t.Helper() |
| + if err := os.MkdirAll(b.liveDir(), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + tmp := b.liveLink(host) + ".tmp" |
| + os.Remove(tmp) |
| + if err := os.Symlink(target, tmp); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Rename(tmp, b.liveLink(host)); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// servingBody is a staged build that records the path it was exec'd by |
| +// ($0) in marker, then becomes the test binary's fake app (TestMain), so it |
| +// binds --socket and answers the health wait. |
| +func servingBody(t *testing.T, marker string) string { |
| + t.Helper() |
| + self, err := os.Executable() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Setenv("ACTIVATOR_FAKE_APP", "1") |
| + return "printf '%s' \"$0\" > '" + marker + "'\nexec '" + self + "' \"$@\"\n" |
| +} |
| + |
| +// Settings spec §4.2: the identity is captured per route. A unit row on a |
| +// SplitBackend has none, whatever its backend could do; an exec row gets |
| +// the exec half. A bare ExecBackend answers for itself, and a backend |
| +// without the capability answers nil. |
| +func TestIdentityIsPerRoute(t *testing.T) { |
| + exec := testExecBackend(t) |
| + // The unit half is an ExecBackend too, so a nil for the unit row is the |
| + // row's answer, not an accident of its backend's type. |
| + split := &SplitBackend{Exec: exec, Unit: testExecBackend(t)} |
| + unitRow := registry.Route{Host: "u.example.com", Backing: registry.BackingUnit} |
| + execRow := registry.Route{Host: "e.example.com"} |
| + if ib := identityFor(split, unitRow); ib != nil { |
| + t.Fatalf("a unit row got an identity backend: %T", ib) |
| + } |
| + if ib := identityFor(split, execRow); ib != identityBackend(exec) { |
| + t.Fatalf("an exec row got %v, want the exec half", ib) |
| + } |
| + if ib := identityFor(exec, execRow); ib != identityBackend(exec) { |
| + t.Fatalf("a bare ExecBackend answered %v for itself", ib) |
| + } |
| + if ib := identityFor(&fakeBackend{}, execRow); ib != nil { |
| + t.Fatalf("a backend without the capability answered %T", ib) |
| + } |
| +} |
| + |
| +// A cold default instance has no live link until the Materialize hook |
| +// stages one. LiveIdentity runs the hook first and captures what it |
| +// materialized, read once. |
| +func TestLiveIdentityMaterializesFirst(t *testing.T) { |
| + b := testExecBackend(t) |
| + host := "cold.example.com" |
| + var bin string |
| + calls := 0 |
| + b.Materialize = func(_ context.Context, h string) error { |
| + calls++ |
| + bin = stageVersion(t, b, h, "v1", "exit 0\n") |
| + return nil |
| + } |
| + id, err := b.LiveIdentity(context.Background(), registry.Route{Host: host}) |
| + if err != nil { |
| + t.Fatalf("LiveIdentity: %v", err) |
| + } |
| + want, _ := release.FileSHA256(bin) |
| + if calls != 1 || id.Version != "v1" || id.StagedPath != bin || id.Digest != want { |
| + t.Fatalf("materialized %d times, identity %+v; want v1 at %s (%s)", calls, id, bin, want) |
| + } |
| + // A link that exists is not materialized again. |
| + if _, err := b.LiveIdentity(context.Background(), registry.Route{Host: host}); err != nil || calls != 1 { |
| + t.Fatalf("second capture: %v, materialized %d times", err, calls) |
| + } |
| + // A materialize failure is the capture's failure. |
| + b.Materialize = func(context.Context, string) error { return errors.New("controld: 502") } |
| + if _, err := b.LiveIdentity(context.Background(), registry.Route{Host: "other.example.com"}); err == nil || !strings.Contains(err.Error(), "502") { |
| + t.Fatalf("a failed materialize = %v", err) |
| + } |
| +} |
| + |
| +// Flat, generations off: StartGenExe execs the recorded file by path on the |
| +// canonical socket, though root repointed the link after the capture. The |
| +// handle names that build, its path and digest, and the host's cache is |
| +// purged as on any wake. A copy whose bytes changed starts nothing. |
| +func TestStartGenExeFlatExecsTheRecordedFile(t *testing.T) { |
| + requireUnixSockets(t) |
| + b := testExecBackend(t) |
| + b.RunDirOverride = shortRunDir(t) |
| + cache := edge.NewCache(edge.DefaultCacheMaxEntries, edge.DefaultCacheMaxBodyBytes) |
| + b.Cache = cache |
| + host := "flat.example.com" |
| + marker := filepath.Join(t.TempDir(), "argv0") |
| + a := stageVersion(t, b, host, "vA", servingBody(t, marker)) |
| + rt := flatGenRoute(b, host) |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + id, err := b.LiveIdentity(ctx, rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stageVersion(t, b, host, "vB", servingBody(t, marker)) // root's repoint lands after the record |
| + |
| + stale := edge.CacheKey{Host: host, Method: "GET", Path: "/"} |
| + cache.Put(stale, "", &edge.CachedResponse{StatusCode: 200, Header: http.Header{}, Body: []byte("old"), StoredAt: time.Now(), TTL: time.Minute}) |
| + |
| + proc, g, err := b.StartGenExe(ctx, rt, 3, false, id) |
| + if err != nil { |
| + t.Fatalf("StartGenExe: %v", err) |
| + } |
| + defer proc.Stop(context.Background()) //nolint:errcheck |
| + if got := waitForFile(t, marker, 2*time.Second); got != a { |
| + t.Fatalf("execd %q, want the recorded %q", got, a) |
| + } |
| + want := Gen{N: 3, Socket: rt.Socket, Control: rt.Socket + ".control", Build: "vA", StagedPath: a, Digest: id.Digest} |
| + if g != want { |
| + t.Fatalf("handle = %+v, want %+v", g, want) |
| + } |
| + if _, ok := cache.GetFor(stale, httptest.NewRequest("GET", "http://"+host+"/", nil)); ok { |
| + t.Fatal("the host's cache survived the spawn") |
| + } |
| + stopCtx, stopCancel := context.WithTimeout(context.Background(), 5*time.Second) |
| + defer stopCancel() |
| + if err := proc.Stop(stopCtx); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + os.Remove(marker) |
| + bad := id |
| + bad.Digest = strings.Repeat("0", 64) |
| + p, _, err := b.StartGenExe(ctx, rt, 4, false, bad) |
| + if !errors.Is(err, release.ErrStagedCopyChanged) || p != nil { |
| + t.Fatalf("a changed copy = %v, %v; want ErrStagedCopyChanged and no process", p, err) |
| + } |
| + time.Sleep(200 * time.Millisecond) |
| + if _, err := os.Stat(marker); !os.IsNotExist(err) { |
| + t.Fatal("a refused identity ran something") |
| + } |
| +} |
| + |
| +// Flat, generations on: the recorded file binds generation n's own socket, |
| +// as StartGen does. |
| +func TestStartGenExeFlatPerGenBindsTheGenerationSocket(t *testing.T) { |
| + requireUnixSockets(t) |
| + b := testExecBackend(t) |
| + b.RunDirOverride = shortRunDir(t) |
| + host := "pergen.example.com" |
| + marker := filepath.Join(t.TempDir(), "argv0") |
| + a := stageVersion(t, b, host, "vA", servingBody(t, marker)) |
| + rt := flatGenRoute(b, host) |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + id, err := b.LiveIdentity(ctx, rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stageVersion(t, b, host, "vB", servingBody(t, marker)) |
| + proc, g, err := b.StartGenExe(ctx, rt, 9, true, id) |
| + if err != nil { |
| + t.Fatalf("StartGenExe: %v", err) |
| + } |
| + defer proc.Stop(context.Background()) //nolint:errcheck |
| + own := genSocketPathIn(filepath.Join(b.RunDirOverride, "g"), host, 9) |
| + if g.Socket != own || g.Control != own+".control" || !isSocket(own) { |
| + t.Fatalf("handle = %+v, want generation 9's bound socket %s", g, own) |
| + } |
| + if got := waitForFile(t, marker, 2*time.Second); got != a { |
| + t.Fatalf("execd %q, want the recorded %q", got, a) |
| + } |
| + if _, err := os.Lstat(rt.Socket); !os.IsNotExist(err) { |
| + t.Fatalf("the canonical path was bound with generations on (lstat: %v)", err) |
| + } |
| +} |
| + |
| +// A foreign row keeps its canonical socket whatever perGen says, as |
| +// StartStaged and StartGen keep it, and execs the recorded file there. |
| +func TestStartGenExeForeignRunsOnTheCanonicalSocket(t *testing.T) { |
| + requireUnixSockets(t) |
| + b := testExecBackend(t) |
| + host := "foreign.example.com" |
| + marker := filepath.Join(t.TempDir(), "argv0") |
| + a := stageVersion(t, b, host, "vA", servingBody(t, marker)) |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: filepath.Join(b.DataDir, host+".db"), Socket: tmpSock(t)} |
| + if l, _ := b.Layout(rt); l != LayoutForeign { |
| + t.Fatalf("fixture layout = %v, want LayoutForeign", l) |
| + } |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + id, err := b.LiveIdentity(ctx, rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stageVersion(t, b, host, "vB", servingBody(t, marker)) |
| + proc, g, err := b.StartGenExe(ctx, rt, 2, true, id) |
| + if err != nil { |
| + t.Fatalf("StartGenExe: %v", err) |
| + } |
| + defer proc.Stop(context.Background()) //nolint:errcheck |
| + if g.Socket != rt.Socket || g.Build != "vA" { |
| + t.Fatalf("handle = %+v, want vA on the canonical %s", g, rt.Socket) |
| + } |
| + if got := waitForFile(t, marker, 2*time.Second); got != a { |
| + t.Fatalf("execd %q, want the recorded %q", got, a) |
| + } |
| +} |
| + |
| +// Flat seed: SeedExe runs the recorded file, though the link moved after |
| +// the capture, and Seed still runs the link. A changed copy seeds nothing. |
| +func TestSeedExeFlatRunsTheRecordedFile(t *testing.T) { |
| + b := testExecBackend(t) |
| + host := "seedexe.example.com" |
| + marker := filepath.Join(t.TempDir(), "argv0") |
| + record := "printf '%s' \"$0\" > '" + marker + "'\n" |
| + a := stageVersion(t, b, host, "vA", record) |
| + rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`} |
| + id, err := b.LiveIdentity(context.Background(), rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + bPath := stageVersion(t, b, host, "vB", record) |
| + if err := b.SeedExe(context.Background(), rt, id); err != nil { |
| + t.Fatalf("SeedExe: %v", err) |
| + } |
| + if got := waitForFile(t, marker, time.Second); got != a { |
| + t.Fatalf("seeded with %q, want the recorded %q", got, a) |
| + } |
| + |
| + os.Remove(marker) |
| + bad := id |
| + bad.Digest = strings.Repeat("0", 64) |
| + if err := b.SeedExe(context.Background(), rt, bad); !errors.Is(err, release.ErrStagedCopyChanged) { |
| + t.Fatalf("a changed copy = %v, want ErrStagedCopyChanged", err) |
| + } |
| + if _, err := os.Stat(marker); !os.IsNotExist(err) { |
| + t.Fatal("a refused identity seeded") |
| + } |
| + |
| + if err := b.Seed(context.Background(), rt); err != nil { |
| + t.Fatalf("Seed: %v", err) |
| + } |
| + if got := waitForFile(t, marker, time.Second); got != b.liveLink(host) { |
| + t.Fatalf("Seed ran %q, want the live link (today's, which resolves to %s)", got, bPath) |
| + } |
| +} |
| + |
| +// identityControl is a root control channel that can name an executable |
| +// for the original templates (StagedExeControl) and for a generation |
| +// (GenControl). Each verb is recorded with the file it named. |
| +type identityControl struct{ *fakeControl } |
| + |
| +func exeLine(verb, host, what string, x instancectl.StagedExe) string { |
| + return fmt.Sprintf("%s %s %s %s %s %s", verb, host, what, x.Path, x.Version, x.Digest) |
| +} |
| + |
| +func (c identityControl) StartExe(_ context.Context, host string, role instancectl.Role, _ int64, x instancectl.StagedExe) (string, error) { |
| + c.rec(exeLine("startexe", host, string(role), x)) |
| + if c.onStart != nil { |
| + c.onStart() |
| + } |
| + return c.inv, c.startErr |
| +} |
| +func (c identityControl) SeedExe(_ context.Context, host string, _ int64, x instancectl.StagedExe) (instancectl.SeedResponse, error) { |
| + c.rec(exeLine("seedexe", host, "seed", x)) |
| + return c.seed, nil |
| +} |
| +func (c identityControl) StartGen(_ context.Context, host string, n uint64, _ int64) (string, error) { |
| + c.rec(fmt.Sprintf("startgen %s %d", host, n)) |
| + return "", errors.New("test: the live-link generation start was used") |
| +} |
| +func (c identityControl) StartGenExe(_ context.Context, host string, n uint64, _ int64, x instancectl.StagedExe) (string, error) { |
| + c.rec(exeLine("startgenexe", host, fmt.Sprint(n), x)) |
| + if c.onStart != nil { |
| + c.onStart() |
| + } |
| + return c.inv, c.startErr |
| +} |
| +func (c identityControl) StartSidecarExe(_ context.Context, host string, _ int64, x instancectl.StagedExe) (string, error) { |
| + c.rec(exeLine("startsidecarexe", host, "sidecar", x)) |
| + return c.inv, nil |
| +} |
| +func (c identityControl) StopGen(_ context.Context, host string, n uint64, kill bool) error { |
| + c.rec(fmt.Sprintf("stopgen %s %d kill=%v", host, n, kill)) |
| + return nil |
| +} |
| +func (c identityControl) ResetGen(context.Context, string, uint64) error { return nil } |
| +func (c identityControl) Parent(context.Context, string, int64) error { return nil } |
| + |
| +// noPlainVerbs fails when ctl saw a start or seed that names no executable: |
| +// the live link's layout. |
| +func noPlainVerbs(t *testing.T, ctl *fakeControl) { |
| + t.Helper() |
| + for _, line := range strings.Split(ctl.joined(), "\n") { |
| + if strings.HasPrefix(line, "start ") || strings.HasPrefix(line, "seed ") || strings.HasPrefix(line, "startgen ") { |
| + t.Errorf("a live-link verb was used: %q in\n%s", line, ctl.joined()) |
| + } |
| + } |
| +} |
| + |
| +// Nested: StartGenExe asks root for the recorded file, by path, version and |
| +// digest, after root's repoint of the live link to B. With generations off |
| +// that is the original run unit (generation 0); with them on, generation n. |
| +// Never a plain Start or StartGen. |
| +func TestNestedStartExeNamesTheRecordedFile(t *testing.T) { |
| + for _, tc := range []struct { |
| + name string |
| + gens bool |
| + n uint64 |
| + want func(host string, x instancectl.StagedExe) string |
| + }{ |
| + {"generation 0", false, 1, func(h string, x instancectl.StagedExe) string { return exeLine("startexe", h, "run", x) }}, |
| + {"generation n", true, 6, func(h string, x instancectl.StagedExe) string { return exeLine("startgenexe", h, "6", x) }}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + ctl := &fakeControl{inv: "inv1"} |
| + unit := &fakeUnit{} |
| + b, rt := nestedBackend(t, ctl, unit) |
| + b.Control = identityControl{ctl} |
| + b.GenContainment = tc.gens |
| + a := stageVersion(t, b, rt.Host, "vA", "exit 0\n") |
| + id, err := b.LiveIdentity(context.Background(), rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stageVersion(t, b, rt.Host, "vB", "exit 0\n") |
| + sock := rt.Socket |
| + if tc.gens { |
| + sock = nestedGenSocketIn(b.pathsFor(rt).runDir, tc.n) |
| + } |
| + var stopServing func() |
| + ctl.onStart = func() { stopServing = serveOK(t, sock) } |
| + defer func() { |
| + if stopServing != nil { |
| + stopServing() |
| + } |
| + }() |
| + unit.set("active", "inv1", true, true) |
| + p, g, err := b.StartGenExe(context.Background(), rt, tc.n, tc.gens, id) |
| + if err != nil { |
| + t.Fatalf("StartGenExe: %v", err) |
| + } |
| + x := instancectl.StagedExe{Path: a, Version: "vA", Digest: id.Digest} |
| + if !strings.Contains(ctl.joined(), tc.want(rt.Host, x)) { |
| + t.Fatalf("root was asked:\n%s\nwant %q", ctl.joined(), tc.want(rt.Host, x)) |
| + } |
| + noPlainVerbs(t, ctl) |
| + if g.Build != "vA" || g.StagedPath != a || g.Digest != id.Digest || g.Socket != sock { |
| + t.Fatalf("handle = %+v", g) |
| + } |
| + unit.set("inactive", "inv1", false, false) |
| + waitFor(t, "the unit to exit", p.Exited) |
| + }) |
| + } |
| +} |
| + |
| +// Nested seed: SeedExe asks root to run the seed unit on the recorded file, |
| +// not a plain Seed on the link root moved to B. A changed copy asks root |
| +// for nothing. |
| +func TestNestedSeedExeNamesTheRecordedFile(t *testing.T) { |
| + ctl := &fakeControl{seed: instancectl.SeedResponse{InvocationID: "s1", Result: "success", Status: "0", Code: "exited"}} |
| + b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| + b.Control = identityControl{ctl} |
| + rt.Seed = `["seed"]` |
| + a := stageVersion(t, b, rt.Host, "vA", "exit 0\n") |
| + id, err := b.LiveIdentity(context.Background(), rt) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stageVersion(t, b, rt.Host, "vB", "exit 0\n") |
| + if err := b.SeedExe(context.Background(), rt, id); err != nil { |
| + t.Fatalf("SeedExe: %v", err) |
| + } |
| + x := instancectl.StagedExe{Path: a, Version: "vA", Digest: id.Digest} |
| + if want := exeLine("seedexe", rt.Host, "seed", x); !strings.Contains(ctl.joined(), want) { |
| + t.Fatalf("root was asked:\n%s\nwant %q", ctl.joined(), want) |
| + } |
| + noPlainVerbs(t, ctl) |
| + |
| + ctl.mu.Lock() |
| + ctl.calls = nil |
| + ctl.mu.Unlock() |
| + bad := id |
| + bad.Digest = strings.Repeat("0", 64) |
| + if err := b.SeedExe(context.Background(), rt, bad); !errors.Is(err, release.ErrStagedCopyChanged) { |
| + t.Fatalf("a changed copy = %v, want ErrStagedCopyChanged", err) |
| + } |
| + if ctl.joined() != "" { |
| + t.Fatalf("a refused identity reached root:\n%s", ctl.joined()) |
| + } |
| +} |
| + |
| +// A control channel that cannot name an executable for the original |
| +// template refuses a generation-0 start that names one. It never drops the |
| +// file and binds the live link instead. |
| +func TestStartRoleRefusesAnExeTheControlCannotName(t *testing.T) { |
| + b := testExecBackend(t) |
| + ctl := &fakeControl{inv: "inv"} |
| + b.Control = ctl |
| + x := instancectl.StagedExe{Path: "/opt/carlos/versions/v2/blog", Version: "v2", Digest: strings.Repeat("a", 64)} |
| + if _, err := b.startRole(context.Background(), registry.Route{Host: "n.example.com"}, instancectl.RoleRun, 0, &x, cgroup.Limit{}); err == nil { |
| + t.Fatal("a generation-0 start with an executable the control cannot name was accepted") |
| + } |
| + if ctl.joined() != "" { |
| + t.Fatalf("root was asked:\n%s", ctl.joined()) |
| + } |
| +} |
diff --git a/internal/instancectl/client.go b/internal/instancectl/client.go| index 3c2a9678b..57a4a5334 100644 |
| --- a/internal/instancectl/client.go |
| +++ b/internal/instancectl/client.go |
| @@ -153,13 +153,38 @@ func (c *Client) Start(ctx context.Context, host string, role Role, memoryMax in |
| // Seed runs the seed unit to completion and returns its retained result. |
| func (c *Client) Seed(ctx context.Context, host string, memoryMax int64) (SeedResponse, error) { |
| + return c.seedQuery(ctx, host, memoryMax, nil) |
| +} |
| + |
| +// SeedExe is Seed with the seed's executable named: the staged build the |
| +// wake recorded (settings spec §4.2), which root validates and binds in |
| +// place of the live link. A 403 is root's refusal, reported as |
| +// ErrStagedExe: nothing was laid out or run. |
| +func (c *Client) SeedExe(ctx context.Context, host string, memoryMax int64, x StagedExe) (SeedResponse, error) { |
| + return c.seedQuery(ctx, host, memoryMax, &x) |
| +} |
| + |
| +func (c *Client) seedQuery(ctx context.Context, host string, memoryMax int64, x *StagedExe) (SeedResponse, error) { |
| q := hostRole(host, "") |
| q.Set("memory_max", strconv.FormatInt(memoryMax, 10)) |
| + if x != nil { |
| + setStagedExe(q, *x) |
| + } |
| var out SeedResponse |
| - _, _, err := c.do(ctx, DeadlineSeed, http.MethodPost, "/seed", q, &out) |
| + status, _, err := c.do(ctx, DeadlineSeed, http.MethodPost, "/seed", q, &out) |
| + if err != nil && status == http.StatusForbidden && x != nil { |
| + return out, fmt.Errorf("%w: %v", ErrStagedExe, err) |
| + } |
| return out, err |
| } |
| +// setStagedExe puts x on a start or seed query. |
| +func setStagedExe(q url.Values, x StagedExe) { |
| + q.Set("exe", x.Path) |
| + q.Set("version", x.Version) |
| + q.Set("digest", x.Digest) |
| +} |
| + |
| func (c *Client) Stop(ctx context.Context, host string, role Role) error { |
| _, _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/stop", hostRole(host, role), nil) |
| return err |
| @@ -191,12 +216,19 @@ func (c *Client) StartGenExe(ctx context.Context, host string, n uint64, memoryM |
| return c.startUnitQuery(ctx, genValues(host, n), memoryMax, &x) |
| } |
| -// StartSidecarExe is Start for the legacy sidecar role with its executable |
| -// named: the serving generation's staged build, which root validates and |
| -// mounts in place of the live link (zero-downtime spec §7.5). A 403 is |
| -// root's refusal, reported as ErrStagedExe. |
| +// StartExe is Start with the unit's executable named: a staged build root |
| +// validates and mounts in place of the live link, for the original |
| +// template's run unit (the build a cold wake recorded, settings spec §4.2) |
| +// or the legacy sidecar. A 403 is root's refusal, reported as |
| +// ErrStagedExe. |
| +func (c *Client) StartExe(ctx context.Context, host string, role Role, memoryMax int64, x StagedExe) (string, error) { |
| + return c.startUnitQuery(ctx, hostRole(host, role), memoryMax, &x) |
| +} |
| + |
| +// StartSidecarExe is StartExe for the legacy sidecar role: the serving |
| +// generation's staged build (zero-downtime spec §7.5). |
| func (c *Client) StartSidecarExe(ctx context.Context, host string, memoryMax int64, x StagedExe) (string, error) { |
| - return c.startUnitQuery(ctx, hostRole(host, RoleSidecar), memoryMax, &x) |
| + return c.StartExe(ctx, host, RoleSidecar, memoryMax, x) |
| } |
| // startUnitQuery is /start for a generation or the sidecar, optionally with |
| @@ -206,9 +238,7 @@ func (c *Client) StartSidecarExe(ctx context.Context, host string, memoryMax int |
| func (c *Client) startUnitQuery(ctx context.Context, q url.Values, memoryMax int64, x *StagedExe) (string, error) { |
| q.Set("memory_max", strconv.FormatInt(memoryMax, 10)) |
| if x != nil { |
| - q.Set("exe", x.Path) |
| - q.Set("version", x.Version) |
| - q.Set("digest", x.Digest) |
| + setStagedExe(q, *x) |
| } |
| var out StartResponse |
| status, _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/start", q, &out) |
diff --git a/internal/instancectl/server.go b/internal/instancectl/server.go| index 1571cd988..bfd6ba99f 100644 |
| --- a/internal/instancectl/server.go |
| +++ b/internal/instancectl/server.go |
| @@ -305,7 +305,10 @@ type verbShape struct { |
| tier bool |
| gen bool |
| kill bool |
| - exe bool // a start verb: a generation's, or the legacy sidecar's, staged build |
| + exe bool // a start or seed verb: the staged build its unit binds |
| + // seed is the /seed verb, which names no role: its unit is the seed |
| + // unit, which may bind a staged build (settings spec §4.2). |
| + seed bool |
| } |
| func parseRequest(r *http.Request, v verbShape) (request, error) { |
| @@ -366,11 +369,14 @@ func parseRequest(r *http.Request, v verbShape) (request, error) { |
| return request{}, errors.New("this verb takes no staged executable") |
| } |
| if x := q.Get("exe"); x != "" { |
| - // A generation start, or the legacy sidecar (zero-downtime spec |
| - // §7.5: it runs its instance's serving artifact). The original |
| - // template's run unit keeps the live link. |
| - if req.gen == 0 && req.role != RoleSidecar { |
| - return request{}, errors.New("exe is only for a generation start or a sidecar") |
| + // A generation start; the legacy sidecar (zero-downtime spec |
| + // §7.5: it runs its instance's serving artifact); or the |
| + // original template's run unit and the seed unit, which run |
| + // the build a cold wake recorded (settings spec §4.2). Every |
| + // one is validated before anything is laid out |
| + // (refuseStagedExe). |
| + if req.gen == 0 && !v.seed && req.role != RoleRun && req.role != RoleSidecar { |
| + return request{}, errors.New("exe is only for a run, seed or sidecar unit, or a generation start") |
| } |
| req.exe = StagedExe{Path: x, Version: q.Get("version"), Digest: q.Get("digest")} |
| } else { |
| @@ -417,8 +423,9 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { |
| } |
| ctx := r.Context() |
| unit := UnitName(req.role, req.host) |
| - // A sidecar naming its generation's staged build is validated before |
| - // anything is laid out, under the run unit's rule (validateStagedExe). |
| + // A sidecar naming its generation's staged build, or a run unit naming |
| + // the build its cold wake recorded, is validated before anything is |
| + // laid out, under the generation rule (validateStagedExe). |
| if s.refuseStagedExe(w, req, unit) { |
| return |
| } |
| @@ -437,8 +444,10 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { |
| // would put both generations under one tier's limit. Its restore is the |
| // swap owner's, so a populated slice is left alone, exactly as startGen |
| // does; a cold slice still gets the tier. A start without exe (flag |
| - // off) sets the tier as today. |
| - if req.exe.Path != "" && s.slicePopulated(req.host) { |
| + // off) sets the tier as today. The exemption is the sidecar's alone: a |
| + // run unit naming its recorded build starts a cold instance, and sets |
| + // its tier as a run start always has. |
| + if req.role == RoleSidecar && req.exe.Path != "" && s.slicePopulated(req.host) { |
| s.logf("instance-control: %s: slice populated, leaving its limit to the generation owner", req.host) |
| } else if err := s.setTier(ctx, req.host, req.tier); err != nil { |
| http.Error(w, "set-property: "+err.Error(), http.StatusBadGateway) |
| @@ -490,15 +499,23 @@ func parseShow(out string) map[string]string { |
| // fields RemainAfterExit retains), then stop, then wait for the cgroup |
| // to empty: a populated cgroup after the wait is still reported as a 502 |
| // naming the unit. |
| +// |
| +// A seed may name its executable (settings spec §4.2): the staged build |
| +// the wake recorded. Root validates it as it does a generation's (403 on |
| +// a refusal, before anything is laid out or started) and binds the host's |
| +// app mount to it. Without one the mount is the live link, as before. |
| func (s *Server) handleSeed(w http.ResponseWriter, r *http.Request) { |
| - req, err := parseRequest(r, verbShape{tier: true}) |
| + req, err := parseRequest(r, verbShape{tier: true, exe: true, seed: true}) |
| if err != nil { |
| http.Error(w, err.Error(), http.StatusBadRequest) |
| return |
| } |
| ctx := r.Context() |
| unit := UnitName(RoleSeed, req.host) |
| - if s.refuseStart(w, req.host, unit, func() error { return s.ensureHostExe(req.host) }) { |
| + if s.refuseStagedExe(w, req, unit) { |
| + return |
| + } |
| + if s.refuseStart(w, req.host, unit, func() error { return s.ensureHostExeAt(req.host, req.exe) }) { |
| return |
| } |
| if err := s.setTier(ctx, req.host, req.tier); err != nil { |
diff --git a/internal/instancectl/stagedexe.go b/internal/instancectl/stagedexe.go| index 39537c3e8..2d677dee8 100644 |
| --- a/internal/instancectl/stagedexe.go |
| +++ b/internal/instancectl/stagedexe.go |
| @@ -84,8 +84,9 @@ func (s *Server) ValidateStagedExe(host string, x StagedExe) error { |
| // refuseStagedExe answers 403 for a start whose staged executable root |
| // cannot vouch for, and reports whether it did. A request that names none |
| -// is not refused here. It is the one place both start paths (a generation's |
| -// and the legacy sidecar's) validate, before anything is laid out. |
| +// is not refused here. It is the one place every path that binds a staged |
| +// build validates (a generation's start; the original run, seed and |
| +// sidecar units), before anything is laid out. |
| func (s *Server) refuseStagedExe(w http.ResponseWriter, req request, unit string) bool { |
| if req.exe.Path == "" { |
| return false |
| @@ -110,8 +111,9 @@ func (s *Server) ensureGenExeAt(host string, n uint64, x StagedExe) error { |
| // ensureHostExeAt lays out the host's own executable mount (the one the |
| // original templates bind: <ExeRoot>/<id>/app). It points at a validated |
| -// staged build when x names one, which only a legacy sidecar's start does, |
| -// and at the live link otherwise (ensureHostExe). |
| +// staged build when x names one (a legacy sidecar's start, or a run or seed |
| +// unit running the build its cold wake recorded), and at the live link |
| +// otherwise (ensureHostExe). |
| func (s *Server) ensureHostExeAt(host string, x StagedExe) error { |
| if x.Path == "" { |
| return s.ensureHostExe(host) |
diff --git a/internal/instancectl/stagedexe_linux_test.go b/internal/instancectl/stagedexe_linux_test.go| index 80e0111ca..da863eb5f 100644 |
| --- a/internal/instancectl/stagedexe_linux_test.go |
| +++ b/internal/instancectl/stagedexe_linux_test.go |
| @@ -158,14 +158,128 @@ func TestSidecarStartMountsTheValidatedStagedExe(t *testing.T) { |
| if !ctlCalled(ctl, "set-property --runtime "+SliceName(genHost)) { |
| t.Fatal("a flag-off sidecar start no longer sets the tier") |
| } |
| - // A run-role start on the original template names no executable. |
| + // A run-role start on the original template may name an executable now |
| + // (settings spec §4.2: the cold wake runs the build it recorded), and |
| + // root still refuses one that is not the host's own validated build. |
| q := hostRole(genHost, RoleRun) |
| q.Set("memory_max", strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10)) |
| q.Set("exe", staged) |
| q.Set("version", "v2") |
| - q.Set("digest", digest) |
| - if code, _, _ := c.do(context.Background(), DeadlineControl, http.MethodPost, "/start", q, nil); code != http.StatusBadRequest { |
| - t.Errorf("a run-role start with exe answered %d, want 400", code) |
| + q.Set("digest", strings.Repeat("0", 64)) |
| + if code, _, _ := c.do(context.Background(), DeadlineControl, http.MethodPost, "/start", q, nil); code != http.StatusForbidden { |
| + t.Errorf("a run-role start with a wrong digest answered %d, want 403", code) |
| + } |
| +} |
| + |
| +// originalTemplateServer serves root's control channel over a version store |
| +// holding the host's own staged v2 and v3, with the live link on v2. It |
| +// returns the client, the systemctl fake, v2 as the exe root must accept, |
| +// the host's app mount and the opt dir. |
| +func originalTemplateServer(t *testing.T) (*Client, *fakeCtl, StagedExe, string, string) { |
| + t.Helper() |
| + opt, exeRoot := t.TempDir(), t.TempDir() |
| + staged := release.VersionPath(opt, release.OpsAccount, "v2", "blog") |
| + digest := writeStaged(t, staged, "build v2") |
| + writeStaged(t, release.VersionPath(opt, release.OpsAccount, "v3", "blog"), "build v3") |
| + repointLiveLink(t, opt, genHost, staged) |
| + ctl := &fakeCtl{answers: map[string]string{ |
| + "show -p InvocationID,Result,ExecMainStatus,ExecMainCode": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n", |
| + }} |
| + _, sock := serveOnSocket(t, ctl, func(s *Server) { |
| + s.DropInDir, s.ExeRoot, s.OptDir = t.TempDir(), exeRoot, opt |
| + s.HostApp = func(string) (string, string, error) { return release.OpsAccount, "blog", nil } |
| + }) |
| + app := filepath.Join(exeRoot, provision.InstanceID(genHost), "app") |
| + return NewClient(sock), ctl, StagedExe{Path: staged, Version: "v2", Digest: digest}, app, opt |
| +} |
| + |
| +// repointLiveLink is the agent's repoint of <opt>/live/<host>. |
| +func repointLiveLink(t *testing.T, opt, host, target string) { |
| + t.Helper() |
| + link := release.LiveLink(opt, host) |
| + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + os.Remove(link) |
| + if err := os.Symlink(target, link); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// Settings spec §4.2: the original run unit (generation 0) binds the |
| +// recorded staged file, validated under zdd-3's rule, and a repoint of the |
| +// live link after the start does not move it. A wrong digest is 403 |
| +// (ErrStagedExe) before systemd or the mount is touched. A start that |
| +// names no executable binds the live link, as today. |
| +func TestTheRunUnitBindsAValidatedStagedExe(t *testing.T) { |
| + c, ctl, x, app, opt := originalTemplateServer(t) |
| + if _, err := c.StartExe(context.Background(), genHost, RoleRun, cgroup.TierDefault.MemoryMax, x); err != nil { |
| + t.Fatalf("the host's own staged build was refused for its run unit: %v", err) |
| + } |
| + if !ctlCalled(ctl, "start "+UnitName(RoleRun, genHost)) { |
| + t.Fatalf("the run unit was not started:\n%s", ctl.joined()) |
| + } |
| + repointLiveLink(t, opt, genHost, release.VersionPath(opt, release.OpsAccount, "v3", "blog")) |
| + if got, err := os.Readlink(app); err != nil || got != x.Path { |
| + t.Fatalf("the run unit's app mount = %q (%v), want the recorded %q", got, err, x.Path) |
| + } |
| + |
| + ctl.reset() |
| + bad := x |
| + bad.Digest = strings.Repeat("0", 64) |
| + if _, err := c.StartExe(context.Background(), genHost, RoleRun, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) { |
| + t.Fatalf("a wrong digest = %v, want ErrStagedExe", err) |
| + } |
| + if ctl.joined() != "" { |
| + t.Fatalf("a refused start reached systemctl:\n%s", ctl.joined()) |
| + } |
| + if got, _ := os.Readlink(app); got != x.Path { |
| + t.Fatalf("a refused start moved the mount to %q", got) |
| + } |
| + |
| + if _, err := c.Start(context.Background(), genHost, RoleRun, cgroup.TierDefault.MemoryMax); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got, _ := os.Readlink(app); got != filepath.Join(LiveDir, genHost) { |
| + t.Fatalf("a run start with no executable bound %q, want the live link", got) |
| + } |
| +} |
| + |
| +// The seed unit binds the recorded staged file the same way, through |
| +// /seed, which names no role. A wrong digest is 403 before anything runs, |
| +// and a seed that names no executable binds the live link, as today. |
| +func TestTheSeedUnitBindsAValidatedStagedExe(t *testing.T) { |
| + c, ctl, x, app, opt := originalTemplateServer(t) |
| + res, err := c.SeedExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, x) |
| + if err != nil || res.Result != "success" { |
| + t.Fatalf("the host's own staged build was refused for its seed: %+v, %v", res, err) |
| + } |
| + if !ctlCalled(ctl, "start "+UnitName(RoleSeed, genHost)) { |
| + t.Fatalf("the seed unit was not started:\n%s", ctl.joined()) |
| + } |
| + repointLiveLink(t, opt, genHost, release.VersionPath(opt, release.OpsAccount, "v3", "blog")) |
| + if got, err := os.Readlink(app); err != nil || got != x.Path { |
| + t.Fatalf("the seed unit's app mount = %q (%v), want the recorded %q", got, err, x.Path) |
| + } |
| + |
| + ctl.reset() |
| + bad := x |
| + bad.Digest = strings.Repeat("0", 64) |
| + if _, err := c.SeedExe(context.Background(), genHost, cgroup.TierDefault.MemoryMax, bad); !errors.Is(err, ErrStagedExe) { |
| + t.Fatalf("a wrong digest = %v, want ErrStagedExe", err) |
| + } |
| + if ctl.joined() != "" { |
| + t.Fatalf("a refused seed reached systemctl:\n%s", ctl.joined()) |
| + } |
| + if got, _ := os.Readlink(app); got != x.Path { |
| + t.Fatalf("a refused seed moved the mount to %q", got) |
| + } |
| + |
| + if _, err := c.Seed(context.Background(), genHost, cgroup.TierDefault.MemoryMax); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got, _ := os.Readlink(app); got != filepath.Join(LiveDir, genHost) { |
| + t.Fatalf("a seed with no executable bound %q, want the live link", got) |
| } |
| } |
| @@ -226,7 +340,9 @@ func TestGenStartExeKeepsTheUnprovenAnswer(t *testing.T) { |
| } |
| } |
| -// exe, version and digest belong to the start verbs only. |
| +// exe, version and digest belong to the start and seed verbs only. On |
| +// those, an executable root cannot vouch for is refused (403) before |
| +// systemd is asked anything. |
| func TestStagedExeParametersRefusedWhereTheyDoNotBelong(t *testing.T) { |
| ctl := genCtl() |
| _, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() }) |
| @@ -234,14 +350,23 @@ func TestStagedExeParametersRefusedWhereTheyDoNotBelong(t *testing.T) { |
| "/stop?host=" + genHost + "&role=run&gen=7&exe=/x", |
| "/stop?host=" + genHost + "&role=sidecar&exe=/x", |
| "/reset?host=" + genHost + "&role=sidecar&version=v2", |
| - "/seed?host=" + genHost + "&memory_max=536870912&exe=/x", |
| - "/start?host=" + genHost + "&role=run&memory_max=536870912&exe=/x&version=v2&digest=d", |
| + "/seed?host=" + genHost + "&memory_max=536870912&version=v2", |
| "/start?host=" + genHost + "&role=run&memory_max=536870912&gen=7&version=v2", |
| } { |
| if resp := post(t, c, path); resp.StatusCode != 400 { |
| t.Errorf("%s: status %d, want 400", path, resp.StatusCode) |
| } |
| } |
| + // Settings spec §4.2: the original run unit and the seed unit may name |
| + // an executable; this one is not the host's own validated build. |
| + for _, path := range []string{ |
| + "/seed?host=" + genHost + "&memory_max=536870912&exe=/x", |
| + "/start?host=" + genHost + "&role=run&memory_max=536870912&exe=/x&version=v2&digest=d", |
| + } { |
| + if resp := post(t, c, path); resp.StatusCode != http.StatusForbidden { |
| + t.Errorf("%s: status %d, want 403", path, resp.StatusCode) |
| + } |
| + } |
| if ctl.joined() != "" { |
| t.Errorf("systemctl was called:\n%s", ctl.joined()) |
| } |