amadan

carlos / platform Public

merge fix M3: a run gets the serving generation's environment

Main separates config delivery from activation: a config swap can be
queued, or fail, while the serving generation keeps the environment it
launched with. A run used to re-read the delivered files at its start, so
it could get a revision the instance was not serving with.

Main keeps no per-generation environment, so this adds the smallest
capture that fits its model: the generation handle a nested run-role
spawn returns (unitProc) keeps the file environment the generation
launched with, beside its other per-generation state. RunCmd takes it
from the current generation's handle with the generation itself, and
StartCmd composes the run's variables over it (cmdVars, unchanged).
prepare no longer reads the delivered files for the cmd role, and a run
whose generation kept no environment is refused before anything is laid
out or started. It ends as the existing start-failed result, so no new
copy is needed.

Tests: with a config revision delivered but not active, a run gets the
serving generation's environment and admin token, not the new files; a
run with no generation environment is refused before the control
service; RunCmd hands StartCmd the handle's environment, or none.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude c5de2ed5413169e57097dc401e2af7cb365cdc9e parent 088cb85
9 files changed, +213 −13
  • internal/activator/activator_test.go +13 −3
  • internal/activator/backend_exec.go +5 −0
  • internal/activator/cmd.go +8 −0
  • internal/activator/cmd_env_test.go +135 −0
  • internal/activator/cmd_test.go +1 −1
  • internal/activator/cmdunit.go +22 −5
  • internal/activator/cmdunit_test.go +2 −1
  • internal/activator/prepare.go +9 −3
  • internal/activator/unitproc.go +18 −0
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
index 66f9e4b92..ea4a35909 100644
--- a/internal/activator/activator_test.go
+++ b/internal/activator/activator_test.go
@@ -106,9 +106,12 @@ func (r *fakeRep) isClosed() bool {
}
type fakeProc struct {
- mu sync.Mutex
- stopped bool
- dead bool
+ // launchEnv is what LaunchEnv answers (merge ruling M3); nil answers
+ // an empty environment, as a generation launched with no files has.
+ launchEnv map[string]string
+ mu sync.Mutex
+ stopped bool
+ dead bool
// stopGate, when non-nil, blocks Stop() until closed — holds an entry in
// the stopping state so a test can poke at it.
stopGate chan struct{}
@@ -136,6 +139,13 @@ type fakeProc struct {
exitAt int
}
+func (p *fakeProc) LaunchEnv() map[string]string {
+ if p.launchEnv == nil {
+ return map[string]string{}
+ }
+ return p.launchEnv
+}
+
func (p *fakeProc) Stop(context.Context) error {
p.mu.Lock()
p.stopEnters++
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index ad4ea6726..d9a9ad98a 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"log"
+ "maps"
"net"
"net/http"
"os"
@@ -1728,6 +1729,10 @@ func (b *ExecBackend) spawnUnitFrom(ctx context.Context, rt registry.Route, sock
}
log.Printf("activator: %s: started %s memory.max=%d", rt.Host, unitName(instancectl.RoleRun, rt.Host, gen), started.MemoryMax)
up := b.newUnitProcFor(rt, instancectl.RoleRun, gen, p, inv, ctl, logs)
+ // The file environment this generation launched with, kept on its
+ // handle: a run against it composes its environment from this, never
+ // from the files config sync has delivered since (merge ruling M3).
+ up.launchEnv = maps.Clone(fileEnv)
if b.Cache != nil {
// A fresh process for this host is exactly the moment its cached
// content becomes suspect, same rule as startChild.
diff --git a/internal/activator/cmd.go b/internal/activator/cmd.go
index e52f8d6e0..303f27724 100644
--- a/internal/activator/cmd.go
+++ b/internal/activator/cmd.go
@@ -499,6 +499,14 @@ func (a *Service) RunCmd(ctx context.Context, rt registry.Route, spec CmdSpec) (
if a.gens && h.gen.StagedPath != "" {
spec.serving = &instancectl.StagedExe{Path: h.gen.StagedPath, Version: h.gen.Build, Digest: h.gen.Digest}
}
+ // And with the environment that generation launched with, from its
+ // handle: never the delivered files, which may hold a config revision
+ // it has not activated, a config swap queued or failed (merge ruling
+ // M3). A handle that kept none gets none, and StartCmd refuses.
+ spec.genEnv = nil
+ if lp, ok := e.proc.(launchEnvProc); ok {
+ spec.genEnv = lp.LaunchEnv()
+ }
a.mu.Unlock()
sctx, scancel := context.WithTimeout(ctx, cmdStartBound)
u, err := cb.StartCmd(sctx, rt, spec)
diff --git a/internal/activator/cmd_env_test.go b/internal/activator/cmd_env_test.go
new file mode 100644
index 000000000..ec1b2adac
--- /dev/null
+++ b/internal/activator/cmd_env_test.go
@@ -0,0 +1,135 @@
+package activator
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// envlessProc is a generation handle that kept no launch environment.
+type envlessProc struct{ Proc }
+
+// A config swap is queued: config sync has delivered a new revision, and
+// the serving generation still runs with the one it launched with. A run
+// gets the serving generation's environment, not the delivered files
+// (merge ruling M3).
+func TestARunGetsTheServingGenerationsEnvironment(t *testing.T) {
+ ctl := &fakeControl{inv: "inv1", cmdInv: "inv2"}
+ unit := &fakeUnit{}
+ b, rt := nestedBackend(t, ctl, unit)
+ b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
+ paths := b.pathsFor(rt)
+ if err := os.MkdirAll(filepath.Dir(paths.configEnv), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ deliver := func(rev string) {
+ t.Helper()
+ if err := os.WriteFile(paths.configEnv, []byte("APP_KEY="+rev+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ deliver("old")
+ var stopServing func()
+ ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) }
+ defer func() {
+ if stopServing != nil {
+ stopServing()
+ }
+ }()
+ unit.set("active", "inv1", true, true)
+ p, _, err := b.Start(context.Background(), rt)
+ if err != nil {
+ t.Fatalf("Start: %v", err)
+ }
+ lp, ok := p.(launchEnvProc)
+ if !ok || lp.LaunchEnv()["APP_KEY"] != "old" {
+ t.Fatalf("the generation's handle kept %v, want its launch environment", p)
+ }
+ deliver("new") // delivered, not activated: the config swap is queued
+
+ dir := filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), CmdRunDirName, "run_a")
+ spec := CmdSpec{RunID: "run_a", RunDir: dir, Stdout: &syncBuffer{}, Stderr: &syncBuffer{}, genEnv: lp.LaunchEnv()}
+ u, err := b.StartCmd(context.Background(), rt, spec)
+ if err != nil {
+ t.Fatalf("StartCmd: %v", err)
+ }
+ u.(*cmdUnit).out.Close()
+ u.(*cmdUnit).errs.Close()
+ raw, err := os.ReadFile(filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), "launch.cmd.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var l instancectl.Launch
+ if err := json.Unmarshal(raw, &l); err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Contains(l.Env, "APP_KEY=old") || slices.Contains(l.Env, "APP_KEY=new") {
+ t.Fatalf("the run's env %q, want the serving generation's APP_KEY=old", l.Env)
+ }
+ token := lp.LaunchEnv()["CARLOS_ADMIN_TOKEN"]
+ if token == "" || !slices.Contains(l.Env, "CARLOS_ADMIN_TOKEN="+token) {
+ t.Fatal("the run did not get the serving generation's admin token")
+ }
+ for _, kv := range l.Env {
+ if strings.HasPrefix(kv, "CARLOS_CONTROL_SOCKET=") || strings.HasPrefix(kv, "CARLOS_PURGE=") {
+ t.Errorf("a run's env carries %s", kv)
+ }
+ }
+
+ // A run with no generation environment is refused before anything is
+ // laid out or started: the delivered files are never its fallback.
+ before := ctl.joined()
+ if _, err := b.StartCmd(context.Background(), rt, CmdSpec{RunID: "run_b", RunDir: filepath.Join(filepath.Dir(dir), "run_b")}); !errors.Is(err, errNoGenEnv) {
+ t.Fatalf("StartCmd without a generation environment = %v, want errNoGenEnv", err)
+ }
+ if ctl.joined() != before {
+ t.Fatalf("the refused run reached the control service:\n%s", ctl.joined())
+ }
+}
+
+// RunCmd hands StartCmd the environment the current generation's handle
+// kept, and nothing for a handle that kept none, which StartCmd refuses
+// (merge ruling M3).
+func TestRunCmdHandsOverTheGenerationsLaunchEnvironment(t *testing.T) {
+ a, cb, _ := testCmdActivator(t, Config{})
+ rt := testRoute("c.example.com")
+ var got []map[string]string
+ cb.startCmd = func(_ context.Context, _ registry.Route, spec CmdSpec) (CmdUnit, error) {
+ got = append(got, spec.genEnv)
+ if spec.genEnv == nil {
+ return nil, errNoGenEnv
+ }
+ return cb.unit, nil
+ }
+ arrangeAwake(t, a, rt)
+ a.mu.Lock()
+ e := a.entries[rt.Host]
+ e.proc.(*fakeProc).launchEnv = map[string]string{"APP_KEY": "serving"}
+ a.mu.Unlock()
+ h, err := a.RunCmd(context.Background(), rt, CmdSpec{RunID: "run_a"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := h.Stop(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ h.Release()
+ a.mu.Lock()
+ e.proc = envlessProc{e.proc}
+ a.mu.Unlock()
+ if _, err := a.RunCmd(context.Background(), rt, CmdSpec{RunID: "run_b"}); !errors.Is(err, errNoGenEnv) {
+ t.Fatalf("a run on a handle with no launch environment = %v, want errNoGenEnv", err)
+ }
+ if len(got) != 2 || got[0]["APP_KEY"] != "serving" || got[1] != nil {
+ t.Fatalf("StartCmd got %v, want the handle's environment, then none", got)
+ }
+}
diff --git a/internal/activator/cmd_test.go b/internal/activator/cmd_test.go
index 384e79f10..2ac9e5373 100644
--- a/internal/activator/cmd_test.go
+++ b/internal/activator/cmd_test.go
@@ -683,7 +683,7 @@ func TestStartCmdNamesTheServingBuild(t *testing.T) {
b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
dir := filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), CmdRunDirName, "run_a")
x := instancectl.StagedExe{Path: "/opt/carlos/versions/v2/blog", Version: "v2", Digest: strings.Repeat("a", 64)}
- spec := CmdSpec{RunID: "run_a", RunDir: dir, Stdout: &syncBuffer{}, Stderr: &syncBuffer{}, serving: &x}
+ spec := CmdSpec{RunID: "run_a", RunDir: dir, Stdout: &syncBuffer{}, Stderr: &syncBuffer{}, serving: &x, genEnv: map[string]string{}}
if u, err := b.StartCmd(context.Background(), rt, spec); !errors.Is(err, errNoCmdExe) || u != nil {
t.Fatalf("StartCmd without CmdExe = %v, %v; want errNoCmdExe and no unit", u, err)
}
diff --git a/internal/activator/cmdunit.go b/internal/activator/cmdunit.go
index 91bc76cf1..a116402cf 100644
--- a/internal/activator/cmdunit.go
+++ b/internal/activator/cmdunit.go
@@ -52,8 +52,20 @@ type CmdSpec struct {
// (zero-downtime spec §7.5: what runs beside an instance is its serving
// artifact). Nil runs the live link. Callers cannot set it.
serving *instancectl.StagedExe
+ // genEnv is the file environment the run's generation launched with,
+ // from its handle (unitProc.LaunchEnv), set by RunCmd with serving
+ // (merge ruling M3). The run's variables are composed over it, never
+ // over the delivered files, which may hold a config revision that
+ // generation never activated. StartCmd refuses a run without one.
+ // Callers cannot set it.
+ genEnv map[string]string
}
+// errNoGenEnv: the run's generation kept no launch environment, so the run
+// would have to read the delivered files, which may differ from what the
+// instance is serving with. It is refused instead (merge ruling M3).
+var errNoGenEnv = errors.New("activator: the serving generation kept no launch environment for a run")
+
// cmdExeStarter is the cmd start with the executable named (Client.CmdExe).
// A separate interface, like GenControl, so a Control without it still
// satisfies Control; StartCmd asks for it only when a run names a build.
@@ -90,7 +102,8 @@ type CmdBackend interface {
var _ CmdBackend = (*ExecBackend)(nil)
-// cmdVars is a run's environment over fileEnv (run spec §4, codex 7): the
+// cmdVars is a run's environment over fileEnv, the file environment its
+// generation launched with (run spec §4, codex 7; merge ruling M3): the
// control-channel variables stripped as seedVars strips them, CARLOS_REGION
// set or removed from the backend's region as agentOwnedVars does,
// CARLOS_HOST from the row, and the three run variables. fileEnv carries
@@ -132,8 +145,9 @@ func (b *ExecBackend) cmdLayout(rt registry.Route) (Layout, string) {
}
// StartCmd starts one run's cmd unit (run spec §5.4): the layout check, the
-// nested gate, the live link, the instance's directories and environment,
-// the two stream listeners, launch.cmd.json, then POST /cmd. It returns
+// nested gate, the live link, the instance's directories, the environment
+// its generation launched with (spec.genEnv, merge ruling M3), the two
+// stream listeners, launch.cmd.json, then POST /cmd. It returns
// once the unit has an invocation. The environment is never logged. A
// failure after the POST was sent returns a unit with the error: the unit
// may be running, and the caller must stop and settle it.
@@ -162,7 +176,10 @@ func (b *ExecBackend) StartCmd(ctx context.Context, rt registry.Route, spec CmdS
} else if err := b.ensureLiveLink(ctx, rt.Host); err != nil {
return nil, err
}
- p, _, _, fileEnv, err := b.prepare(rt, instancectl.RoleCmd)
+ if spec.genEnv == nil {
+ return nil, fmt.Errorf("%s: %w", rt.Host, errNoGenEnv)
+ }
+ p, _, _, _, err := b.prepare(rt, instancectl.RoleCmd)
if err != nil {
return nil, err
}
@@ -181,7 +198,7 @@ func (b *ExecBackend) StartCmd(ctx context.Context, rt registry.Route, spec CmdS
return nil, err
}
argv := append([]string{b.liveLink(rt.Host)}, spec.Args...)
- env := childEnv(b.cmdVars(rt, fileEnv, p.dbPath, spec.RunID, spec.RunDir))
+ env := childEnv(b.cmdVars(rt, spec.genEnv, p.dbPath, spec.RunID, spec.RunDir))
if err := b.writeLaunch(p, instancectl.RoleCmd, rt.Host, argv, env); err != nil {
return fail(err)
}
diff --git a/internal/activator/cmdunit_test.go b/internal/activator/cmdunit_test.go
index a8a4a585f..ce15edbcd 100644
--- a/internal/activator/cmdunit_test.go
+++ b/internal/activator/cmdunit_test.go
@@ -66,7 +66,8 @@ func startTestCmd(t *testing.T) (*ExecBackend, registry.Route, *fakeControl, *fa
b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }
out, errs := &syncBuffer{}, &syncBuffer{}
dir := filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), CmdRunDirName, "run_a")
- u, err := b.StartCmd(context.Background(), rt, CmdSpec{RunID: "run_a", Args: []string{"import", "--dry-run"}, RunDir: dir, Stdout: out, Stderr: errs})
+ u, err := b.StartCmd(context.Background(), rt, CmdSpec{RunID: "run_a", Args: []string{"import", "--dry-run"}, RunDir: dir, Stdout: out, Stderr: errs,
+ genEnv: map[string]string{"CARLOS_ADMIN_TOKEN": "tok"}})
if err != nil {
t.Fatalf("StartCmd: %v", err)
}
diff --git a/internal/activator/prepare.go b/internal/activator/prepare.go
index 0d3c6d550..0c33038a1 100644
--- a/internal/activator/prepare.go
+++ b/internal/activator/prepare.go
@@ -27,7 +27,7 @@ import (
// prepare is everything a nested role needs before its unit starts
// (design §6.1): the two directories, the admin token and merged env (for
-// run and seed), the control listener (run only), the role's log listener,
+// run and seed; a cmd's env is its generation's, and prepare returns none), the control listener (run only), the role's log listener,
// and stale records cleared. It returns the listeners so the caller can
// hand them to the proc or close them on a failed start.
func (b *ExecBackend) prepare(rt registry.Route, role instancectl.Role) (instancePaths, *edge.ControlListener, *logListener, map[string]string, error) {
@@ -53,7 +53,8 @@ func (b *ExecBackend) prepareAt(rt registry.Route, role instancectl.Role, sock s
return p, nil, nil, nil, err
}
var env map[string]string
- if role == instancectl.RoleRun || role == instancectl.RoleSeed || role == instancectl.RoleCmd {
+ switch role {
+ case instancectl.RoleRun, instancectl.RoleSeed:
// p.dataDir is the instance directory here, tenant-writable ground
// (design §3.2), so the token file is minted and read through
// instancefs rather than a plain path-based open.
@@ -65,7 +66,12 @@ func (b *ExecBackend) prepareAt(rt registry.Route, role instancectl.Role, sock s
if err != nil {
return p, nil, nil, nil, err
}
- } else {
+ case instancectl.RoleCmd:
+ // A run's environment is the one its generation launched with
+ // (merge ruling M3), which StartCmd has. The delivered files are
+ // not read: config sync may have written a revision the serving
+ // generation has not activated.
+ default:
var err error
env, err = readEnvFileBeneath(p.dataDir, filepath.Base(p.sidecarEnv))
if err != nil {
diff --git a/internal/activator/unitproc.go b/internal/activator/unitproc.go
index e8b6de72b..984ec0f39 100644
--- a/internal/activator/unitproc.go
+++ b/internal/activator/unitproc.go
@@ -61,8 +61,26 @@ type unitProc struct {
// control service removed the unit's drop-in and exe layout with it,
// so finaliseGen need not ask again.
genFilesGone atomic.Bool
+ // launchEnv is the file environment (the config bundle with the
+ // instance's own file on top) a run role's unit launched with, before
+ // the agent-owned variables. It is written once, before the handle
+ // leaves spawnUnitFrom, and never again. A run against this generation
+ // composes its environment from it (LaunchEnv; merge ruling M3). nil
+ // for every other role.
+ launchEnv map[string]string
}
+// launchEnvProc is a generation handle that kept the file environment its
+// generation launched with (merge ruling M3).
+type launchEnvProc interface {
+ LaunchEnv() map[string]string
+}
+
+// LaunchEnv is the file environment this generation launched with, or nil
+// when it kept none. The map is the handle's own: callers must not write
+// it.
+func (p *unitProc) LaunchEnv() map[string]string { return p.launchEnv }
+
func (b *ExecBackend) newUnitProc(rt registry.Route, role instancectl.Role, p instancePaths, inv string, ctl *edge.ControlListener, logs *logListener) *unitProc {
return b.newUnitProcFor(rt, role, 0, p, inv, ctl, logs)
}