carlos / platform Public

activator: one guard for the two ensure entry points, one open and one sync for both layouts, the test seams in one struct, the dropped key fields named (final review minors 3, 4, 7)

ensureInstanceKeyAt is the replica and fenced/conflict guard that
ensureInstanceKeyFor and BackfillInstanceKey each repeated; openOwnEnv is
the one place the nested and flat opens meet, used by the append open
and by the claim's settle; syncInstanceEnv is the two syncs the claim's
settle re-implemented; ensureAdminTokenAt is the token's layout switch
the claim path alone needed. The before-open and replica-gate hooks are
one instanceKeyHooks struct, nil in production; the sync and write seams
stay as they are. claimServingLaunchSource says it drops the snapshot's
key digest and source until O6 threads them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Paul Campbell pushed by claude 1 week ago b628e4eea9d516b10b0dc6a1ca30bde1e36314d2 parent 1ea86df

7 files changed, +75 −59

internal/activator/admintoken.go +11 −0
internal/activator/claim_snapshot.go +1 −5
internal/activator/instancekey.go +52 −42
internal/activator/instancekey_test.go +3 −3
internal/activator/replicakey.go +2 −7
internal/activator/replicakey_race_test.go +2 −2
internal/host/claimproduction.go +4 −0

diff --git a/internal/activator/admintoken.go b/internal/activator/admintoken.go

index 5b79338e3..239632806 100644

--- a/internal/activator/admintoken.go

+++ b/internal/activator/admintoken.go

@@ -66,6 +66,17 @@ var adminTokenLogMu sync.Mutex

// deduplicated before it is said again.

const adminTokenLogRepeat = time.Hour

+// ensureAdminTokenAt is EnsureAdminToken on p's layout: through instancefs

+// for a nested row's tenant-writable directory, by path otherwise. The

+// legacy launch bodies each know their layout and call one or the other;

+// the claim path, which serves both, calls this.

+func ensureAdminTokenAt(p instancePaths, name string) error {

+ if p.layout == LayoutNested {

+ return EnsureAdminTokenBeneath(p.dataDir, name)

+ }

+ return EnsureAdminToken(filepath.Join(p.dataDir, name))

// logPerFile emits msg unless this (kind, path) pair has already said it

// within adminTokenLogRepeat. Keyed by kind as well as path so a file that

// is both un-tightenable AND holding a malformed token still reports both —

diff --git a/internal/activator/claim_snapshot.go b/internal/activator/claim_snapshot.go

index 149f6e494..d9b20c9a7 100644

--- a/internal/activator/claim_snapshot.go

+++ b/internal/activator/claim_snapshot.go

@@ -104,11 +104,7 @@ func (b *ExecBackend) EnsureClaimAdminToken(ctx context.Context, rt registry.Rou

// supplies is the one the launch delivers, and a second one in the

// own file would shadow it.

if vars["CARLOS_ADMIN_TOKEN"] == "" {

- if paths.layout == LayoutNested {

- if err := EnsureAdminTokenBeneath(paths.dataDir, rt.Host+".env"); err != nil {

- return err

- }

- } else if err := EnsureAdminToken(paths.envFile); err != nil {

+ if err := ensureAdminTokenAt(paths, rt.Host+".env"); err != nil {

return err

}

diff --git a/internal/activator/instancekey.go b/internal/activator/instancekey.go

index 79a6629eb..8c1126e21 100644

--- a/internal/activator/instancekey.go

+++ b/internal/activator/instancekey.go

@@ -145,12 +145,19 @@ func syncDir(dir string) error {

// adminTokenMu.

var instanceKeyWrite = func(f *os.File, b []byte) (int, error) { return f.Write(b) }

-// instanceKeyBeforeOpen, when set, runs under adminTokenMu after the read

-// and the decision and before the ensure opens the file to append or

-// settle: a test hook to change the file between the two, so the open's

-// own symlink refusal is pinned and not only the read's. nil outside

-// tests.

-var instanceKeyBeforeOpen func(path string)

+// instanceKeyHooks are the two test seams on the launch path's hot path,

+// both nil in production; the sync and write seams above are what the

+// durability tests need and stay on their own. beforeOpen runs under

+// adminTokenMu after the read and the decision and before the ensure

+// opens the file to append or settle, so a test can change the file

+// between the two and pin the open's own symlink refusal, not only the

+// read's. replicaGate runs at the start of every replica gate, after the

+// env was read, so a test can change the env files between the read and

+// the gate, as a concurrent config sync can.

+var instanceKeyHooks struct {

+ beforeOpen func(path string)

+ replicaGate func()

func mintInstanceKey() (string, error) {

raw := make([]byte, instanceKeyBytes)

@@ -241,14 +248,7 @@ func ensureInstanceKeyBeneathNoConfig(dir, name string) (InstanceKeyOutcome, err

// mint would use. Every caller names the host around the error ("instance

// key for <host>: ..."), so the refusal does not.

func (b *ExecBackend) ensureInstanceKeyFor(ctx context.Context, rt registry.Route, p instancePaths) (InstanceKeyOutcome, error) {

- if rt.Replica {

- return InstanceKeyReplica, nil

- }

- switch p.layout {

- case LayoutFenced, LayoutConflict:

- return InstanceKeyFailed, fmt.Errorf("not minted: %s", p.reason)

- }

- return ensureInstanceKey(ctx, p.dataDir, rt.Host+".env", p.layout == LayoutNested, true, b.configuredKeyFor(rt))

+ return b.ensureInstanceKeyAt(ctx, rt, p, true, "minted")

}

// BackfillInstanceKey is the startup backfill's step for one row: a key is

@@ -260,15 +260,34 @@ func (b *ExecBackend) ensureInstanceKeyFor(ctx context.Context, rt registry.Rout

// The agent's backfill names the host when it logs the error, so the

// refusal does not.

func (b *ExecBackend) BackfillInstanceKey(ctx context.Context, rt registry.Route) (InstanceKeyOutcome, error) {

+ return b.ensureInstanceKeyAt(ctx, rt, b.pathsFor(rt), false, "backfilled")

+// ensureInstanceKeyAt is the two entry points' one guard: a replica row is

+// never minted; a fenced or conflicting row is refused on p, the paths the

+// mint would use, with verb naming which caller refused ("not minted",

+// "not backfilled"); every other row is ensureInstanceKey on p's layout,

+// with the bucket reader for rt.

+func (b *ExecBackend) ensureInstanceKeyAt(ctx context.Context, rt registry.Route, p instancePaths, create bool, verb string) (InstanceKeyOutcome, error) {

if rt.Replica {

return InstanceKeyReplica, nil

}

- p := b.pathsFor(rt)

switch p.layout {

case LayoutFenced, LayoutConflict:

- return InstanceKeyFailed, fmt.Errorf("not backfilled: %s", p.reason)

+ return InstanceKeyFailed, fmt.Errorf("not %s: %s", verb, p.reason)

+ }

+ return ensureInstanceKey(ctx, p.dataDir, rt.Host+".env", p.layout == LayoutNested, create, b.configuredKeyFor(rt))

+// openOwnEnv opens an own env file, <host>.env under dir: through

+// instancefs when beneath (a nested row's tenant-writable directory), else

+// through openOwnEnvNoFollow. Both refuse a link or a FIFO in the open.

+// The one place the two layouts' opens meet.

+func openOwnEnv(dir, name string, beneath bool, flags int, perm os.FileMode) (*os.File, error) {

+ if beneath {

+ return instancefs.OpenBeneath(dir, name, flags, perm)

}

- return ensureInstanceKey(ctx, p.dataDir, rt.Host+".env", p.layout == LayoutNested, false, b.configuredKeyFor(rt))

+ return openOwnEnvNoFollow(filepath.Join(dir, name), flags, perm)

}

// openInstanceEnv opens an own env file for append. Only a create passes

@@ -279,10 +298,7 @@ func openInstanceEnv(dir, name string, beneath, create bool) (*os.File, error) {

if create {

flags, perm = flags|os.O_CREATE, 0o600

}

- if beneath {

- return instancefs.OpenBeneath(dir, name, flags, perm)

- }

- return openOwnEnvNoFollow(filepath.Join(dir, name), flags, perm)

+ return openOwnEnv(dir, name, beneath, flags, perm)

}

// errOwnEnvSymlink is a flat row's own env file found to be a symlink. The

@@ -328,12 +344,19 @@ func readOwnEnv(path string) ([]byte, error) {

}

// settleInstanceEnv is the durability step every successful ensure ends

-// with: 0600 on the descriptor, then the file, then its directory. Caller

-// holds adminTokenMu.

+// with: 0600 on the descriptor, then syncInstanceEnv. Caller holds

+// adminTokenMu.

func settleInstanceEnv(f *os.File, dir, path string) error {

if err := f.Chmod(0o600); err != nil {

return fmt.Errorf("chmod %s: %w", path, err)

}

+ return syncInstanceEnv(f, dir, path)

+// syncInstanceEnv fsyncs an own env file, then its directory: the two

+// syncs every durability step ends with, the claim's settle included.

+// Caller holds adminTokenMu.

+func syncInstanceEnv(f *os.File, dir, path string) error {

if err := instanceKeySync.file(f); err != nil {

return fmt.Errorf("sync %s: %w", path, err)

}

@@ -389,8 +412,8 @@ func ensureInstanceKey(ctx context.Context, dir, name string, beneath, create bo

// Present is not a no-op: a mint whose sync failed left these

// bytes behind, and this call is the retry that makes them

// durable before anything is started with them.

- if instanceKeyBeforeOpen != nil {

- instanceKeyBeforeOpen(path)

+ if instanceKeyHooks.beforeOpen != nil {

+ instanceKeyHooks.beforeOpen(path)

}

f, err := openInstanceEnv(dir, name, beneath, false)

if err != nil {

@@ -425,8 +448,8 @@ func ensureInstanceKey(ctx context.Context, dir, name string, beneath, create bo

if err != nil {

return InstanceKeyFailed, fmt.Errorf("mint instance key for %s: %w", path, err)

}

- if instanceKeyBeforeOpen != nil {

- instanceKeyBeforeOpen(path)

+ if instanceKeyHooks.beforeOpen != nil {

+ instanceKeyHooks.beforeOpen(path)

}

f, err := openInstanceEnv(dir, name, beneath, create)

if errors.Is(err, os.ErrNotExist) && !create {

@@ -492,16 +515,9 @@ var errInstanceEnvNeedsWrite = errors.New("activator: the instance env file need

// row is a replica: EnsureClaimAdminToken).

func (b *ExecBackend) settleClaimInstanceEnv(rt registry.Route) error {

p := b.pathsFor(rt)

- name := rt.Host + ".env"

adminTokenMu.Lock()

defer adminTokenMu.Unlock()

- var f *os.File

- var err error

- if p.layout == LayoutNested {

- f, err = instancefs.OpenBeneath(p.dataDir, name, os.O_RDONLY, 0)

- } else {

- f, err = openOwnEnvNoFollow(p.envFile, os.O_RDONLY, 0)

- }

+ f, err := openOwnEnv(p.dataDir, rt.Host+".env", p.layout == LayoutNested, os.O_RDONLY, 0)

if err != nil {

return fmt.Errorf("open %s: %w", p.envFile, err)

}

@@ -513,11 +529,5 @@ func (b *ExecBackend) settleClaimInstanceEnv(rt registry.Route) error {

if st.Mode().Perm() != 0o600 {

return errInstanceEnvNeedsWrite

}

- if err := instanceKeySync.file(f); err != nil {

- return fmt.Errorf("sync %s: %w", p.envFile, err)

- }

- if err := instanceKeySync.dir(p.dataDir); err != nil {

- return fmt.Errorf("sync %s: %w", p.dataDir, err)

- }

- return nil

+ return syncInstanceEnv(f, p.dataDir, p.envFile)

}

diff --git a/internal/activator/instancekey_test.go b/internal/activator/instancekey_test.go

index 18bafbfb8..5bc223dfe 100644

--- a/internal/activator/instancekey_test.go

+++ b/internal/activator/instancekey_test.go

@@ -466,17 +466,17 @@ func TestOpenOwnEnvNoFollowRefusesALinkForEveryFlagSet(t *testing.T) {

func setInstanceKeySeams(t *testing.T, write func(*os.File, []byte) (int, error), beforeOpen func(string)) {

t.Helper()

adminTokenMu.Lock()

- oldWrite, oldHook := instanceKeyWrite, instanceKeyBeforeOpen

+ oldWrite, oldHook := instanceKeyWrite, instanceKeyHooks.beforeOpen

if write != nil {

instanceKeyWrite = write

}

if beforeOpen != nil {

- instanceKeyBeforeOpen = beforeOpen

+ instanceKeyHooks.beforeOpen = beforeOpen

}

adminTokenMu.Unlock()

t.Cleanup(func() {

adminTokenMu.Lock()

- instanceKeyWrite, instanceKeyBeforeOpen = oldWrite, oldHook

+ instanceKeyWrite, instanceKeyHooks.beforeOpen = oldWrite, oldHook

adminTokenMu.Unlock()

})

}

diff --git a/internal/activator/replicakey.go b/internal/activator/replicakey.go

index 74545df80..d0d2b3e88 100644

--- a/internal/activator/replicakey.go

+++ b/internal/activator/replicakey.go

@@ -75,11 +75,6 @@ func sidecarKeyLayer(env map[string]string) instancekey.Layer {

return instancekey.LayerNone

}

-// replicaGateHook runs at the start of every replica gate, after the env

-// was read. A test seam: it lets a test change the env files between the

-// read and the gate, as a concurrent config sync can. Nil in production.

-var replicaGateHook func()

// replicaKeyGate is rulings 26, 27 and 29's check for a launch of rt with

// env (the composed env files), layer (where env's key came from, decided

// by the same read) and, when haveControl, a bound control socket. It reads

@@ -94,8 +89,8 @@ func (b *ExecBackend) replicaKeyGate(rt registry.Route, env map[string]string, l

if !rt.Replica {

return nil

}

- if replicaGateHook != nil {

- replicaGateHook()

+ if instanceKeyHooks.replicaGate != nil {

+ instanceKeyHooks.replicaGate()

}

refuse := func(why string, cause error) error {

if cause != nil {

diff --git a/internal/activator/replicakey_race_test.go b/internal/activator/replicakey_race_test.go

index b7dfd5bb4..49ccec0f8 100644

--- a/internal/activator/replicakey_race_test.go

+++ b/internal/activator/replicakey_race_test.go

@@ -35,13 +35,13 @@ func TestReplicaGateDecidesOnTheLaunchesOwnRead(t *testing.T) {

b.ReplicaHomeKeys = func(string) (string, bool) { return instancekey.Digest(keyA), true }

b.ConfiguredKey = func(context.Context, registry.Route) (string, error) { return "", nil } // the bucket: key removed

syncs := 0

- replicaGateHook = func() {

+ instanceKeyHooks.replicaGate = func() {

syncs++

if err := os.WriteFile(cfgPath, []byte("OTHER=1\n"), 0o600); err != nil {

t.Error(err)

}

- t.Cleanup(func() { replicaGateHook = nil })

+ t.Cleanup(func() { instanceKeyHooks.replicaGate = nil })

out := stageHelperApp(t, b, rt.Host, "v1")

ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)

defer cancel()

diff --git a/internal/host/claimproduction.go b/internal/host/claimproduction.go

index 0e3654a0b..e6d936701 100644

--- a/internal/host/claimproduction.go

+++ b/internal/host/claimproduction.go

@@ -450,6 +450,10 @@ func claimServingLaunchSource(backend *activator.ExecBackend, tokens claimAdminT

return ClaimServingLaunch{}, err

}

snapshot, err := backend.SelectClaimLaunch(ctx, rt)

+ // snapshot.InstanceKeySHA256 and InstanceKeySource are dropped here

+ // until O6 threads them to the launch's adoption verb (plan ruling

+ // 23; O6's obligation): a claim launch binds no control socket yet,

+ // so nothing can report against them.

return ClaimServingLaunch{Identity: snapshot.Identity, Executable: snapshot.Executable, ExecutableSHA256: snapshot.SHA256, Args: snapshot.Args, Env: snapshot.Env, Roles: snapshot.Roles, Version: snapshot.Version, Limit: snapshot.Limit}, err

}