carlos / platform Public

config guard: drop the stale-file fallback, make the no-sealed-config bypass loud (#213)

A fresh acquisition or respawn starts only if the whole merged config renders
now; a leftover config.env proves nothing about its values. The
CARLOS_NO_SEALED_CONFIG bypass is logged at startup and per host and shown in
the private status block as metrics.config_guard. Tests: leftover files refuse,
a configured-store error with the opt-out refuses, build convergence refusal.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 699c6c732d710267e4314032b2cd9a9ea3c138bd parent d0498ac

9 files changed, +303 −215

internal/activator/config_gate_test.go +35 −0
internal/appconfig/canprovide.go +16 −77
internal/appconfig/canprovide_test.go +19 −101
internal/host/agent.go +4 −1
internal/host/configguard.go +66 −10
internal/host/configguard_test.go +126 −24
internal/status/contract_test.go +2 −1
internal/status/snapshot.go +11 −1
internal/status/status_test.go +24 −0

CI passed — run details

diff --git a/internal/activator/config_gate_test.go b/internal/activator/config_gate_test.go

index 0e7832ff5..5c4aa5c17 100644

--- a/internal/activator/config_gate_test.go

+++ b/internal/activator/config_gate_test.go

@@ -132,6 +132,41 @@ func TestWorkerRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) {

}

+// Build convergence shares the respawn helper but is its own trigger: a ship

+// repoints live/<host>, the drift outlasts ConvergeGrace, and the always-on

+// worker would be cycled onto the new build. With the gate refusing, it stands

+// down instead of starting the new build without its config.

+func TestWorkerConvergeRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) {

+ a, b, clk, ver := shippingActivator(t, time.Hour)

+ host := "bot.x.internal"

+ if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil {

+ t.Fatalf("WakeWorker: %v", err)

+ }

+ first := b.lastWorker()

+ b.mu.Lock()

+ b.admitErr = errors.New("cannot open sealed config")

+ b.mu.Unlock()

+ ver.Store("v2")

+ a.sweepOnce() // first sighting of the drift

+ clk.advance(ConvergeGrace + time.Minute)

+ a.sweepOnce() // the converge respawn, refused

+ waitFor(t, "the refused convergence to stand down", func() bool { return a.entryCount() == 0 })

+ if n := b.workerStarts(); n != 1 {

+ t.Fatalf("workerStarts = %d: a refused convergence still started the new build", n)

+ }

+ if !first.isStopped() {

+ t.Error("the old child was left running")

+ }

+ if n := b.releaseCount(); n != 1 {

+ t.Fatalf("lease releases = %d, want 1", n)

+ }

+ if !b.lastRep().isClosed() {

+ t.Error("replication was left open")

+ }

// And a gate that allows it changes nothing: the respawn runs in place and

// keeps its lease, as before.

func TestWorkerRespawnProceedsWhenTheConfigGateAllows(t *testing.T) {

diff --git a/internal/appconfig/canprovide.go b/internal/appconfig/canprovide.go

index e1e2f137d..3063c534c 100644

--- a/internal/appconfig/canprovide.go

+++ b/internal/appconfig/canprovide.go

@@ -1,16 +1,10 @@

package appconfig

import (

- "bytes"

"context"

"crypto/ecdh"

"fmt"

- "os"

- "path/filepath"

- "sort"

- "github.com/carlosframework/platform/internal/instancefs"

- "github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

)

@@ -71,33 +65,6 @@ func (r *ConfigRefusal) Error() string {

func (r *ConfigRefusal) Unwrap() error { return r.Err }

-// ReadDeliveredConfig returns the sync-rendered <host>.config.env this box

-// holds for rt, or nil if there is none or it cannot be read. It reads the

-// way SyncExec does (a nested row's file sits on tenant-owned ground, so it

-// goes through instancefs), so a planted FIFO cannot hang an admission.

-func ReadDeliveredConfig(dataDir string, rt registry.Route) []byte {

- path := ConfigEnvPath(dataDir, rt)

- var (

- b []byte

- err error

- )

- if nestedRoute(dataDir, rt) {

- b, err = instancefs.ReadBeneathLimit(filepath.Dir(path), filepath.Base(path), 1<<20)

- } else {

- var fi os.FileInfo

- if fi, err = os.Lstat(path); err == nil && !fi.Mode().IsRegular() {

- return nil

- }

- if err == nil {

- b, err = os.ReadFile(path)

- }

- if err != nil {

- return nil

- }

- return b

// CanProvideConfig reports nil when this box can provide the config of the

// instance at host, and a *ConfigRefusal when it cannot or cannot tell.

@@ -119,17 +86,15 @@ func ReadDeliveredConfig(dataDir string, rt registry.Route) []byte {

// file as optional, so a half-openable manifest on a box with no rendered

// file starts the instance with none of its sealed config.

-// The one exception is evidence that this box already delivered a usable

-// file. If rendering fails, but (a) the key opens at least one envelope, so

-// the key is this deployment's, and (b) delivered() returns a rendered

-// <host>.config.env that carries a NAME= line for every secret var in the

-// merged view, then the sync pass is stalled on a stale or corrupt value

-// while the instance has a file from an earlier good render, and refusing

-// would turn that sync fault into an outage of a running tenant. A secret

-// var with no line in the file (new since the last render) is no evidence,

-// and neither is a missing key, a wrong one, or no file. delivered may be

-// nil: no evidence available.

-func CanProvideConfig(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, ks KeyState, acct, app, envName, host string, delivered func() []byte) error {

+// There is no last-known-good fallback, deliberately. A rendered

+// <host>.config.env left on disk from an earlier pass proves nothing about

+// its VALUES: names alone pass for "A=old" or "A=" or credentials rendered for

+// another environment, and sync leaves such a file untouched when it cannot

+// render. A fresh acquisition or respawn starts only if the whole merged

+// config renders now. The fix for a mixed-key or partly corrupt manifest is

+// to give the box the key or re-seal the config. (Continuing a process that is

+// already running is a different policy and not this function's.)

+func CanProvideConfig(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, ks KeyState, acct, app, envName, host string) error {

var (

m *Manifest

err error

@@ -144,14 +109,13 @@ func CanProvideConfig(ctx context.Context, s release.Store, priv *ecdh.PrivateKe

if err != nil {

return &ConfigRefusal{Reason: ReasonStoreError, Err: err}

}

- var names, sealed []string

- for k, v := range m.Vars {

+ sealed := 0

+ for _, v := range m.Vars {

if v.Kind == "secret" {

- names = append(names, k)

- sealed = append(sealed, v.Envelope)

+ sealed++

}

- if len(sealed) == 0 {

+ if sealed == 0 {

return nil

}

if priv == nil || ks != KeyOK {

@@ -161,33 +125,8 @@ func CanProvideConfig(ctx context.Context, s release.Store, priv *ecdh.PrivateKe

return &ConfigRefusal{Reason: ReasonNoKey}

}

bind := Binding{Account: acct, App: app}

- _, rerr := RenderEnv(m, priv, bind)

- if rerr == nil {

- return nil

- }

- opened := false

- for _, env := range sealed {

- if _, err := Open(priv, bind, env); err == nil {

- opened = true

- break

- }

- if opened && delivered != nil && deliveredCovers(delivered(), names) {

- return nil

- }

- return &ConfigRefusal{Reason: ReasonCannotOpen, Err: fmt.Errorf("this box's key cannot render the config and there is no earlier rendered file to fall back on: %w", rerr)}

-// deliveredCovers reports whether file has a NAME= line for every name.

-func deliveredCovers(file []byte, names []string) bool {

- if len(file) == 0 {

- return false

- }

- sort.Strings(names)

- for _, n := range names {

- if !bytes.HasPrefix(file, []byte(n+"=")) && !bytes.Contains(file, []byte("\n"+n+"=")) {

- return false

- }

+ if _, err := RenderEnv(m, priv, bind); err != nil {

+ return &ConfigRefusal{Reason: ReasonCannotOpen, Err: fmt.Errorf("this box's key cannot render the config: %w", err)}

}

- return true

+ return nil

}

diff --git a/internal/appconfig/canprovide_test.go b/internal/appconfig/canprovide_test.go

index 01c076378..8ebc1832c 100644

--- a/internal/appconfig/canprovide_test.go

+++ b/internal/appconfig/canprovide_test.go

@@ -5,11 +5,9 @@ import (

"crypto/ecdh"

"encoding/json"

"errors"

- "os"

"strings"

"testing"

- "github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

)

@@ -72,17 +70,17 @@ func TestCanProvideConfigNothingSealedNeedsNothing(t *testing.T) {

e := newCanProvideEnv(t)

ctx := context.Background()

// No manifest anywhere: a dev deployment. No key either.

- if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com"); err != nil {

t.Fatalf("no manifest, no key: %v", err)

}

// A manifest with plain vars only has no sealed config.

e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}})

- if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com"); err != nil {

t.Fatalf("plain-only manifest, no key: %v", err)

}

// A manifest that was fully unset exists but is empty.

e.put(Key("bab", "other"), Manifest{App: "other", Vars: map[string]Var{}})

- if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "other", "", "a.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "other", "", "a.example.com"); err != nil {

t.Fatalf("empty manifest, no key: %v", err)

}

@@ -92,16 +90,16 @@ func TestCanProvideConfigSealedNeedsAKeyThatOpensIt(t *testing.T) {

ctx := context.Background()

e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"DB": e.secret("pw")}})

- if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com"); err != nil {

t.Fatalf("right key: %v", err)

}

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com", nil)); got != ReasonNoKey {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com")); got != ReasonNoKey {

t.Errorf("absent key reason = %q, want %q", got, ReasonNoKey)

}

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyUnparseable, "bab", "shop", "", "a.example.com", nil)); got != ReasonKeyUnparseable {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyUnparseable, "bab", "shop", "", "a.example.com")); got != ReasonKeyUnparseable {

t.Errorf("unparseable key reason = %q, want %q", got, ReasonKeyUnparseable)

}

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.other, KeyOK, "bab", "shop", "", "a.example.com", nil)); got != ReasonCannotOpen {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.other, KeyOK, "bab", "shop", "", "a.example.com")); got != ReasonCannotOpen {

t.Errorf("wrong key reason = %q, want %q", got, ReasonCannotOpen)

}

@@ -116,18 +114,18 @@ func TestCanProvideConfigLooksAtEveryLayerTheBoxRenders(t *testing.T) {

e.put(InstanceConfigKey("bab", "shop", "cust.example.com"), Manifest{App: "shop", Vars: map[string]Var{"T": e.secret("y")}})

// Unbound route, no overlay for this host: nothing sealed.

- if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "plain.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "plain.example.com"); err != nil {

t.Errorf("unbound plain host: %v", err)

}

// Bound to the environment: sealed.

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "staging", "plain.example.com", nil)); got != ReasonNoKey {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "staging", "plain.example.com")); got != ReasonNoKey {

t.Errorf("environment secret reason = %q", got)

}

// The overlay for cust.example.com only.

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "cust.example.com", nil)); got != ReasonNoKey {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "cust.example.com")); got != ReasonNoKey {

t.Errorf("overlay secret reason = %q", got)

}

- if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "cust.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "cust.example.com"); err != nil {

t.Errorf("overlay with the key: %v", err)

}

@@ -162,93 +160,13 @@ func TestCanProvideConfigRefusesPartlyOpenableConfigOnFreshAdmission(t *testing.

}

vars := mk(e, foreign)

e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: vars})

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com", nil)); got != ReasonCannotOpen {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com")); got != ReasonCannotOpen {

t.Errorf("reason = %q, want cannot_open", got)

}

- // A file that is not there, or empty, is no evidence either.

- none := func() []byte { return nil }

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com", none)); got != ReasonCannotOpen {

- t.Errorf("empty evidence reason = %q", got)

- }

})

}

-// The safe fallback: the key is this deployment's (one envelope opens), the

-// sync pass is stalled on a stale value, and this box already delivered a

-// rendered file that carries every secret var. Refusing there would turn a

-// sync fault into an outage of a running tenant.

-func TestCanProvideConfigFallsBackOnlyToEvidenceOfADeliveredFile(t *testing.T) {

- e := newCanProvideEnv(t)

- ctx := context.Background()

- foreign, err := SealUnbound(e.other.PublicKey(), []byte("z"))

- if err != nil {

- t.Fatal(err)

- }

- e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{

- "A": {Kind: "secret", Envelope: foreign},

- "C": e.secret("ok"),

- "P": {Kind: "env", Value: "1"},

- }})

- file := func(s string) func() []byte { return func() []byte { return []byte(s) } }

- ask := func(f func() []byte) error {

- return CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com", f)

- }

- if err := ask(file("# header\nA=old\nC=ok\nP=1\n")); err != nil {

- t.Errorf("file with every secret var: %v", err)

- }

- if err := ask(file("A=old\nP=1\n")); err == nil {

- t.Error("a file missing secret var C counted as evidence")

- }

- if err := ask(file("XA=old\nXC=ok\n")); err == nil {

- t.Error("a suffix match counted as evidence")

- }

- if err := ask(file("")); err == nil {

- t.Error("an empty file counted as evidence")

- }

- if err := ask(nil); err == nil {

- t.Error("no evidence source counted as evidence")

- }

- // The key must be shown to be this deployment's: with the wrong key

- // nothing opens, and a leftover file is not enough.

- third, err := ecdh.P256().GenerateKey(randReader)

- if err != nil {

- t.Fatal(err)

- }

- good := file("A=old\nC=ok\n")

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, third, KeyOK, "bab", "shop", "", "a.example.com", good)); got != ReasonCannotOpen {

- t.Errorf("wrong key with a leftover file: reason %q", got)

- }

- // And no key at all never falls back, file or not.

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com", good)); got != ReasonNoKey {

- t.Errorf("no key with a leftover file: reason %q", got)

- }

-func TestReadDeliveredConfigReadsTheSyncOwnedFile(t *testing.T) {

- dir := t.TempDir()

- rt := registry.Route{Host: "a.example.com", Kind: "instance", App: "shop", Socket: "/run/a.sock"}

- if got := ReadDeliveredConfig(dir, rt); got != nil {

- t.Errorf("no file: got %q", got)

- }

- if err := os.WriteFile(ConfigEnvPath(dir, rt), []byte("A=1\n"), 0o600); err != nil {

- t.Fatal(err)

- }

- if got := string(ReadDeliveredConfig(dir, rt)); got != "A=1\n" {

- t.Errorf("flat file: got %q", got)

- }

- // A non-regular file at the path is no evidence (and must not block).

- rt2 := registry.Route{Host: "b.example.com", Kind: "instance", App: "shop", Socket: "/run/b.sock"}

- if err := os.Mkdir(ConfigEnvPath(dir, rt2), 0o700); err != nil {

- t.Fatal(err)

- }

- if got := ReadDeliveredConfig(dir, rt2); got != nil {

- t.Errorf("directory at the path: got %q", got)

- }

// A v2 envelope is bound to its (account, app): the guard opens it under the

// route's own binding, as the sync pass does.

func TestCanProvideConfigOpensBoundEnvelopesUnderTheRoutesBinding(t *testing.T) {

@@ -259,12 +177,12 @@ func TestCanProvideConfigOpensBoundEnvelopesUnderTheRoutesBinding(t *testing.T)

t.Fatal(err)

}

e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"DB": {Kind: "secret", Envelope: env}}})

- if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com"); err != nil {

t.Fatalf("bound envelope, own app: %v", err)

}

// The same bytes copied under another app do not open there.

e.put(Key("bab", "thief"), Manifest{App: "thief", Vars: map[string]Var{"DB": {Kind: "secret", Envelope: env}}})

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "thief", "", "a.example.com", nil)); got != ReasonCannotOpen {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "thief", "", "a.example.com")); got != ReasonCannotOpen {

t.Errorf("copied bound envelope reason = %q", got)

}

@@ -300,7 +218,7 @@ func TestCanProvideConfigFailsClosedWhenItCannotRead(t *testing.T) {

priv *ecdh.PrivateKey

st KeyState

}{{e.priv, KeyOK}, {nil, KeyAbsent}} {

- err := CanProvideConfig(ctx, fs, k.priv, k.st, "bab", "shop", "staging", "a.example.com", nil)

+ err := CanProvideConfig(ctx, fs, k.priv, k.st, "bab", "shop", "staging", "a.example.com")

if got := reasonOf(t, err); got != ReasonStoreError {

t.Errorf("reason = %q, want %q", got, ReasonStoreError)

}

@@ -320,11 +238,11 @@ func TestCanProvideConfigFailsClosedOnUnreadableConfig(t *testing.T) {

if _, err := e.s.Put(ctx, Key("bab", "garbled"), []byte("{not json"), ""); err != nil {

t.Fatal(err)

}

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "garbled", "", "a.example.com", nil)); got != ReasonStoreError {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "garbled", "", "a.example.com")); got != ReasonStoreError {

t.Errorf("garbled manifest reason = %q", got)

}

e.put(Key("bab", "bound"), Manifest{App: "bound", Vars: map[string]Var{}})

- if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "bound", "gone", "a.example.com", nil)); got != ReasonStoreError {

+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "bound", "gone", "a.example.com")); got != ReasonStoreError {

t.Errorf("missing bound environment reason = %q", got)

}

@@ -335,7 +253,7 @@ func TestCanProvideConfigHostWithNoPossibleOverlay(t *testing.T) {

e := newCanProvideEnv(t)

ctx := context.Background()

e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}})

- if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "Mixed.Case.example.com", nil); err != nil {

+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "Mixed.Case.example.com"); err != nil {

t.Fatalf("host without a possible overlay: %v", err)

}

@@ -346,7 +264,7 @@ func TestConfigRefusalReasonIsAClosedWord(t *testing.T) {

ctx := context.Background()

v := e.secret("hunter2")

e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"DB": v}})

- err := CanProvideConfig(ctx, e.s, e.other, KeyOK, "bab", "shop", "", "a.example.com", nil)

+ err := CanProvideConfig(ctx, e.s, e.other, KeyOK, "bab", "shop", "", "a.example.com")

var r *ConfigRefusal

if !errors.As(err, &r) {

t.Fatal(err)

diff --git a/internal/host/agent.go b/internal/host/agent.go

index a0064e26a..2204c77e8 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -283,7 +283,9 @@ func RunAgent(args []string) {

// whether or not the activator is, because the mint gate is the reconcile

// pass's. A box with no deployment store reads it as "nothing sealed

// anywhere" and admits as before.

- armConfigGuard(newConfigGuard(os.Getenv, openDeployStore))

+ guard := newConfigGuard(os.Getenv, openDeployStore)

+ armConfigGuard(guard)

+ guard.Announce()

var act edge.Activator

var svc *activator.Service

@@ -2745,6 +2747,7 @@ func statusLoop(reg *registry.Registry) {

// The hosts this box is refusing because it cannot provide their

// config (amadan #213): private Metrics only, nothing public.

p.ConfigRefusals, p.ConfigRefusalsTotal = configRefusalProbe()

+ p.ConfigGuard = configGuardProbe()

// A fleet box (CARLOS_FLEET set) never gets here as a contender: it

// self-reports only, never attempting the election at all — no

diff --git a/internal/host/configguard.go b/internal/host/configguard.go

index f7e5bb38d..8c4e8fee3 100644

--- a/internal/host/configguard.go

+++ b/internal/host/configguard.go

@@ -84,15 +84,13 @@ type configGuard struct {

keyState appconfig.KeyState

openStore func(context.Context) (release.Store, error)

now func() time.Time

- // dataDir is where SyncExec renders <host>.config.env, the evidence a

- // partly openable manifest may fall back on (appconfig.CanProvideConfig).

- dataDir string

// noSealedConfig is the operator's explicit "this deployment has no

// sealed config" (CARLOS_NO_SEALED_CONFIG), honoured only when there is

// no deployment store at all.

noSealedConfig bool

mu sync.Mutex

+ bypassed map[string]bool // hosts already logged as admitted by the bypass

store release.Store

verdicts map[string]guardVerdict // host -> cached refusal

refused map[string]*guardRefusal

@@ -105,13 +103,9 @@ func newConfigGuard(getenv func(string) string, openStore func(context.Context)

g := &configGuard{

openStore: openStore,

now: time.Now,

- dataDir: "/data",

verdicts: map[string]guardVerdict{},

refused: map[string]*guardRefusal{},

}

- if d := getenv("CARLOS_DATA_DIR"); d != "" {

- g.dataDir = d

- }

switch strings.ToLower(strings.TrimSpace(getenv("CARLOS_NO_SEALED_CONFIG"))) {

case "1", "true":

g.noSealedConfig = true

@@ -197,15 +191,68 @@ func (g *configGuard) decide(ctx context.Context, rt registry.Route) error {

s, err := g.deployStore(ctx)

if errors.Is(err, errNoDeployStore) {

if g.noSealedConfig {

- return nil // the operator said this deployment has none

+ // The operator said this deployment has none. Said loudly, once

+ // per host: this is the one path that admits without looking.

+ g.noteBypass(rt.Host)

+ return nil

}

return &appconfig.ConfigRefusal{Reason: appconfig.ReasonNoSource, Err: err}

}

if err != nil {

return &appconfig.ConfigRefusal{Reason: appconfig.ReasonStoreError, Err: err}

}

- delivered := func() []byte { return appconfig.ReadDeliveredConfig(g.dataDir, rt) }

- return appconfig.CanProvideConfig(ctx, s, g.priv, g.keyState, release.RouteAccount(rt), rt.App, rt.Environment, rt.Host, delivered)

+ return appconfig.CanProvideConfig(ctx, s, g.priv, g.keyState, release.RouteAccount(rt), rt.App, rt.Environment, rt.Host)

+// configGuardBypassStatus is the private status word published while the

+// CARLOS_NO_SEALED_CONFIG bypass is in effect.

+const configGuardBypassStatus = "bypassed (CARLOS_NO_SEALED_CONFIG)"

+// noteBypass logs, once per host, that a placed instance was admitted without

+// its sealed config being checked.

+func (g *configGuard) noteBypass(host string) {

+ g.mu.Lock()

+ if g.bypassed == nil {

+ g.bypassed = map[string]bool{}

+ }

+ seen := g.bypassed[host]

+ g.bypassed[host] = true

+ g.mu.Unlock()

+ if !seen {

+ log.Printf("agent: config guard: BYPASS: admitting %s WITHOUT checking its sealed config, because CARLOS_NO_SEALED_CONFIG is set and this box has no deployment store. Never set it on a box that serves tenants", host)

+ }

+// Bypassed reports whether the CARLOS_NO_SEALED_CONFIG bypass is in effect:

+// the flag is set AND the box has no deployment store configured. A flag set

+// beside a store changes nothing, so it does not count.

+func (g *configGuard) Bypassed() bool {

+ if !g.noSealedConfig {

+ return false

+ }

+ _, err := g.deployStore(context.Background())

+ return errors.Is(err, errNoDeployStore)

+// Status is the private status-block word for the guard's mode: the bypass

+// words while it is in effect, "" otherwise.

+func (g *configGuard) Status() string {

+ if g.Bypassed() {

+ return configGuardBypassStatus

+ }

+ return ""

+// Announce logs the guard's mode once at startup. The bypass is the only mode

+// worth shouting about, and it is shouted whether or not it is in effect, so a

+// flag copied into the wrong host.env is visible either way.

+func (g *configGuard) Announce() {

+ switch {

+ case g.Bypassed():

+ log.Printf("agent: config guard: BYPASSED. CARLOS_NO_SEALED_CONFIG is set and no deployment store is configured, so this box admits placed exec instances without checking their sealed config. Intended for config-free development only; it is listed in this box's status snapshot")

+ case g.noSealedConfig:

+ log.Printf("agent: config guard: CARLOS_NO_SEALED_CONFIG is set but a deployment store is configured, so it is ignored and sealed config is checked as usual")

+ }

}

// deployStore opens the store on first use and keeps it. A failure is not

@@ -306,6 +353,15 @@ func configGate() func(context.Context, registry.Route) error {

return nil

}

+// configGuardProbe feeds status.Probes.ConfigGuard: "" unless the bypass is

+// in effect.

+func configGuardProbe() string {

+ if g := armedConfigGuard.Load(); g != nil {

+ return g.Status()

+ }

+ return ""

// configRefusalProbe feeds status.Probes.ConfigRefusals.

func configRefusalProbe() ([]status.ConfigRefusal, int) {

if g := armedConfigGuard.Load(); g != nil {

diff --git a/internal/host/configguard_test.go b/internal/host/configguard_test.go

index 8cb7bbe8f..828003eaa 100644

--- a/internal/host/configguard_test.go

+++ b/internal/host/configguard_test.go

@@ -32,7 +32,8 @@ type guardEnv struct {

priv *ecdh.PrivateKey

other *ecdh.PrivateKey

clock *guardClock

- // dataDir is where the guard looks for a previously rendered config file.

+ // dataDir holds leftover rendered files the tests plant to show the guard

+ // ignores them.

dataDir string

}

@@ -120,8 +121,6 @@ func (e *guardEnv) guard(keyB64 string) *configGuard {

switch k {

case "CARLOS_SECRETS_KEY":

return keyB64

- case "CARLOS_DATA_DIR":

- return e.dataDir

}

return ""

}, func(context.Context) (release.Store, error) { return e.store, nil })

@@ -409,33 +408,136 @@ func TestGuardGatesAUnitNamedExecInstance(t *testing.T) {

}

-// Partly openable config on a box with no rendered file is refused (SyncExec

-// would write nothing and the instance would start with none of it); with a

-// file this box already delivered, covering every secret var, it is admitted.

-func TestGuardPartlyOpenableConfigNeedsADeliveredFile(t *testing.T) {

+// There is no last-known-good fallback. A rendered <host>.config.env left on

+// disk proves nothing about its values: names alone pass for "A=old", for empty

+// values, or for credentials rendered for another environment. A fresh start

+// needs the whole merged config to render now.

+func TestGuardIgnoresLeftoverRenderedFiles(t *testing.T) {

+ foreign := func(e *guardEnv) string {

+ v, err := appconfig.SealUnbound(e.other.PublicKey(), []byte("z"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ return v

+ }

+ cases := map[string]struct {

+ file string

+ env string // the route's bound environment

+ }{

+ "wrong values": {"A=old\nC=wrong\n", ""},

+ "empty values": {"A=\nC=\n", ""},

+ "every name present": {"# header\nA=x\nC=y\nP=1\n", ""},

+ "environment rebound": {"A=prod-secret\nC=prod-secret\n", "staging"},

+ }

+ for name, tc := range cases {

+ t.Run(name, func(t *testing.T) {

+ e := newGuardEnv(t)

+ e.putConfig("bab", "shop", map[string]appconfig.Var{

+ "A": {Kind: "secret", Envelope: foreign(e)}, // sealed for another key

+ "C": e.seal("ok"),

+ "P": {Kind: "env", Value: "1"},

+ })

+ if tc.env != "" {

+ body, _ := json.Marshal(appconfig.Manifest{Schema: 1, App: "shop", Vars: map[string]appconfig.Var{"S": e.seal("staging")}})

+ if _, err := e.store.Store.Put(context.Background(), appconfig.EnvironmentKey("bab", "shop", tc.env), body, ""); err != nil {

+ t.Fatal(err)

+ }

+ g := e.guard(keyB64(e.priv))

+ rt := guardRoute("a.example.com")

+ rt.Environment = tc.env

+ if err := os.WriteFile(appconfig.ConfigEnvPath(e.dataDir, rt), []byte(tc.file), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ if got := reasonOfGate(t, g.Gate(context.Background(), rt)); got != appconfig.ReasonCannotOpen {

+ t.Fatalf("reason = %q, want cannot_open: a leftover file must not admit a mixed-key config", got)

+ }

+ })

+ }

+// The bypass is loud: logged at startup and once per admitted host, and

+// published in the private status block, and only while it is in effect.

+func TestGuardBypassIsLoud(t *testing.T) {

e := newGuardEnv(t)

- foreign, err := appconfig.SealUnbound(e.other.PublicKey(), []byte("z"))

- if err != nil {

- t.Fatal(err)

+ noStore := func(context.Context) (release.Store, error) { return nil, errNoDeployStore }

+ env := func(k string) string {

+ if k == "CARLOS_NO_SEALED_CONFIG" {

+ return "1"

+ }

+ return ""

+ }

+ g := newConfigGuard(env, noStore)

+ g.now = e.clock.now

+ logs := guardCaptureLog(t)

+ if got := g.Status(); got != "bypassed (CARLOS_NO_SEALED_CONFIG)" {

+ t.Errorf("status = %q", got)

+ }

+ g.Announce()

+ if !strings.Contains(logs.String(), "BYPASSED") {

+ t.Errorf("startup line missing:\n%s", logs.String())

+ }

+ for i := 0; i < 3; i++ {

+ if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil {

+ t.Fatal(err)

+ }

+ _ = g.Gate(context.Background(), guardRoute("b.example.com"))

+ if n := strings.Count(logs.String(), "BYPASS: admitting a.example.com"); n != 1 {

+ t.Errorf("a.example.com logged %d times, want once:\n%s", n, logs.String())

+ }

+ if n := strings.Count(logs.String(), "BYPASS: admitting b.example.com"); n != 1 {

+ t.Errorf("b.example.com logged %d times, want once", n)

}

- e.putConfig("bab", "shop", map[string]appconfig.Var{

- "A": {Kind: "secret", Envelope: foreign},

- "C": e.seal("ok"),

- })

- g := e.guard(keyB64(e.priv))

- rt := guardRoute("a.example.com")

- if got := reasonOfGate(t, g.Gate(context.Background(), rt)); got != appconfig.ReasonCannotOpen {

- t.Fatalf("fresh box: reason = %q, want cannot_open", got)

+ // The flag beside a real store is ignored, and says so, and is not

+ // reported as a bypass.

+ withStore := newConfigGuard(env, func(context.Context) (release.Store, error) { return e.store, nil })

+ logs2 := guardCaptureLog(t)

+ if got := withStore.Status(); got != "" {

+ t.Errorf("flag with a store reported %q", got)

+ }

+ withStore.Announce()

+ if !strings.Contains(logs2.String(), "ignored") || strings.Contains(logs2.String(), "BYPASSED") {

+ t.Errorf("flag-with-store line wrong:\n%s", logs2.String())

}

- // The same box after sync once delivered a file for this host.

- e.clock.advance(configGuardRefuseTTL + time.Second)

- if err := os.WriteFile(appconfig.ConfigEnvPath(e.dataDir, rt), []byte("A=old\nC=ok\n"), 0o600); err != nil {

- t.Fatal(err)

+ // No flag: quiet, and nothing published.

+ plain := newConfigGuard(func(string) string { return "" }, noStore)

+ logs3 := guardCaptureLog(t)

+ plain.Announce()

+ if plain.Status() != "" || logs3.Len() != 0 {

+ t.Errorf("a box without the flag said something: %q / %q", plain.Status(), logs3.String())

}

- if err := g.Gate(context.Background(), rt); err != nil {

- t.Fatalf("box with a delivered file covering both secrets: %v", err)

+// The opt-out never covers a store that is configured and failing: the box

+// cannot tell, and refuses, flag or not.

+func TestGuardOptOutDoesNotCoverAConfiguredStoreError(t *testing.T) {

+ e := newGuardEnv(t)

+ env := func(k string) string {

+ if k == "CARLOS_NO_SEALED_CONFIG" {

+ return "1"

+ }

+ return ""

+ }

+ // Opening fails.

+ broken := newConfigGuard(env, func(context.Context) (release.Store, error) { return nil, fmt.Errorf("aws: no credentials") })

+ broken.now = e.clock.now

+ if got := reasonOfGate(t, broken.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError {

+ t.Errorf("open failure with the opt-out: reason = %q, want store_error", got)

+ }

+ if broken.Status() != "" {

+ t.Errorf("an open failure was reported as a bypass: %q", broken.Status())

+ }

+ // Opens, then reads fail.

+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})

+ e.store.setFail(errors.New("s3: 503"))

+ flaky := newConfigGuard(env, func(context.Context) (release.Store, error) { return e.store, nil })

+ flaky.now = e.clock.now

+ if got := reasonOfGate(t, flaky.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError {

+ t.Errorf("read failure with the opt-out: reason = %q, want store_error", got)

}

diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go

index bd6866f7d..445720e6f 100644

--- a/internal/status/contract_test.go

+++ b/internal/status/contract_test.go

@@ -75,7 +75,7 @@ var allowedBoxKeys = map[string][]string{

"": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"},

"routes": {"static", "resident", "hibernating", "awake", "asleep"},

"replication": {"healthy", "lag_seconds"},

- "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "config_refusals", "config_refusals_total"},

+ "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "config_refusals", "config_refusals_total", "config_guard"},

"metrics.route_versions": {wildcardAnyKey},

"metrics.upstreams": {wildcardAnyKey},

"metrics.upstreams.*[]": {"addr", "up", "since", "fails", "last_error"},

@@ -302,6 +302,7 @@ func fixtureProbes() status.Probes {

{Host: "contract-refused.internal", Reason: "cannot_open", Since: time.Now().Add(-time.Hour), Last: time.Now(), Count: 2},

ConfigRefusalsTotal: 1,

+ ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)",

MemFrac: &memFrac,

DiskFrac: &diskFrac,

Listening: listening,

diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go

index e75efb795..d9560f94b 100644

--- a/internal/status/snapshot.go

+++ b/internal/status/snapshot.go

@@ -138,6 +138,9 @@ type Probes struct {

// how many are refused in all when the guard capped the list.

ConfigRefusals []ConfigRefusal

ConfigRefusalsTotal int

+ // ConfigGuard is "" in the normal mode and a words-only note when the

+ // config guard's explicit bypass is in effect on this box (amadan #213).

+ ConfigGuard string

}

// ConfigRefusal is one host this box is refusing to take because it cannot

@@ -286,6 +289,12 @@ type BoxMetrics struct {

// ConfigRefusalsTotal is the count of refused hosts when ConfigRefusals

// was capped, so a long list reads as long. Omitted when zero.

ConfigRefusalsTotal int `json:"config_refusals_total,omitempty"`

+ // ConfigGuard says the box's admission guard is bypassed, e.g.

+ // "bypassed (CARLOS_NO_SEALED_CONFIG)": it admits placed exec instances

+ // without checking their sealed config. Omitted in the normal mode, so an

+ // operator scanning snapshots sees only the boxes that are exposed. A

+ // fixed phrase, never a value from the environment.

+ ConfigGuard string `json:"config_guard,omitempty"`

}

// AliasReading is one alias hostname's status as CollectBox saw it this

@@ -654,7 +663,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {

p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" ||

p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil ||

len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 ||

- len(routeDBBytes) > 0 || len(p.ConfigRefusals) > 0 {

+ len(routeDBBytes) > 0 || len(p.ConfigRefusals) > 0 || p.ConfigGuard != "" {

box.Metrics = &BoxMetrics{

MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes,

Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes,

@@ -673,6 +682,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {

// A fresh copy: the guard's own slice is its to mutate.

ConfigRefusals: append([]ConfigRefusal(nil), p.ConfigRefusals...),

ConfigRefusalsTotal: p.ConfigRefusalsTotal,

+ ConfigGuard: p.ConfigGuard,

}

diff --git a/internal/status/status_test.go b/internal/status/status_test.go

index 8d6c2d195..c9f6c9035 100644

--- a/internal/status/status_test.go

+++ b/internal/status/status_test.go

@@ -1496,3 +1496,27 @@ func TestCollectBoxCarriesConfigRefusals(t *testing.T) {

t.Errorf("a box with no refusal publishes config_refusals: %s", raw)

}

+// The config guard's bypass is visible in the private block, and only there.

+func TestCollectBoxCarriesTheConfigGuardBypass(t *testing.T) {

+ b := status.CollectBox(nil, status.Probes{MachineID: "m", ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)"}, time.Now())

+ if b.Metrics == nil || b.Metrics.ConfigGuard != "bypassed (CARLOS_NO_SEALED_CONFIG)" {

+ t.Fatalf("metrics = %+v, want the bypass word", b.Metrics)

+ }

+ s := teststore(t)

+ ctx := context.Background()

+ if err := status.WriteBox(ctx, s, b); err != nil {

+ t.Fatal(err)

+ }

+ f, err := status.BuildFleet(ctx, s, nil, time.Now(), 5*time.Minute)

+ if err != nil {

+ t.Fatal(err)

+ }

+ fj, _ := json.Marshal(f)

+ if strings.Contains(string(fj), "bypass") || strings.Contains(string(fj), "NO_SEALED") {

+ t.Errorf("fleet.json carries the bypass: %s", fj)

+ }

+ if quiet := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); quiet.Metrics != nil {

+ t.Errorf("a box in the normal mode attached metrics: %+v", quiet.Metrics)

+ }