amadan

carlos / platform Public

fix/keyless-box-refuses

Squashed from fix/keyless-box-refuses into main.
claude 5cd92594f97cf8ac3005e990cf20fb8a584b0292 parent 6956db6
19 files changed, +2085 −24
  • README.md +6 −2
  • docs/superpowers/plans/2026-10-02-nearest-region-copy.json +1 −1
  • internal/activator/activator.go +56 −4
  • internal/activator/activator_test.go +14 −0
  • internal/activator/backend_exec.go +29 −0
  • internal/activator/backend_unit.go +9 −0
  • internal/activator/config_gate_test.go +219 −0
  • internal/appconfig/canprovide.go +132 −0
  • internal/appconfig/canprovide_test.go +280 −0
  • internal/host/agent.go +20 −1
  • internal/host/configguard.go +371 −0
  • internal/host/configguard_test.go +615 −0
  • internal/provision/config_gate_test.go +116 −0
  • internal/provision/reconcile.go +23 −0
  • internal/provision/sidecar_reconcile.go +8 −0
  • internal/status/contract_test.go +26 −15
  • internal/status/dryrun.go +31 −0
  • internal/status/snapshot.go +53 −1
  • internal/status/status_test.go +76 −0

CI cancelled — superseded by a newer push — run details

diff --git a/README.md b/README.md
index bee763f04..8482dcd84 100644
--- a/README.md
+++ b/README.md
@@ -276,8 +276,8 @@ own rollback: it stops new non-home regions and keeps existing ones in force.
1. Stage the deployment's `CARLOS_SECRETS_KEY` on every box in every region
that should take placed instances, and set `CARLOS_SECRETS_PUBKEY` on the
- console. A box without the key runs placed instances with no config, no
- secrets and no store.
+ console. A box without the key refuses to mint or wake placed exec instances
+ whose config contains sealed secrets.
2. Roll the box release that publishes `serves_shared_pool`,
`accepts_inbound` and `secrets_key_fp`. Check every region's
`control/edges/*.json` carries them, and that each `secrets_key_fp` equals
@@ -601,6 +601,10 @@ no single reader owns the whole contract:
`status/fleet.json` itself under that prefix too — moot today, since
nothing sets a prefix on a publishing (non-fleet) box, and a fleet box
never publishes at all (see `CARLOS_FLEET`, below).
+- A box with `CARLOS_HIBERNATE_BUCKET` set but no deployment store
+ (`CARLOS_DEPLOYMENT_BUCKET` or `CARLOS_DEPLOYMENT_DIR`) refuses placed exec
+ instances, because it cannot tell whether their config is sealed. Set
+ `CARLOS_NO_SEALED_CONFIG=1` to say this deployment has none.
- `CARLOS_FLEET` — read by `carlos agent`'s status loop (customer-fleets
plan, Task 5). Set (to the fleet's name) marks this agent as a fleet box:
it still self-reports its own snapshot every tick, but never contends for
diff --git a/docs/superpowers/plans/2026-10-02-nearest-region-copy.json b/docs/superpowers/plans/2026-10-02-nearest-region-copy.json
index 9a15c3d8c..addb19a06 100644
--- a/docs/superpowers/plans/2026-10-02-nearest-region-copy.json
+++ b/docs/superpowers/plans/2026-10-02-nearest-region-copy.json
@@ -597,7 +597,7 @@
{
"id": "D7",
"file": "README.md",
- "text": "**Turning app regions on.** In order:\n\n1. Stage the deployment's `CARLOS_SECRETS_KEY` on every box in every region\n that should take placed instances, and set `CARLOS_SECRETS_PUBKEY` on the\n console. A box without the key runs placed instances with no config, no\n secrets and no store.\n2. Roll the box release that publishes `serves_shared_pool`,\n `accepts_inbound` and `secrets_key_fp`. Check every region's\n `control/edges/*.json` carries them, and that each `secrets_key_fp` equals\n the console's.\n3. Roll the console with the switch off.\n4. On dev, turn the switch on. Create an app and an instance, read their\n region, and run the service gate below for every region.\n5. On production, with the switch still off, create one probe instance per\n eligible region with an explicit `--region`, deploy a known build, and run\n `scripts/region-service-gate.sh <probe-host> <build> <every other edge EIP>`\n for each. Every region must pass. Delete the probes.\n6. Turn the switch on. Re-run step 5 after any period with the switch off,\n and whenever a new region becomes eligible."
+ "text": "**Turning app regions on.** In order:\n\n1. Stage the deployment's `CARLOS_SECRETS_KEY` on every box in every region\n that should take placed instances, and set `CARLOS_SECRETS_PUBKEY` on the\n console. A box without the key refuses to mint or wake placed exec instances\n whose config contains sealed secrets.\n2. Roll the box release that publishes `serves_shared_pool`,\n `accepts_inbound` and `secrets_key_fp`. Check every region's\n `control/edges/*.json` carries them, and that each `secrets_key_fp` equals\n the console's.\n3. Roll the console with the switch off.\n4. On dev, turn the switch on. Create an app and an instance, read their\n region, and run the service gate below for every region.\n5. On production, with the switch still off, create one probe instance per\n eligible region with an explicit `--region`, deploy a known build, and run\n `scripts/region-service-gate.sh <probe-host> <build> <every other edge EIP>`\n for each. Every region must pass. Delete the probes.\n6. Turn the switch on. Re-run step 5 after any period with the switch off,\n and whenever a new region becomes eligible."
},
{
"id": "D8",
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index 15e9b11f5..0a719ed03 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -574,6 +574,30 @@ func (e *errLeaseHeld) Error() string {
return fmt.Sprintf("instance lease held by %q (retry in %s)", e.holder, e.retryAfter)
}
+// errConfigRefused: this box cannot provide the instance's sealed config
+// (no key, the wrong key, or it could not tell), so it will not take the
+// lease (amadan #213). The condition outlives one request and a keyed
+// sibling may take the lease in the meantime, so the edge answers 503 with a
+// longer Retry-After than a lease contention gets.
+type errConfigRefused struct{ err error }
+
+func (e *errConfigRefused) Error() string {
+ return "instance config unavailable on this box: " + e.err.Error()
+}
+func (e *errConfigRefused) Unwrap() error { return e.err }
+
+// configAdmitter is the optional Backend capability behind the config gate
+// (amadan #213): AdmitConfig answers whether this box can provide rt's config.
+// Acquire asks it before taking a lease; paths that keep a lease and start a
+// child anyway (a worker's in-place respawn) ask it directly. A backend
+// without it is not gated.
+type configAdmitter interface {
+ AdmitConfig(ctx context.Context, rt registry.Route) error
+}
+
+// configRefusedRetryAfter is the Retry-After (seconds) a refused wake sends.
+const configRefusedRetryAfter = 30
+
var errWakeTimeout = errors.New("timed out waiting for instance wake")
// Service is the activator (Eleven's activatorSvc). It structurally satisfies
@@ -1826,6 +1850,12 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string) {
spawnVer string
tok string
err error
+ // refused is the config gate's answer for this respawn (amadan
+ // #213). The lease is still held, so Acquire's gate is not in
+ // the way, and a respawn onto config this box cannot provide
+ // would start a worker without its secrets while keeping the
+ // lease from a sibling that could run it.
+ refused error
)
if stopOld != nil {
stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout)
@@ -1856,11 +1886,16 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string) {
// Placed inside `wanted` deliberately: an abandoned respawn (the
// entry was stopped or replaced before we got here) must not
// clear a row that isn't this attempt's to touch.
- if cerr := a.due.SetVersions(host, "", rt.Version); cerr != nil {
- log.Printf("activator: clear worker version for %s: %v", host, cerr)
+ if g, ok := a.backend.(configAdmitter); ok {
+ refused = g.AdmitConfig(ctx, rt)
}
- if tok, err = newWorkerToken(); err == nil {
- proc, spawnVer, err = a.backend.StartWorker(ctx, rt, workerEnv(a.agentSocket(), host, tok))
+ if refused == nil {
+ if cerr := a.due.SetVersions(host, "", rt.Version); cerr != nil {
+ log.Printf("activator: clear worker version for %s: %v", host, cerr)
+ }
+ if tok, err = newWorkerToken(); err == nil {
+ proc, spawnVer, err = a.backend.StartWorker(ctx, rt, workerEnv(a.agentSocket(), host, tok))
+ }
}
cancel()
}
@@ -1879,6 +1914,15 @@ func (a *Service) respawnWorkerLocked(e *actEntry, stopOld Proc, why string) {
_ = proc.Stop(shedCtx)
shedCancel()
}
+ case refused != nil:
+ // Stand down: the old child is already gone, and nothing may
+ // start in its place. A normal stop checkpoints and releases the
+ // lease, so a keyed sibling can take it. Nothing re-wakes it here
+ // until the box can provide the config (the config gate refuses
+ // every wake before then).
+ a.beginStopLocked(e)
+ a.mu.Unlock()
+ log.Printf("activator: worker %s NOT respawned (%s): %v — standing down and releasing its lease", host, why, refused)
case err != nil:
a.mu.Unlock()
log.Printf("activator: WORKER %s failed to respawn (%s): %v — the sweep's crash restart takes it from here", host, why, err)
@@ -2969,9 +3013,17 @@ func (a *Service) serve(w http.ResponseWriter, r *http.Request, rt registry.Rout
release, err := a.begin(r.Context(), rt, stamp)
if err != nil {
var held *errLeaseHeld
+ var refused *errConfigRefused
switch {
case errors.Is(err, ErrFenced):
maintenanceResponse(w, r)
+ case errors.As(err, &refused):
+ // Quiet by design: the config guard logs once per host and
+ // publishes the refusal in the box's status snapshot. A line per
+ // request here would bury that one.
+ w.Header().Set("Retry-After", strconv.Itoa(configRefusedRetryAfter))
+ edge.MarkAnswered(r)
+ http.Error(w, "instance failed to wake", http.StatusServiceUnavailable)
case errors.As(err, &held):
retry := int(held.retryAfter/time.Second) + 1
w.Header().Set("Retry-After", strconv.Itoa(retry))
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go
index 93f38aac7..9faa04c4f 100644
--- a/internal/activator/activator_test.go
+++ b/internal/activator/activator_test.go
@@ -215,11 +215,15 @@ func (l *fakeLease) Release(context.Context) error {
type fakeBackend struct {
mu sync.Mutex
acquires int
+ admits int // AdmitConfig calls
restores int
starts int
seeds int
releases int // lease releases — the failure paths' assertion
acquireErr error
+ // admitErr is what AdmitConfig answers: the config gate a retained-lease
+ // respawn must ask again (amadan #213).
+ admitErr error
// restoreFresh is what Restore reports: a first boot (local DB absent AND
// no replica). False by default, so every pre-seed test doubles as the
// "a restored instance is never seeded" case.
@@ -345,6 +349,16 @@ func (b *fakeBackend) Acquire(context.Context, registry.Route) (Lease, error) {
return &fakeLease{rep: rep, b: b}, nil
}
+func (b *fakeBackend) AdmitConfig(context.Context, registry.Route) error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.admits++
+ if b.admitErr != nil {
+ return &errConfigRefused{err: b.admitErr}
+ }
+ return nil
+}
+
func (b *fakeBackend) Restore(context.Context, registry.Route) (bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 114ee0987..60fbaa181 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -142,6 +142,14 @@ type ExecBackend struct {
// gate (every unit test and any box that predates the check). Wired in
// production by Task 6.
NestedOK func() error
+ // ConfigGate answers whether this box can provide rt's config (amadan
+ // #213): nil means yes, an error means refuse. Acquire asks it BEFORE
+ // lease.Acquire, so a box that would start the instance without its
+ // secrets never holds the lease, and a keyed sibling takes it instead.
+ // The gate owns its own policy, logging and operator-visible record of
+ // the refusal; this backend only honours the answer. Nil: no gate (every
+ // unit test, and a box with no deploy store to read config from).
+ ConfigGate func(ctx context.Context, rt registry.Route) error
// ObservePoll is how often the exit observer (unitproc.go) and a
// seed's cgroup-empty wait re-check unit state. 0: one second. A field
// so tests can shrink the wait without a package-global var shared
@@ -227,6 +235,13 @@ func (b *ExecBackend) LiveVersion(host string) (string, error) {
// Retry-After that tracks the holder's expiry.
func (b *ExecBackend) Acquire(ctx context.Context, rt registry.Route) (Lease, error) {
host := rt.Host
+ // Admission before the lease: the instance's config has to be
+ // providable from THIS box, or it must not be the one that starts it.
+ // Here, not in doWake, because this is the only place an instance or
+ // sidecar lease is taken; every wake source reaches it.
+ if err := b.AdmitConfig(ctx, rt); err != nil {
+ return nil, err
+ }
held, err := lease.Acquire(ctx, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host), b.Cfg.Region)
if err != nil {
var he *lease.HeldError
@@ -242,6 +257,20 @@ func (b *ExecBackend) Acquire(ctx context.Context, rt registry.Route) (Lease, er
return &execLease{held: held, defaultDB: b.pathsFor(rt).dbPath}, nil
}
+// AdmitConfig asks the config gate whether this box can provide rt's config:
+// nil means yes (or no gate), an *errConfigRefused means no. Acquire calls it
+// before the lease; the activator calls it again before an in-place worker
+// respawn, which keeps its lease and so never passes through Acquire.
+func (b *ExecBackend) AdmitConfig(ctx context.Context, rt registry.Route) error {
+ if b.ConfigGate == nil {
+ return nil
+ }
+ if err := b.ConfigGate(ctx, rt); err != nil {
+ return &errConfigRefused{err: err}
+ }
+ return nil
+}
+
// execLease adapts *lease.Held to the Lease seam.
type execLease struct {
held *lease.Held
diff --git a/internal/activator/backend_unit.go b/internal/activator/backend_unit.go
index aa59652b6..137f63de0 100644
--- a/internal/activator/backend_unit.go
+++ b/internal/activator/backend_unit.go
@@ -299,6 +299,15 @@ func (b *SplitBackend) pick(rt registry.Route) Backend {
func (b *SplitBackend) Acquire(ctx context.Context, rt registry.Route) (Lease, error) {
return b.pick(rt).Acquire(ctx, rt)
}
+
+// AdmitConfig dispatches like every other call: on Backing alone. A unit
+// tenant's config is the root adopt pass's, so only the exec half is gated.
+func (b *SplitBackend) AdmitConfig(ctx context.Context, rt registry.Route) error {
+ if g, ok := b.pick(rt).(configAdmitter); ok {
+ return g.AdmitConfig(ctx, rt)
+ }
+ return nil
+}
func (b *SplitBackend) Restore(ctx context.Context, rt registry.Route) (bool, error) {
return b.pick(rt).Restore(ctx, rt)
}
diff --git a/internal/activator/config_gate_test.go b/internal/activator/config_gate_test.go
new file mode 100644
index 000000000..5c4aa5c17
--- /dev/null
+++ b/internal/activator/config_gate_test.go
@@ -0,0 +1,219 @@
+package activator
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// Issue #213: a box that cannot provide an instance's config must not take
+// its lease. The gate runs inside ExecBackend.Acquire, before lease.Acquire,
+// so every wake source (request, tick, sweeper, worker boot) is covered by
+// the one place a lease is taken.
+
+// The gate is asked about the route being woken, and a refusal means the
+// lease is never attempted. Cfg.Bucket is empty here, so a call that fell
+// through to lease.Acquire would fail with a different error.
+func TestExecBackendAcquireRefusesBeforeTheLease(t *testing.T) {
+ var asked []registry.Route
+ boom := errors.New("cannot open sealed config")
+ b := &ExecBackend{ConfigGate: func(_ context.Context, rt registry.Route) error {
+ asked = append(asked, rt)
+ return boom
+ }}
+ rt := registry.Route{Host: "kpmg.example.com", Kind: "instance", App: "shop", Hibernate: true}
+
+ lease, err := b.Acquire(context.Background(), rt)
+ if lease != nil {
+ t.Fatalf("lease = %v, want nil", lease)
+ }
+ var refused *errConfigRefused
+ if !errors.As(err, &refused) {
+ t.Fatalf("err = %v, want *errConfigRefused", err)
+ }
+ if !errors.Is(err, boom) {
+ t.Errorf("err %v does not wrap the gate's reason", err)
+ }
+ if len(asked) != 1 || asked[0].Host != rt.Host {
+ t.Errorf("gate asked about %v, want exactly %s", asked, rt.Host)
+ }
+}
+
+// The refusal is the member's 503 with a Retry-After, answered quietly (the
+// guard logs once per host; a log line per request would drown it), and the
+// failed wake leaves no entry behind so the next request asks again.
+func TestActivatorConfigRefusal503(t *testing.T) {
+ a, b, _, reg := testActivator(t, time.Minute)
+ b.acquireErr = &errConfigRefused{err: errors.New("no key")}
+
+ if err := reg.Upsert(registry.Route{Host: "nokey.example.com", Kind: "instance", Socket: "/tmp/nokey.sock", Hibernate: true}); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ rt, err := newTestRouter(reg, a)
+ if err != nil {
+ t.Fatalf("new router: %v", err)
+ }
+ front := httptest.NewServer(rt)
+ t.Cleanup(front.Close)
+
+ req, _ := http.NewRequest("GET", front.URL+"/", nil)
+ req.Host = "nokey.example.com"
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("request: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusServiceUnavailable {
+ t.Fatalf("status %d, want 503", resp.StatusCode)
+ }
+ if ra := resp.Header.Get("Retry-After"); ra != "30" {
+ t.Fatalf("Retry-After %q, want 30", ra)
+ }
+ if a.entryCount() != 0 {
+ t.Fatal("refused wake left an entry behind")
+ }
+ // Nothing past Acquire ran: no restore, no spawn.
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ if b.restores != 0 || b.starts != 0 {
+ t.Fatalf("refused wake went on: restores=%d starts=%d", b.restores, b.starts)
+ }
+}
+
+// A worker respawn keeps its lease and calls StartWorker directly, so it
+// bypasses Acquire. The config gate has to be asked again there, and a refusal
+// has to stand the worker down (stop it, release the lease) rather than leave
+// a childless entry holding the lease from a sibling that could run it.
+func TestWorkerRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) {
+ for _, why := range []string{"crash", "max_run"} {
+ t.Run(why, func(t *testing.T) {
+ a, b, clk, _ := testActivator(t, time.Hour)
+ rt := alwaysOnWorkerRoute("bot.x.internal")
+ rt.MaxRunSeconds = 30
+ if err := a.WakeWorker(rt); err != nil {
+ t.Fatalf("WakeWorker: %v", err)
+ }
+ if b.workerStarts() != 1 {
+ t.Fatalf("workerStarts = %d, want 1", b.workerStarts())
+ }
+ first := b.lastWorker()
+
+ // Secrets arrive after the worker started; this box cannot open them.
+ b.mu.Lock()
+ b.admitErr = errors.New("no key")
+ b.mu.Unlock()
+
+ if why == "crash" {
+ first.kill()
+ } else {
+ clk.advance(31 * time.Second)
+ }
+ a.sweepOnce()
+
+ waitFor(t, "the refused worker to stand down", func() bool { return a.entryCount() == 0 })
+ if n := b.workerStarts(); n != 1 {
+ t.Fatalf("workerStarts = %d: a refused respawn still started a child", n)
+ }
+ if !first.isStopped() && why == "max_run" {
+ t.Error("the old child was left running")
+ }
+ if n := b.releaseCount(); n != 1 {
+ t.Fatalf("lease releases = %d, want 1: a refused worker must give its lease back", n)
+ }
+ if !b.lastRep().isClosed() {
+ t.Error("replication was left open on a refused worker")
+ }
+ })
+ }
+}
+
+// Build convergence shares the respawn helper but is its own trigger: a ship
+// repoints live/<host>, the drift outlasts ConvergeGrace, and the always-on
+// worker would be cycled onto the new build. With the gate refusing, it stands
+// down instead of starting the new build without its config.
+func TestWorkerConvergeRespawnIsRefusedByTheConfigGateAndStandsDown(t *testing.T) {
+ a, b, clk, ver := shippingActivator(t, time.Hour)
+ host := "bot.x.internal"
+ if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil {
+ t.Fatalf("WakeWorker: %v", err)
+ }
+ first := b.lastWorker()
+ b.mu.Lock()
+ b.admitErr = errors.New("cannot open sealed config")
+ b.mu.Unlock()
+
+ ver.Store("v2")
+ a.sweepOnce() // first sighting of the drift
+ clk.advance(ConvergeGrace + time.Minute)
+ a.sweepOnce() // the converge respawn, refused
+
+ waitFor(t, "the refused convergence to stand down", func() bool { return a.entryCount() == 0 })
+ if n := b.workerStarts(); n != 1 {
+ t.Fatalf("workerStarts = %d: a refused convergence still started the new build", n)
+ }
+ if !first.isStopped() {
+ t.Error("the old child was left running")
+ }
+ if n := b.releaseCount(); n != 1 {
+ t.Fatalf("lease releases = %d, want 1", n)
+ }
+ if !b.lastRep().isClosed() {
+ t.Error("replication was left open")
+ }
+}
+
+// And a gate that allows it changes nothing: the respawn runs in place and
+// keeps its lease, as before.
+func TestWorkerRespawnProceedsWhenTheConfigGateAllows(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Hour)
+ if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil {
+ t.Fatalf("WakeWorker: %v", err)
+ }
+ b.lastWorker().kill()
+ a.sweepOnce()
+ waitForRestart(t, a, b, "bot.x.internal", 2)
+ if n := b.releaseCount(); n != 0 {
+ t.Fatalf("lease releases = %d, want 0", n)
+ }
+ b.mu.Lock()
+ admits := b.admits
+ b.mu.Unlock()
+ if admits == 0 {
+ t.Error("the respawn never asked the config gate")
+ }
+}
+
+// The gate follows dispatch, which is Backing alone (SplitBackend.pick). A
+// socket-backed hibernating instance may carry a Unit with an empty Backing
+// and still reaches ExecBackend.Acquire, so it is gated; only Backing == unit
+// is exempt.
+func TestSplitBackendGatesWhatDispatchSendsToExec(t *testing.T) {
+ var asked []string
+ exec := &ExecBackend{ConfigGate: func(_ context.Context, rt registry.Route) error {
+ asked = append(asked, rt.Host)
+ return errors.New("no key")
+ }}
+ b := &SplitBackend{Exec: exec, Unit: &UnitBackend{}}
+
+ unitNamed := registry.Route{Host: "named.example.com", Kind: "instance", App: "shop",
+ Socket: "/run/named.sock", Unit: "x.service", Hibernate: true}
+ if _, err := b.Acquire(context.Background(), unitNamed); err == nil {
+ t.Fatal("Acquire of a Unit-named, empty-Backing instance was not refused")
+ }
+ if err := b.AdmitConfig(context.Background(), unitNamed); err == nil {
+ t.Fatal("AdmitConfig of a Unit-named, empty-Backing instance was not refused")
+ }
+ unitBacked := unitNamed
+ unitBacked.Host, unitBacked.Backing = "backed.example.com", registry.BackingUnit
+ if err := b.AdmitConfig(context.Background(), unitBacked); err != nil {
+ t.Fatalf("a Backing=unit route must not be gated: %v", err)
+ }
+ if len(asked) != 2 || asked[0] != "named.example.com" || asked[1] != "named.example.com" {
+ t.Fatalf("gate asked %v, want only the Unit-named exec route, twice", asked)
+ }
+}
diff --git a/internal/appconfig/canprovide.go b/internal/appconfig/canprovide.go
new file mode 100644
index 000000000..3063c534c
--- /dev/null
+++ b/internal/appconfig/canprovide.go
@@ -0,0 +1,132 @@
+package appconfig
+
+import (
+ "context"
+ "crypto/ecdh"
+ "fmt"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The admission question (amadan #213): can THIS box provide this instance's
+// config? A box that cannot must not take the instance's lease or mint its
+// record, because the instance would start without its secrets. That is what
+// a keyless UK box did to kpmg for 1h54m on 2026-10-01: the config sync loop
+// is gated on CARLOS_SECRETS_KEY, instanceEnv treats a missing
+// <host>.config.env as optional, and the instance booted with nothing.
+//
+// The answer is the same question the sync pass asks, asked before the side
+// effect instead of after: resolve the merged manifest the box would render
+// for this host (default < bound environment < instance overlay), and if it
+// carries sealed secrets, prove this box's key opens them.
+
+// KeyState says what the box found when it read CARLOS_SECRETS_KEY. The
+// private key itself travels separately; this is only why there is none.
+type KeyState int
+
+const (
+ // KeyOK: the key parsed. The *ecdh.PrivateKey argument is non-nil.
+ KeyOK KeyState = iota
+ // KeyAbsent: CARLOS_SECRETS_KEY is unset or blank.
+ KeyAbsent
+ // KeyUnparseable: it is set but is not a P-256 private key.
+ KeyUnparseable
+)
+
+// The closed set of refusal reasons. They are published in the box's status
+// snapshot, so they are words, never text built from a store or crypto error.
+const (
+ ReasonNoKey = "no_key"
+ ReasonKeyUnparseable = "key_unparseable"
+ ReasonCannotOpen = "cannot_open"
+ ReasonStoreError = "store_error"
+ // ReasonNoSource: the box has no deployment store configured, so it
+ // cannot read the instance's config to know whether any is sealed. It is
+ // a refusal unless the operator has said, explicitly, that this
+ // deployment has no sealed config (the host's CARLOS_NO_SEALED_CONFIG).
+ ReasonNoSource = "no_config_source"
+)
+
+// ConfigRefusal is CanProvideConfig's refusal. Reason is one of the Reason*
+// words above and is the only part safe to publish; Err is the operator-log
+// detail (never a secret: crypto errors name no plaintext and no envelope)
+// and is what errors.Is/As see through.
+type ConfigRefusal struct {
+ Reason string
+ Err error
+}
+
+func (r *ConfigRefusal) Error() string {
+ if r.Err == nil {
+ return "cannot provide this instance's config: " + r.Reason
+ }
+ return "cannot provide this instance's config: " + r.Reason + ": " + r.Err.Error()
+}
+
+func (r *ConfigRefusal) Unwrap() error { return r.Err }
+
+// CanProvideConfig reports nil when this box can provide the config of the
+// instance at host, and a *ConfigRefusal when it cannot or cannot tell.
+//
+// - Nothing sealed in the merged view (no manifest at all, or only plain
+// vars): nothing needs a key, so nil. A deployment with no sealed config
+// anywhere is unaffected, key or no key.
+// - Something sealed: the box needs a parseable key, and the merged view
+// must RENDER, which is what SyncExec does to produce <host>.config.env
+// and what instanceEnv then reads: every envelope opens under the route's
+// own binding and every value is writable. Sealing records no recipient
+// (an envelope is ephemeral point, nonce, ciphertext), so there is no
+// fingerprint to compare; rendering is the proof, and the only one that
+// matches what delivery needs.
+// - Any read or parse failure refuses (ReasonStoreError). An error is never
+// "no sealed config".
+//
+// Complete renderability, not "one envelope opens": SyncExec refuses to write
+// the WHOLE file when any one decrypt fails, and instanceEnv treats a missing
+// file as optional, so a half-openable manifest on a box with no rendered
+// file starts the instance with none of its sealed config.
+//
+// There is no last-known-good fallback, deliberately. A rendered
+// <host>.config.env left on disk from an earlier pass proves nothing about
+// its VALUES: names alone pass for "A=old" or "A=" or credentials rendered for
+// another environment, and sync leaves such a file untouched when it cannot
+// render. A fresh acquisition or respawn starts only if the whole merged
+// config renders now. The fix for a mixed-key or partly corrupt manifest is
+// to give the box the key or re-seal the config. (Continuing a process that is
+// already running is a different policy and not this function's.)
+func CanProvideConfig(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, ks KeyState, acct, app, envName, host string) error {
+ var (
+ m *Manifest
+ err error
+ )
+ if ValidInstanceHost(host) {
+ m, _, err = LoadMergedHost(ctx, s, acct, app, envName, host)
+ } else {
+ // A host that cannot name an overlay key has no overlay (the sweeps
+ // treat it the same way); the app-level layers still apply.
+ m, err = LoadMerged(ctx, s, acct, app, envName)
+ }
+ if err != nil {
+ return &ConfigRefusal{Reason: ReasonStoreError, Err: err}
+ }
+ sealed := 0
+ for _, v := range m.Vars {
+ if v.Kind == "secret" {
+ sealed++
+ }
+ }
+ if sealed == 0 {
+ return nil
+ }
+ if priv == nil || ks != KeyOK {
+ if ks == KeyUnparseable {
+ return &ConfigRefusal{Reason: ReasonKeyUnparseable}
+ }
+ return &ConfigRefusal{Reason: ReasonNoKey}
+ }
+ bind := Binding{Account: acct, App: app}
+ if _, err := RenderEnv(m, priv, bind); err != nil {
+ return &ConfigRefusal{Reason: ReasonCannotOpen, Err: fmt.Errorf("this box's key cannot render the config: %w", err)}
+ }
+ return nil
+}
diff --git a/internal/appconfig/canprovide_test.go b/internal/appconfig/canprovide_test.go
new file mode 100644
index 000000000..8ebc1832c
--- /dev/null
+++ b/internal/appconfig/canprovide_test.go
@@ -0,0 +1,280 @@
+package appconfig
+
+import (
+ "context"
+ "crypto/ecdh"
+ "encoding/json"
+ "errors"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// Issue #213: a box takes an instance's lease only if it can provide that
+// instance's config. These tests pin the predicate the activator and the
+// reconcile pass both ask.
+
+type canProvideEnv struct {
+ t *testing.T
+ s release.Store
+ priv *ecdh.PrivateKey
+ pub *ecdh.PublicKey
+ other *ecdh.PrivateKey // a key this deployment never sealed for
+}
+
+func newCanProvideEnv(t *testing.T) *canProvideEnv {
+ t.Helper()
+ priv, err := ecdh.P256().GenerateKey(randReader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ other, err := ecdh.P256().GenerateKey(randReader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &canProvideEnv{t: t, s: release.NewDirStore(t.TempDir()), priv: priv, pub: priv.PublicKey(), other: other}
+}
+
+func (e *canProvideEnv) put(key string, m Manifest) {
+ e.t.Helper()
+ m.Schema = 1
+ body, err := json.Marshal(m)
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ if _, err := e.s.Put(context.Background(), key, body, ""); err != nil {
+ e.t.Fatal(err)
+ }
+}
+
+func (e *canProvideEnv) secret(plain string) Var {
+ e.t.Helper()
+ env, err := SealUnbound(e.pub, []byte(plain))
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ return Var{Kind: "secret", Envelope: env}
+}
+
+func reasonOf(t *testing.T, err error) string {
+ t.Helper()
+ var r *ConfigRefusal
+ if !errors.As(err, &r) {
+ t.Fatalf("err = %v, want a *ConfigRefusal", err)
+ }
+ return r.Reason
+}
+
+func TestCanProvideConfigNothingSealedNeedsNothing(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ // No manifest anywhere: a dev deployment. No key either.
+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com"); err != nil {
+ t.Fatalf("no manifest, no key: %v", err)
+ }
+ // A manifest with plain vars only has no sealed config.
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}})
+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com"); err != nil {
+ t.Fatalf("plain-only manifest, no key: %v", err)
+ }
+ // A manifest that was fully unset exists but is empty.
+ e.put(Key("bab", "other"), Manifest{App: "other", Vars: map[string]Var{}})
+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "other", "", "a.example.com"); err != nil {
+ t.Fatalf("empty manifest, no key: %v", err)
+ }
+}
+
+func TestCanProvideConfigSealedNeedsAKeyThatOpensIt(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"DB": e.secret("pw")}})
+
+ if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com"); err != nil {
+ t.Fatalf("right key: %v", err)
+ }
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "a.example.com")); got != ReasonNoKey {
+ t.Errorf("absent key reason = %q, want %q", got, ReasonNoKey)
+ }
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyUnparseable, "bab", "shop", "", "a.example.com")); got != ReasonKeyUnparseable {
+ t.Errorf("unparseable key reason = %q, want %q", got, ReasonKeyUnparseable)
+ }
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.other, KeyOK, "bab", "shop", "", "a.example.com")); got != ReasonCannotOpen {
+ t.Errorf("wrong key reason = %q, want %q", got, ReasonCannotOpen)
+ }
+}
+
+// The instance overlay and the bound environment are part of what the box
+// renders, so a secret only there still needs the key.
+func TestCanProvideConfigLooksAtEveryLayerTheBoxRenders(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}})
+ e.put(EnvironmentKey("bab", "shop", "staging"), Manifest{App: "shop", Vars: map[string]Var{"S": e.secret("x")}})
+ e.put(InstanceConfigKey("bab", "shop", "cust.example.com"), Manifest{App: "shop", Vars: map[string]Var{"T": e.secret("y")}})
+
+ // Unbound route, no overlay for this host: nothing sealed.
+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "plain.example.com"); err != nil {
+ t.Errorf("unbound plain host: %v", err)
+ }
+ // Bound to the environment: sealed.
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "staging", "plain.example.com")); got != ReasonNoKey {
+ t.Errorf("environment secret reason = %q", got)
+ }
+ // The overlay for cust.example.com only.
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "cust.example.com")); got != ReasonNoKey {
+ t.Errorf("overlay secret reason = %q", got)
+ }
+ if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "cust.example.com"); err != nil {
+ t.Errorf("overlay with the key: %v", err)
+ }
+}
+
+// SyncExec writes the WHOLE config file or none of it (RenderEnv fails if any
+// one decrypt fails), and instanceEnv accepts a missing file. So a manifest
+// that is only partly openable on a box with no rendered file would start the
+// instance with none of its sealed config: fresh admission needs every
+// envelope to open.
+func TestCanProvideConfigRefusesPartlyOpenableConfigOnFreshAdmission(t *testing.T) {
+ ctx := context.Background()
+ cases := map[string]func(e *canProvideEnv, foreign string) map[string]Var{
+ "one foreign-key envelope": func(e *canProvideEnv, f string) map[string]Var {
+ return map[string]Var{"A": {Kind: "secret", Envelope: f}, "C": e.secret("ok")}
+ },
+ "one corrupt envelope": func(e *canProvideEnv, f string) map[string]Var {
+ return map[string]Var{"B": {Kind: "secret", Envelope: "not-base64!!"}, "C": e.secret("ok")}
+ },
+ "an unwritable plain value": func(e *canProvideEnv, f string) map[string]Var {
+ return map[string]Var{"P": {Kind: "env", Value: "has \"quote"}, "C": e.secret("ok")}
+ },
+ "all foreign": func(e *canProvideEnv, f string) map[string]Var {
+ return map[string]Var{"A": {Kind: "secret", Envelope: f}}
+ },
+ }
+ for name, mk := range cases {
+ t.Run(name, func(t *testing.T) {
+ e := newCanProvideEnv(t)
+ foreign, err := SealUnbound(e.other.PublicKey(), []byte("z"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ vars := mk(e, foreign)
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: vars})
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com")); got != ReasonCannotOpen {
+ t.Errorf("reason = %q, want cannot_open", got)
+ }
+ })
+ }
+}
+
+// A v2 envelope is bound to its (account, app): the guard opens it under the
+// route's own binding, as the sync pass does.
+func TestCanProvideConfigOpensBoundEnvelopesUnderTheRoutesBinding(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ env, err := SealBound(e.pub, Binding{Account: "bab", App: "shop"}, []byte("pw"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"DB": {Kind: "secret", Envelope: env}}})
+ if err := CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "shop", "", "a.example.com"); err != nil {
+ t.Fatalf("bound envelope, own app: %v", err)
+ }
+ // The same bytes copied under another app do not open there.
+ e.put(Key("bab", "thief"), Manifest{App: "thief", Vars: map[string]Var{"DB": {Kind: "secret", Envelope: env}}})
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "thief", "", "a.example.com")); got != ReasonCannotOpen {
+ t.Errorf("copied bound envelope reason = %q", got)
+ }
+}
+
+// failingStore fails reads of the keys it is told to, and delegates the rest.
+type failingStore struct {
+ release.Store
+ failGet map[string]error
+}
+
+func (f *failingStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ if err, ok := f.failGet[key]; ok {
+ return nil, "", err
+ }
+ return f.Store.Get(ctx, key)
+}
+
+// "We could not tell" is never "there is no sealed config": a read error on
+// any layer refuses, with or without a key.
+func TestCanProvideConfigFailsClosedWhenItCannotRead(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ boom := errors.New("s3: 503 slow down")
+ e.put(EnvironmentKey("bab", "shop", "staging"), Manifest{App: "shop", Vars: map[string]Var{}})
+ for name, key := range map[string]string{
+ "default": Key("bab", "shop"),
+ "environment": EnvironmentKey("bab", "shop", "staging"),
+ "overlay": InstanceConfigKey("bab", "shop", "a.example.com"),
+ } {
+ t.Run(name, func(t *testing.T) {
+ fs := &failingStore{Store: e.s, failGet: map[string]error{key: boom}}
+ for _, k := range []struct {
+ priv *ecdh.PrivateKey
+ st KeyState
+ }{{e.priv, KeyOK}, {nil, KeyAbsent}} {
+ err := CanProvideConfig(ctx, fs, k.priv, k.st, "bab", "shop", "staging", "a.example.com")
+ if got := reasonOf(t, err); got != ReasonStoreError {
+ t.Errorf("reason = %q, want %q", got, ReasonStoreError)
+ }
+ if !errors.Is(err, boom) {
+ t.Errorf("err %v does not wrap the store error", err)
+ }
+ }
+ })
+ }
+}
+
+// A manifest the box cannot parse and a bound environment that is gone are
+// both "cannot tell".
+func TestCanProvideConfigFailsClosedOnUnreadableConfig(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ if _, err := e.s.Put(ctx, Key("bab", "garbled"), []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "garbled", "", "a.example.com")); got != ReasonStoreError {
+ t.Errorf("garbled manifest reason = %q", got)
+ }
+ e.put(Key("bab", "bound"), Manifest{App: "bound", Vars: map[string]Var{}})
+ if got := reasonOf(t, CanProvideConfig(ctx, e.s, e.priv, KeyOK, "bab", "bound", "gone", "a.example.com")); got != ReasonStoreError {
+ t.Errorf("missing bound environment reason = %q", got)
+ }
+}
+
+// A host that fails ValidInstanceHost cannot have an overlay (the sweeps read
+// that as "no overlay"); the guard must answer, not error on the host shape.
+func TestCanProvideConfigHostWithNoPossibleOverlay(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"A": {Kind: "env", Value: "1"}}})
+ if err := CanProvideConfig(ctx, e.s, nil, KeyAbsent, "bab", "shop", "", "Mixed.Case.example.com"); err != nil {
+ t.Fatalf("host without a possible overlay: %v", err)
+ }
+}
+
+// Nothing secret leaves in the error: not the envelope, not the plaintext.
+func TestConfigRefusalReasonIsAClosedWord(t *testing.T) {
+ e := newCanProvideEnv(t)
+ ctx := context.Background()
+ v := e.secret("hunter2")
+ e.put(Key("bab", "shop"), Manifest{App: "shop", Vars: map[string]Var{"DB": v}})
+ err := CanProvideConfig(ctx, e.s, e.other, KeyOK, "bab", "shop", "", "a.example.com")
+ var r *ConfigRefusal
+ if !errors.As(err, &r) {
+ t.Fatal(err)
+ }
+ switch r.Reason {
+ case ReasonNoKey, ReasonKeyUnparseable, ReasonCannotOpen, ReasonStoreError, ReasonNoSource:
+ default:
+ t.Errorf("reason %q is not one of the closed words", r.Reason)
+ }
+ if strings.Contains(err.Error(), v.Envelope) || strings.Contains(err.Error(), "hunter2") {
+ t.Errorf("error leaks sealed material: %v", err)
+ }
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 91300c1cc..9862b90c0 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -277,6 +277,16 @@ func RunAgent(args []string) {
}
}
+ // The config guard (amadan #213), armed before the activator and every
+ // reconcile loop that asks it: a box takes an instance's lease, or mints
+ // its record, only if it can provide that instance's config. Armed
+ // whether or not the activator is, because the mint gate is the reconcile
+ // pass's. A box with no deployment store reads it as "nothing sealed
+ // anywhere" and admits as before.
+ guard := newConfigGuard(os.Getenv, openDeployStore)
+ armConfigGuard(guard)
+ guard.Announce()
+
var act edge.Activator
var svc *activator.Service
if cfg != nil {
@@ -305,6 +315,8 @@ func RunAgent(args []string) {
// so the row is due-indexed by the next tick with nothing in
// between. reg satisfies schedule.TimerStore directly.
Timers: reg,
+ // Asked inside Acquire, before the lease (amadan #213).
+ ConfigGate: configGate(),
}
if logSink != nil { // a typed-nil in the interface would NOT be nil
eb.Logs = logSink
@@ -2154,6 +2166,9 @@ func provisionTickConfig(getenv func(string) string) provision.ReconcileConfig {
DataDir: dd,
Now: time.Now,
Inbound: inbound,
+ // The armed config guard, asked before a mint (amadan #213). Nil
+ // when none is armed (every test that does not arm one).
+ ConfigGate: configGate(),
}
// The root release public key, same source and same parser as adoptLoop's
// and domainsTickConfig's. It guards ONE decision in this pass: whether a
@@ -2829,6 +2844,10 @@ func statusLoop(reg *registry.Registry) {
// is what `carlos instances list` and `carlos routes` show as DB.
DBBytes: dbFileBytes,
}
+ // The hosts this box is refusing because it cannot provide their
+ // config (amadan #213): private Metrics only, nothing public.
+ p.ConfigRefusals, p.ConfigRefusalsTotal = configRefusalProbe()
+ p.ConfigGuard = configGuardProbe()
// A fleet box (CARLOS_FLEET set) never gets here as a contender: it
// self-reports only, never attempting the election at all — no
@@ -3216,7 +3235,7 @@ func openDeployStoreEnv(ctx context.Context, getenv func(string) string) (releas
// operator hunting for a bucket var that is deliberately absent.
return nil, err
}
- return nil, errors.New("neither CARLOS_DEPLOYMENT_BUCKET nor CARLOS_DEPLOYMENT_DIR is set")
+ return nil, errNoDeployStore
}
// openFleetDoorStore builds the fleetapi client a bucketless fleet box adopts
diff --git a/internal/host/configguard.go b/internal/host/configguard.go
new file mode 100644
index 000000000..8c4e8fee3
--- /dev/null
+++ b/internal/host/configguard.go
@@ -0,0 +1,371 @@
+package host
+
+import (
+ "context"
+ "crypto/ecdh"
+ "errors"
+ "log"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// The config guard (amadan #213). A box takes an instance's lease, or mints
+// its record, only if it can provide that instance's config.
+//
+// The failure it ends: a box with no CARLOS_SECRETS_KEY (or the wrong one)
+// still admitted and woke socket-backed shared-pool instances in its region.
+// Its config sync loops are off without the key, and ExecBackend.instanceEnv
+// treats a missing <host>.config.env as optional, so the instance started
+// without its secrets or store config. kpmg was down 1h54m on 2026-10-01
+// because the UK box had no key. The console cannot prove what reconcile will
+// admit, so the invariant lives here, at admission.
+//
+// One guard per process, built from CARLOS_SECRETS_KEY and the deploy store,
+// and asked through Gate from three places, each BEFORE the side effect it
+// guards: activator.ExecBackend.Acquire (before lease.Acquire),
+// provision.Reconcile and provision.ReconcileSidecars (before a mint). The
+// decision itself is appconfig.CanProvideConfig; this type adds what that
+// pure function cannot: the box's key, the store, a short refusal cache, the
+// once-per-host log and the list the status snapshot publishes.
+
+const (
+ // configGuardRefuseTTL bounds how long a refusal is answered from memory.
+ // A request storm on a refused host would otherwise cost a set of bucket
+ // reads per request, and the reconcile pass asks about every unminted
+ // record every tick. It is also the retry interval after a store error:
+ // the box can tell again within this long of the store coming back. An
+ // admission is never cached: wakes are rare, and the answer must be
+ // current when it is taken.
+ configGuardRefuseTTL = 30 * time.Second
+ // configGuardListCap bounds the refusal list in the status snapshot; the
+ // total is published beside it so a capped list reads as long.
+ configGuardListCap = 100
+ // configGuardRefusalMaxAge is how long a host stays in the published list
+ // without being asked about again. A wake-time refusal is only re-asked
+ // when someone requests the host, so this is what makes the list mean
+ // "currently refused" rather than "ever refused since boot".
+ configGuardRefusalMaxAge = 24 * time.Hour
+)
+
+// errNoDeployStore is openDeployStoreEnv's answer for a box with no
+// deployment store configured at all: no bucket, no dir, no fleet identity.
+//
+// That is NOT proof that nothing is sealed. Config lives in the deployment
+// store, but CARLOS_HIBERNATE_BUCKET enables leases independently of it, so a
+// box can wake instances whose sealed config sits in a store it was never told
+// about. The guard therefore refuses (no_config_source) unless the operator
+// has said, with CARLOS_NO_SEALED_CONFIG=1 (or "true"), that this deployment
+// has no sealed config: the config-free development setup. The flag only
+// matters when there is no store; with one, config is read as usual.
+var errNoDeployStore = errors.New("neither CARLOS_DEPLOYMENT_BUCKET nor CARLOS_DEPLOYMENT_DIR is set")
+
+type guardRefusal struct {
+ reason string
+ since time.Time
+ last time.Time
+ count int
+}
+
+type guardVerdict struct {
+ err error
+ until time.Time
+}
+
+type configGuard struct {
+ priv *ecdh.PrivateKey
+ keyState appconfig.KeyState
+ openStore func(context.Context) (release.Store, error)
+ now func() time.Time
+ // noSealedConfig is the operator's explicit "this deployment has no
+ // sealed config" (CARLOS_NO_SEALED_CONFIG), honoured only when there is
+ // no deployment store at all.
+ noSealedConfig bool
+
+ mu sync.Mutex
+ bypassed map[string]bool // hosts already logged as admitted by the bypass
+ store release.Store
+ verdicts map[string]guardVerdict // host -> cached refusal
+ refused map[string]*guardRefusal
+}
+
+// newConfigGuard reads CARLOS_SECRETS_KEY once, the way the config loops do.
+// A key that does not parse is logged here, once, at arming; per-host lines
+// say only that the host was refused.
+func newConfigGuard(getenv func(string) string, openStore func(context.Context) (release.Store, error)) *configGuard {
+ g := &configGuard{
+ openStore: openStore,
+ now: time.Now,
+ verdicts: map[string]guardVerdict{},
+ refused: map[string]*guardRefusal{},
+ }
+ switch strings.ToLower(strings.TrimSpace(getenv("CARLOS_NO_SEALED_CONFIG"))) {
+ case "1", "true":
+ g.noSealedConfig = true
+ }
+ raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY"))
+ switch {
+ case raw == "":
+ g.keyState = appconfig.KeyAbsent
+ default:
+ priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY")
+ if err != nil {
+ log.Printf("agent: config guard: %v", err)
+ g.keyState = appconfig.KeyUnparseable
+ } else {
+ g.priv, g.keyState = priv, appconfig.KeyOK
+ }
+ }
+ return g
+}
+
+// needsBoxConfig reports whether a route is one this box would spawn with a
+// config.env of its own: a socket-backed instance or a sidecar worker that
+// belongs to an app. Static sites, addr-backed routes, Backing=unit routes
+// (whose config the root adopt pass delivers), platform routes and app-less
+// rows take no part: the guard must not touch them.
+//
+// The exclusion is Backing and nothing else, because Backing is what
+// SplitBackend dispatches on. A socket-backed hibernating instance may carry a
+// Unit name with an empty Backing (the record validator and `carlos route
+// -unit` allow it) and still reaches ExecBackend.Acquire.
+func needsBoxConfig(rt registry.Route) bool {
+ if rt.App == "" || rt.Platform || rt.Backing == registry.BackingUnit {
+ return false
+ }
+ switch {
+ case rt.IsSidecar():
+ return true
+ case rt.Kind == "instance":
+ return rt.Socket != ""
+ }
+ return false
+}
+
+// Gate is the seam both ExecBackend.ConfigGate and ReconcileConfig.ConfigGate
+// are wired to. nil means this box can provide rt's config (or none is
+// needed); an error means refuse, and is an *appconfig.ConfigRefusal.
+func (g *configGuard) Gate(ctx context.Context, rt registry.Route) error {
+ if !needsBoxConfig(rt) {
+ return nil
+ }
+ now := g.now()
+ if err := g.cached(rt.Host, now); err != nil {
+ g.note(rt.Host, err, now)
+ return err
+ }
+ err := g.decide(ctx, rt)
+ if err != nil {
+ g.mu.Lock()
+ g.verdicts[rt.Host] = guardVerdict{err: err, until: now.Add(configGuardRefuseTTL)}
+ g.mu.Unlock()
+ g.note(rt.Host, err, now)
+ return err
+ }
+ g.clear(rt.Host)
+ return nil
+}
+
+func (g *configGuard) cached(host string, now time.Time) error {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ v, ok := g.verdicts[host]
+ if !ok {
+ return nil
+ }
+ if !now.Before(v.until) {
+ delete(g.verdicts, host)
+ return nil
+ }
+ return v.err
+}
+
+func (g *configGuard) decide(ctx context.Context, rt registry.Route) error {
+ s, err := g.deployStore(ctx)
+ if errors.Is(err, errNoDeployStore) {
+ if g.noSealedConfig {
+ // The operator said this deployment has none. Said loudly, once
+ // per host: this is the one path that admits without looking.
+ g.noteBypass(rt.Host)
+ return nil
+ }
+ return &appconfig.ConfigRefusal{Reason: appconfig.ReasonNoSource, Err: err}
+ }
+ if err != nil {
+ return &appconfig.ConfigRefusal{Reason: appconfig.ReasonStoreError, Err: err}
+ }
+ return appconfig.CanProvideConfig(ctx, s, g.priv, g.keyState, release.RouteAccount(rt), rt.App, rt.Environment, rt.Host)
+}
+
+// configGuardBypassStatus is the private status word published while the
+// CARLOS_NO_SEALED_CONFIG bypass is in effect.
+const configGuardBypassStatus = "bypassed (CARLOS_NO_SEALED_CONFIG)"
+
+// noteBypass logs, once per host, that a placed instance was admitted without
+// its sealed config being checked.
+func (g *configGuard) noteBypass(host string) {
+ g.mu.Lock()
+ if g.bypassed == nil {
+ g.bypassed = map[string]bool{}
+ }
+ seen := g.bypassed[host]
+ g.bypassed[host] = true
+ g.mu.Unlock()
+ if !seen {
+ log.Printf("agent: config guard: BYPASS: admitting %s WITHOUT checking its sealed config, because CARLOS_NO_SEALED_CONFIG is set and this box has no deployment store. Never set it on a box that serves tenants", host)
+ }
+}
+
+// Bypassed reports whether the CARLOS_NO_SEALED_CONFIG bypass is in effect:
+// the flag is set AND the box has no deployment store configured. A flag set
+// beside a store changes nothing, so it does not count.
+func (g *configGuard) Bypassed() bool {
+ if !g.noSealedConfig {
+ return false
+ }
+ _, err := g.deployStore(context.Background())
+ return errors.Is(err, errNoDeployStore)
+}
+
+// Status is the private status-block word for the guard's mode: the bypass
+// words while it is in effect, "" otherwise.
+func (g *configGuard) Status() string {
+ if g.Bypassed() {
+ return configGuardBypassStatus
+ }
+ return ""
+}
+
+// Announce logs the guard's mode once at startup. The bypass is the only mode
+// worth shouting about, and it is shouted whether or not it is in effect, so a
+// flag copied into the wrong host.env is visible either way.
+func (g *configGuard) Announce() {
+ switch {
+ case g.Bypassed():
+ log.Printf("agent: config guard: BYPASSED. CARLOS_NO_SEALED_CONFIG is set and no deployment store is configured, so this box admits placed exec instances without checking their sealed config. Intended for config-free development only; it is listed in this box's status snapshot")
+ case g.noSealedConfig:
+ log.Printf("agent: config guard: CARLOS_NO_SEALED_CONFIG is set but a deployment store is configured, so it is ignored and sealed config is checked as usual")
+ }
+}
+
+// deployStore opens the store on first use and keeps it. A failure is not
+// kept, so the next ask (after the refusal TTL) tries again.
+func (g *configGuard) deployStore(ctx context.Context) (release.Store, error) {
+ g.mu.Lock()
+ s := g.store
+ g.mu.Unlock()
+ if s != nil {
+ return s, nil
+ }
+ s, err := g.openStore(ctx)
+ if err != nil {
+ return nil, err
+ }
+ g.mu.Lock()
+ g.store = s
+ g.mu.Unlock()
+ return s, nil
+}
+
+// note records a refusal for the snapshot and logs it: once per host, and
+// again only when the reason changes. The detail in the log is the operator's;
+// the snapshot carries the closed reason word alone.
+func (g *configGuard) note(host string, err error, now time.Time) {
+ reason := appconfig.ReasonStoreError
+ var r *appconfig.ConfigRefusal
+ if errors.As(err, &r) {
+ reason = r.Reason
+ }
+ g.mu.Lock()
+ e := g.refused[host]
+ fresh := e == nil || e.reason != reason
+ if e == nil {
+ e = &guardRefusal{since: now}
+ g.refused[host] = e
+ }
+ e.reason, e.last = reason, now
+ e.count++
+ g.mu.Unlock()
+ if fresh {
+ log.Printf("agent: config guard: refusing %s: %v. A box that cannot open an instance's sealed config does not take its lease or mint its record, so a keyed sibling can; it is listed in this box's status snapshot", host, err)
+ }
+}
+
+// clear forgets a host that now passes, saying so once if it was refused.
+func (g *configGuard) clear(host string) {
+ g.mu.Lock()
+ _, was := g.refused[host]
+ delete(g.refused, host)
+ delete(g.verdicts, host)
+ g.mu.Unlock()
+ if was {
+ log.Printf("agent: config guard: %s can be provided from this box again", host)
+ }
+}
+
+// Refusals is what the status snapshot publishes: the currently refused
+// hosts, newest first and capped, plus how many there are in all. A host not
+// asked about for configGuardRefusalMaxAge is dropped.
+func (g *configGuard) Refusals() ([]status.ConfigRefusal, int) {
+ now := g.now()
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ var out []status.ConfigRefusal
+ for host, e := range g.refused {
+ if now.Sub(e.last) > configGuardRefusalMaxAge {
+ delete(g.refused, host)
+ continue
+ }
+ out = append(out, status.ConfigRefusal{Host: host, Reason: e.reason, Since: e.since, Last: e.last, Count: e.count})
+ }
+ total := len(out)
+ sort.Slice(out, func(i, j int) bool {
+ if !out[i].Last.Equal(out[j].Last) {
+ return out[i].Last.After(out[j].Last)
+ }
+ return out[i].Host < out[j].Host
+ })
+ if len(out) > configGuardListCap {
+ out = out[:configGuardListCap]
+ }
+ return out, total
+}
+
+// armedConfigGuard is the process's one guard, set by RunAgent before any
+// loop that asks it starts. Nil is "not armed": every consumer treats that as
+// today's behaviour, so tests and callers that never arm one are unaffected.
+var armedConfigGuard atomic.Pointer[configGuard]
+
+func armConfigGuard(g *configGuard) { armedConfigGuard.Store(g) }
+
+// configGate is the armed guard's Gate, or nil when none is armed.
+func configGate() func(context.Context, registry.Route) error {
+ if g := armedConfigGuard.Load(); g != nil {
+ return g.Gate
+ }
+ return nil
+}
+
+// configGuardProbe feeds status.Probes.ConfigGuard: "" unless the bypass is
+// in effect.
+func configGuardProbe() string {
+ if g := armedConfigGuard.Load(); g != nil {
+ return g.Status()
+ }
+ return ""
+}
+
+// configRefusalProbe feeds status.Probes.ConfigRefusals.
+func configRefusalProbe() ([]status.ConfigRefusal, int) {
+ if g := armedConfigGuard.Load(); g != nil {
+ return g.Refusals()
+ }
+ return nil, 0
+}
diff --git a/internal/host/configguard_test.go b/internal/host/configguard_test.go
new file mode 100644
index 000000000..828003eaa
--- /dev/null
+++ b/internal/host/configguard_test.go
@@ -0,0 +1,615 @@
+package host
+
+import (
+ "bytes"
+ "context"
+ "crypto/ecdh"
+ "crypto/rand"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// Issue #213: the box-side guard. A box takes an instance's lease, or mints
+// its record, only if it can provide that instance's config. These tests
+// drive the guard through its Gate seam, the one function the activator and
+// the reconcile pass both call.
+
+type guardEnv struct {
+ t *testing.T
+ store *guardStore
+ priv *ecdh.PrivateKey
+ other *ecdh.PrivateKey
+ clock *guardClock
+ // dataDir holds leftover rendered files the tests plant to show the guard
+ // ignores them.
+ dataDir string
+}
+
+type guardClock struct {
+ mu sync.Mutex
+ t time.Time
+}
+
+func (c *guardClock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
+func (c *guardClock) advance(d time.Duration) {
+ c.mu.Lock()
+ c.t = c.t.Add(d)
+ c.mu.Unlock()
+}
+
+// guardStore counts Gets and can be made to fail every Get.
+type guardStore struct {
+ release.Store
+ mu sync.Mutex
+ gets int
+ failGet error
+}
+
+func (s *guardStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ s.mu.Lock()
+ s.gets++
+ err := s.failGet
+ s.mu.Unlock()
+ if err != nil {
+ return nil, "", err
+ }
+ return s.Store.Get(ctx, key)
+}
+
+func (s *guardStore) getCount() int { s.mu.Lock(); defer s.mu.Unlock(); return s.gets }
+func (s *guardStore) setFail(err error) {
+ s.mu.Lock()
+ s.failGet = err
+ s.mu.Unlock()
+}
+
+func newGuardEnv(t *testing.T) *guardEnv {
+ t.Helper()
+ priv, err := ecdh.P256().GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ other, err := ecdh.P256().GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return &guardEnv{
+ t: t,
+ store: &guardStore{Store: release.NewDirStore(t.TempDir())},
+ priv: priv, other: other,
+ clock: &guardClock{t: time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC)},
+ dataDir: t.TempDir(),
+ }
+}
+
+func (e *guardEnv) seal(plain string) appconfig.Var {
+ e.t.Helper()
+ env, err := appconfig.SealUnbound(e.priv.PublicKey(), []byte(plain))
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ return appconfig.Var{Kind: "secret", Envelope: env}
+}
+
+func (e *guardEnv) putConfig(acct, app string, vars map[string]appconfig.Var) {
+ e.t.Helper()
+ body, err := json.Marshal(appconfig.Manifest{Schema: 1, App: app, Vars: vars})
+ if err != nil {
+ e.t.Fatal(err)
+ }
+ if _, err := e.store.Store.Put(context.Background(), appconfig.Key(acct, app), body, ""); err != nil {
+ e.t.Fatal(err)
+ }
+}
+
+// guard builds the guard with the given key in the environment ("" = unset)
+// and this env's store.
+func (e *guardEnv) guard(keyB64 string) *configGuard {
+ g := newConfigGuard(func(k string) string {
+ switch k {
+ case "CARLOS_SECRETS_KEY":
+ return keyB64
+ }
+ return ""
+ }, func(context.Context) (release.Store, error) { return e.store, nil })
+ g.now = e.clock.now
+ return g
+}
+
+func keyB64(k *ecdh.PrivateKey) string { return base64.StdEncoding.EncodeToString(k.Bytes()) }
+
+func guardRoute(host string) registry.Route {
+ return registry.Route{Host: host, Kind: "instance", App: "shop", Account: "bab",
+ Socket: "/run/carlos/" + host + ".sock", Hibernate: true, Provisioned: true}
+}
+
+func reasonOfGate(t *testing.T, err error) string {
+ t.Helper()
+ var r *appconfig.ConfigRefusal
+ if !errors.As(err, &r) {
+ t.Fatalf("err = %v, want *appconfig.ConfigRefusal", err)
+ }
+ return r.Reason
+}
+
+func guardCaptureLog(t *testing.T) *bytes.Buffer {
+ t.Helper()
+ var buf bytes.Buffer
+ old := log.Writer()
+ log.SetOutput(&buf)
+ t.Cleanup(func() { log.SetOutput(old) })
+ return &buf
+}
+
+func TestNeedsConfigIsTheShapeTheActivatorSpawns(t *testing.T) {
+ cases := []struct {
+ name string
+ rt registry.Route
+ want bool
+ }{
+ {"socket instance", guardRoute("a.example.com"), true},
+ {"non-hibernating socket instance", registry.Route{Host: "a", Kind: "instance", App: "shop", Socket: "/s"}, true},
+ {"sidecar worker", registry.Route{Host: "a", Kind: "sidecar", App: "shop", Hibernate: true}, true},
+ {"unit-backed by column", registry.Route{Host: "a", Kind: "instance", App: "shop", Socket: "/s", Backing: registry.BackingUnit}, false},
+ {"unit-backed sidecar", registry.Route{Host: "a", Kind: "sidecar", App: "shop", Backing: registry.BackingUnit}, false},
+ // Dispatch is on Backing alone (SplitBackend.pick): a socket-backed
+ // instance that merely NAMES a unit still reaches ExecBackend.Acquire.
+ {"unit-named, socket-backed, empty Backing", registry.Route{Host: "a", Kind: "instance", App: "shop", Socket: "/s", Unit: "x.service", Hibernate: true}, true},
+ {"unit-named, addr-backed", registry.Route{Host: "a", Kind: "instance", App: "shop", Addr: "127.0.0.1:1", Unit: "x.service"}, false},
+ {"addr-backed", registry.Route{Host: "a", Kind: "instance", App: "shop", Addr: "127.0.0.1:9"}, false},
+ {"static", registry.Route{Host: "a", Kind: "static", App: "shop"}, false},
+ {"service", registry.Route{Host: "a", Kind: "service", App: "shop", Addr: "127.0.0.1:9"}, false},
+ {"platform", registry.Route{Host: "a", Kind: "instance", App: "console", Socket: "/s", Platform: true}, false},
+ {"no app", registry.Route{Host: "a", Kind: "instance", Socket: "/s"}, false},
+ }
+ for _, c := range cases {
+ if got := needsBoxConfig(c.rt); got != c.want {
+ t.Errorf("%s: needsBoxConfig = %v, want %v", c.name, got, c.want)
+ }
+ }
+}
+
+// A deployment with no sealed config anywhere (dev) is unaffected: no key, no
+// manifest, no refusal, nothing published.
+func TestGuardNoSealedConfigAdmitsKeylessBox(t *testing.T) {
+ e := newGuardEnv(t)
+ g := e.guard("")
+ if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil {
+ t.Fatalf("keyless box, nothing sealed: %v", err)
+ }
+ if list, total := g.Refusals(); len(list) != 0 || total != 0 {
+ t.Errorf("refusals = %v (%d), want none", list, total)
+ }
+}
+
+func TestGuardKeylessBoxRefusesSealedInstanceAndPublishesIt(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ logs := guardCaptureLog(t)
+ g := e.guard("")
+
+ err := g.Gate(context.Background(), guardRoute("kpmg.example.com"))
+ if got := reasonOfGate(t, err); got != appconfig.ReasonNoKey {
+ t.Fatalf("reason = %q, want no_key", got)
+ }
+ list, total := g.Refusals()
+ if total != 1 || len(list) != 1 {
+ t.Fatalf("refusals = %v (%d)", list, total)
+ }
+ r := list[0]
+ if r.Host != "kpmg.example.com" || r.Reason != "no_key" || r.Count != 1 || !r.Since.Equal(e.clock.now()) {
+ t.Errorf("refusal = %+v", r)
+ }
+ if n := strings.Count(logs.String(), "kpmg.example.com"); n != 1 {
+ t.Errorf("log mentions the host %d times, want once:\n%s", n, logs.String())
+ }
+}
+
+func TestGuardKeyedBoxTakesIt(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard(keyB64(e.priv))
+ if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil {
+ t.Fatalf("keyed box: %v", err)
+ }
+ if _, total := g.Refusals(); total != 0 {
+ t.Errorf("a keyed box publishes %d refusals", total)
+ }
+}
+
+func TestGuardWrongAndBrokenKeysRefuse(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ for name, tc := range map[string]struct {
+ key string
+ want string
+ }{
+ "wrong key": {keyB64(e.other), appconfig.ReasonCannotOpen},
+ "not base64": {"%%%not-a-key", appconfig.ReasonKeyUnparseable},
+ "short": {base64.StdEncoding.EncodeToString([]byte("abc")), appconfig.ReasonKeyUnparseable},
+ "blank": {" ", appconfig.ReasonNoKey},
+ } {
+ t.Run(name, func(t *testing.T) {
+ g := e.guard(tc.key)
+ err := g.Gate(context.Background(), guardRoute("a.example.com"))
+ if got := reasonOfGate(t, err); got != tc.want {
+ t.Errorf("reason = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
+// Fail closed: a store the box cannot read is not "no sealed config".
+func TestGuardStoreErrorRefusesAndRetriesAfterTheTTL(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard(keyB64(e.priv))
+ ctx := context.Background()
+
+ e.store.setFail(errors.New("s3: 503"))
+ err := g.Gate(ctx, guardRoute("a.example.com"))
+ if got := reasonOfGate(t, err); got != appconfig.ReasonStoreError {
+ t.Fatalf("reason = %q, want store_error", got)
+ }
+
+ // Within the TTL a repeat is answered from the verdict, without another
+ // bucket read, and still refuses even though the store has recovered.
+ e.store.setFail(nil)
+ before := e.store.getCount()
+ if err := g.Gate(ctx, guardRoute("a.example.com")); err == nil {
+ t.Fatal("refusal forgotten inside the TTL")
+ }
+ if got := e.store.getCount(); got != before {
+ t.Errorf("a cached refusal read the store (%d -> %d)", before, got)
+ }
+ if list, _ := g.Refusals(); len(list) != 1 || list[0].Count != 2 {
+ t.Errorf("refusals = %+v, want one with count 2", list)
+ }
+
+ // After the TTL it asks again and, the store being back, admits and
+ // stops publishing the refusal.
+ e.clock.advance(configGuardRefuseTTL + time.Second)
+ if err := g.Gate(ctx, guardRoute("a.example.com")); err != nil {
+ t.Fatalf("after the TTL with the store back: %v", err)
+ }
+ if _, total := g.Refusals(); total != 0 {
+ t.Errorf("a recovered host still published (%d)", total)
+ }
+}
+
+func TestGuardLogsOncePerHostAndAgainWhenTheReasonChanges(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard("")
+ logs := guardCaptureLog(t)
+ ctx := context.Background()
+
+ for i := 0; i < 5; i++ {
+ _ = g.Gate(ctx, guardRoute("a.example.com"))
+ e.clock.advance(configGuardRefuseTTL + time.Second) // defeat the verdict cache
+ }
+ if n := strings.Count(logs.String(), "a.example.com"); n != 1 {
+ t.Fatalf("5 refusals logged %d lines, want 1:\n%s", n, logs.String())
+ }
+ _ = g.Gate(ctx, guardRoute("b.example.com"))
+ if n := strings.Count(logs.String(), "b.example.com"); n != 1 {
+ t.Errorf("a second host logged %d lines, want 1", n)
+ }
+ // The reason changes (a store outage now hides the answer): said again.
+ e.store.setFail(errors.New("s3: down"))
+ _ = g.Gate(ctx, guardRoute("a.example.com"))
+ if n := strings.Count(logs.String(), "a.example.com"); n != 2 {
+ t.Errorf("a changed reason logged %d lines for the host, want 2:\n%s", n, logs.String())
+ }
+}
+
+// Routes the box does not spawn never reach the store.
+func TestGuardIgnoresRoutesItDoesNotSpawn(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard("")
+ for _, rt := range []registry.Route{
+ {Host: "s", Kind: "static", App: "shop"},
+ {Host: "u", Kind: "instance", App: "shop", Socket: "/s", Backing: registry.BackingUnit},
+ {Host: "p", Kind: "instance", App: "shop", Socket: "/s", Platform: true},
+ } {
+ if err := g.Gate(context.Background(), rt); err != nil {
+ t.Errorf("%s: %v", rt.Host, err)
+ }
+ }
+ if e.store.getCount() != 0 {
+ t.Errorf("read the store %d times for routes it does not spawn", e.store.getCount())
+ }
+}
+
+// Config lives in the deployment store, and CARLOS_HIBERNATE_BUCKET enables
+// leases without one. A box with no store configured therefore cannot know
+// that nothing is sealed, and refuses placed exec instances, unless the
+// operator has said explicitly that this deployment has no sealed config.
+func TestGuardNoDeployStoreFailsClosedUnlessOptedOut(t *testing.T) {
+ e := newGuardEnv(t)
+ build := func(env map[string]string) *configGuard {
+ g := newConfigGuard(func(k string) string { return env[k] },
+ func(context.Context) (release.Store, error) { return nil, errNoDeployStore })
+ g.now = e.clock.now
+ return g
+ }
+
+ g := build(nil)
+ err := g.Gate(context.Background(), guardRoute("a.example.com"))
+ if got := reasonOfGate(t, err); got != appconfig.ReasonNoSource {
+ t.Fatalf("no store, no opt-out: reason = %q, want no_config_source", got)
+ }
+ if list, _ := g.Refusals(); len(list) != 1 || list[0].Reason != "no_config_source" {
+ t.Errorf("refusals = %+v", list)
+ }
+
+ for _, v := range []string{"1", "true", " TRUE "} {
+ g := build(map[string]string{"CARLOS_NO_SEALED_CONFIG": v})
+ if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil {
+ t.Errorf("CARLOS_NO_SEALED_CONFIG=%q: %v", v, err)
+ }
+ }
+ for _, v := range []string{"", "0", "no", "yes please"} {
+ g := build(map[string]string{"CARLOS_NO_SEALED_CONFIG": v})
+ if err := g.Gate(context.Background(), guardRoute("a.example.com")); err == nil {
+ t.Errorf("CARLOS_NO_SEALED_CONFIG=%q opted out", v)
+ }
+ }
+
+ // The opt-out is only about having NO store. With one, config is read.
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ withStore := newConfigGuard(func(k string) string {
+ if k == "CARLOS_NO_SEALED_CONFIG" {
+ return "1"
+ }
+ return ""
+ }, func(context.Context) (release.Store, error) { return e.store, nil })
+ withStore.now = e.clock.now
+ if got := reasonOfGate(t, withStore.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonNoKey {
+ t.Errorf("opt-out with a real store and sealed config: reason = %q, want no_key", got)
+ }
+
+ // A store that is configured but will not open is "cannot tell".
+ broken := newConfigGuard(func(string) string { return "" },
+ func(context.Context) (release.Store, error) { return nil, fmt.Errorf("aws: no credentials") })
+ broken.now = e.clock.now
+ if got := reasonOfGate(t, broken.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError {
+ t.Errorf("unopenable store reason = %q, want store_error", got)
+ }
+}
+
+// A route whose Unit is named but whose Backing is empty is exec-managed and
+// gated like any other.
+func TestGuardGatesAUnitNamedExecInstance(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard("")
+ rt := guardRoute("a.example.com")
+ rt.Unit = "x.service"
+ if got := reasonOfGate(t, g.Gate(context.Background(), rt)); got != appconfig.ReasonNoKey {
+ t.Errorf("reason = %q, want no_key", got)
+ }
+ rt.Backing = registry.BackingUnit
+ if err := g.Gate(context.Background(), rt); err != nil {
+ t.Errorf("a true unit-backed route was gated: %v", err)
+ }
+}
+
+// There is no last-known-good fallback. A rendered <host>.config.env left on
+// disk proves nothing about its values: names alone pass for "A=old", for empty
+// values, or for credentials rendered for another environment. A fresh start
+// needs the whole merged config to render now.
+func TestGuardIgnoresLeftoverRenderedFiles(t *testing.T) {
+ foreign := func(e *guardEnv) string {
+ v, err := appconfig.SealUnbound(e.other.PublicKey(), []byte("z"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return v
+ }
+ cases := map[string]struct {
+ file string
+ env string // the route's bound environment
+ }{
+ "wrong values": {"A=old\nC=wrong\n", ""},
+ "empty values": {"A=\nC=\n", ""},
+ "every name present": {"# header\nA=x\nC=y\nP=1\n", ""},
+ "environment rebound": {"A=prod-secret\nC=prod-secret\n", "staging"},
+ }
+ for name, tc := range cases {
+ t.Run(name, func(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{
+ "A": {Kind: "secret", Envelope: foreign(e)}, // sealed for another key
+ "C": e.seal("ok"),
+ "P": {Kind: "env", Value: "1"},
+ })
+ if tc.env != "" {
+ body, _ := json.Marshal(appconfig.Manifest{Schema: 1, App: "shop", Vars: map[string]appconfig.Var{"S": e.seal("staging")}})
+ if _, err := e.store.Store.Put(context.Background(), appconfig.EnvironmentKey("bab", "shop", tc.env), body, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ g := e.guard(keyB64(e.priv))
+ rt := guardRoute("a.example.com")
+ rt.Environment = tc.env
+ if err := os.WriteFile(appconfig.ConfigEnvPath(e.dataDir, rt), []byte(tc.file), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if got := reasonOfGate(t, g.Gate(context.Background(), rt)); got != appconfig.ReasonCannotOpen {
+ t.Fatalf("reason = %q, want cannot_open: a leftover file must not admit a mixed-key config", got)
+ }
+ })
+ }
+}
+
+// The bypass is loud: logged at startup and once per admitted host, and
+// published in the private status block, and only while it is in effect.
+func TestGuardBypassIsLoud(t *testing.T) {
+ e := newGuardEnv(t)
+ noStore := func(context.Context) (release.Store, error) { return nil, errNoDeployStore }
+ env := func(k string) string {
+ if k == "CARLOS_NO_SEALED_CONFIG" {
+ return "1"
+ }
+ return ""
+ }
+ g := newConfigGuard(env, noStore)
+ g.now = e.clock.now
+ logs := guardCaptureLog(t)
+
+ if got := g.Status(); got != "bypassed (CARLOS_NO_SEALED_CONFIG)" {
+ t.Errorf("status = %q", got)
+ }
+ g.Announce()
+ if !strings.Contains(logs.String(), "BYPASSED") {
+ t.Errorf("startup line missing:\n%s", logs.String())
+ }
+ for i := 0; i < 3; i++ {
+ if err := g.Gate(context.Background(), guardRoute("a.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ }
+ _ = g.Gate(context.Background(), guardRoute("b.example.com"))
+ if n := strings.Count(logs.String(), "BYPASS: admitting a.example.com"); n != 1 {
+ t.Errorf("a.example.com logged %d times, want once:\n%s", n, logs.String())
+ }
+ if n := strings.Count(logs.String(), "BYPASS: admitting b.example.com"); n != 1 {
+ t.Errorf("b.example.com logged %d times, want once", n)
+ }
+
+ // The flag beside a real store is ignored, and says so, and is not
+ // reported as a bypass.
+ withStore := newConfigGuard(env, func(context.Context) (release.Store, error) { return e.store, nil })
+ logs2 := guardCaptureLog(t)
+ if got := withStore.Status(); got != "" {
+ t.Errorf("flag with a store reported %q", got)
+ }
+ withStore.Announce()
+ if !strings.Contains(logs2.String(), "ignored") || strings.Contains(logs2.String(), "BYPASSED") {
+ t.Errorf("flag-with-store line wrong:\n%s", logs2.String())
+ }
+
+ // No flag: quiet, and nothing published.
+ plain := newConfigGuard(func(string) string { return "" }, noStore)
+ logs3 := guardCaptureLog(t)
+ plain.Announce()
+ if plain.Status() != "" || logs3.Len() != 0 {
+ t.Errorf("a box without the flag said something: %q / %q", plain.Status(), logs3.String())
+ }
+}
+
+// The opt-out never covers a store that is configured and failing: the box
+// cannot tell, and refuses, flag or not.
+func TestGuardOptOutDoesNotCoverAConfiguredStoreError(t *testing.T) {
+ e := newGuardEnv(t)
+ env := func(k string) string {
+ if k == "CARLOS_NO_SEALED_CONFIG" {
+ return "1"
+ }
+ return ""
+ }
+ // Opening fails.
+ broken := newConfigGuard(env, func(context.Context) (release.Store, error) { return nil, fmt.Errorf("aws: no credentials") })
+ broken.now = e.clock.now
+ if got := reasonOfGate(t, broken.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError {
+ t.Errorf("open failure with the opt-out: reason = %q, want store_error", got)
+ }
+ if broken.Status() != "" {
+ t.Errorf("an open failure was reported as a bypass: %q", broken.Status())
+ }
+ // Opens, then reads fail.
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ e.store.setFail(errors.New("s3: 503"))
+ flaky := newConfigGuard(env, func(context.Context) (release.Store, error) { return e.store, nil })
+ flaky.now = e.clock.now
+ if got := reasonOfGate(t, flaky.Gate(context.Background(), guardRoute("a.example.com"))); got != appconfig.ReasonStoreError {
+ t.Errorf("read failure with the opt-out: reason = %q, want store_error", got)
+ }
+}
+
+func TestGuardRefusalsAreSortedCappedAndPruned(t *testing.T) {
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard("")
+ ctx := context.Background()
+ n := configGuardListCap + 5
+ for i := 0; i < n; i++ {
+ _ = g.Gate(ctx, guardRoute(fmt.Sprintf("h%03d.example.com", i)))
+ e.clock.advance(time.Second)
+ }
+ list, total := g.Refusals()
+ if total != n || len(list) != configGuardListCap {
+ t.Fatalf("list %d total %d, want %d and %d", len(list), total, configGuardListCap, n)
+ }
+ // Newest first, so the cap drops the oldest.
+ if list[0].Host != fmt.Sprintf("h%03d.example.com", n-1) {
+ t.Errorf("first = %s, want the newest", list[0].Host)
+ }
+
+ // A refusal nobody has asked about for a day is no longer current.
+ e.clock.advance(configGuardRefusalMaxAge + time.Hour)
+ if list, total := g.Refusals(); len(list) != 0 || total != 0 {
+ t.Errorf("stale refusals survived: %d/%d", len(list), total)
+ }
+}
+
+// Nothing secret in what is published: reason words and hosts only.
+func TestGuardPublishesNoSecretMaterial(t *testing.T) {
+ e := newGuardEnv(t)
+ v := e.seal("hunter2")
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": v})
+ g := e.guard(keyB64(e.other))
+ logs := guardCaptureLog(t)
+ _ = g.Gate(context.Background(), guardRoute("a.example.com"))
+ list, _ := g.Refusals()
+ raw, _ := json.Marshal(list)
+ for _, s := range []string{string(raw), logs.String()} {
+ for _, bad := range []string{v.Envelope, "hunter2", keyB64(e.other), keyB64(e.priv)} {
+ if strings.Contains(s, bad) {
+ t.Errorf("secret material %q in %s", bad, s)
+ }
+ }
+ }
+}
+
+// The agent wires one armed guard into the reconcile config and the status
+// probe, and a box with none armed behaves exactly as before.
+func TestArmedGuardReachesReconcileConfigAndStatus(t *testing.T) {
+ armConfigGuard(nil)
+ t.Cleanup(func() { armConfigGuard(nil) })
+ if cfg := provisionTickConfig(func(string) string { return "" }); cfg.ConfigGate != nil {
+ t.Error("no armed guard, but the reconcile config carries a gate")
+ }
+ if list, total := configRefusalProbe(); list != nil || total != 0 {
+ t.Errorf("no armed guard, but the probe reports %v/%d", list, total)
+ }
+
+ e := newGuardEnv(t)
+ e.putConfig("bab", "shop", map[string]appconfig.Var{"DB": e.seal("pw")})
+ g := e.guard("")
+ armConfigGuard(g)
+ cfg := provisionTickConfig(func(string) string { return "" })
+ if cfg.ConfigGate == nil {
+ t.Fatal("armed guard not wired into the reconcile config")
+ }
+ if err := cfg.ConfigGate(context.Background(), guardRoute("a.example.com")); err == nil {
+ t.Error("the reconcile gate admitted a sealed instance on a keyless box")
+ }
+ if list, total := configRefusalProbe(); len(list) != 1 || total != 1 {
+ t.Errorf("probe = %v/%d, want the one refusal", list, total)
+ }
+}
diff --git a/internal/provision/config_gate_test.go b/internal/provision/config_gate_test.go
new file mode 100644
index 000000000..a13eecbf1
--- /dev/null
+++ b/internal/provision/config_gate_test.go
@@ -0,0 +1,116 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// Issue #213: a box mints an instance's record only if it can provide that
+// instance's config. ReconcileConfig.ConfigGate is the seam; the host agent
+// wires it to the same guard the activator asks before taking a lease.
+
+type gateLog struct {
+ asked []registry.Route
+ refuse map[string]error // host -> refusal
+}
+
+func (g *gateLog) gate(_ context.Context, rt registry.Route) error {
+ g.asked = append(g.asked, rt)
+ return g.refuse[rt.Host]
+}
+
+func TestReconcileDoesNotMintWhatTheGateRefuses(t *testing.T) {
+ s, sqid := setup(t)
+ if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ g := &gateLog{refuse: map[string]error{"acme.go.tito.io": errors.New("no key")}}
+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ConfigGate: g.gate}
+
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ noRoute(t, reg, "acme.go.tito.io", "a refused record was minted")
+ if len(g.asked) != 1 {
+ t.Fatalf("gate asked %d times, want 1", len(g.asked))
+ }
+ // The gate sees the route that WOULD be minted: the activator's own gate
+ // reads App, Account, Environment off it.
+ if got := g.asked[0]; got.App != "tito" || got.Account != sqid || got.Kind != "instance" || got.Socket == "" {
+ t.Errorf("gate saw %+v, want the would-be minted socket-backed instance route", got)
+ }
+
+ // Retried next pass: once the box can provide the config, it mints.
+ delete(g.refuse, "acme.go.tito.io")
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ hasRoute(t, reg, "acme.go.tito.io", "not minted once the gate allowed it")
+}
+
+// A nil gate is today's behaviour exactly.
+func TestReconcileNilGateMintsAsBefore(t *testing.T) {
+ s, sqid := setup(t)
+ if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil {
+ t.Fatal(err)
+ }
+ reg := regWith(t)
+ if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil {
+ t.Fatal(err)
+ }
+ hasRoute(t, reg, "acme.go.tito.io", "nil gate stopped a mint")
+}
+
+// The gate is asked about every minted shape and decides itself which need
+// config; a refusal of one record leaves the others minted, and an existing
+// route is never unminted by a refusal (the wake gate is what protects it).
+func TestReconcileGateRefusesOnlyTheRecordItNames(t *testing.T) {
+ s, sqid := setup(t)
+ a := acmeRecord(sqid)
+ b := acmeRecord(sqid)
+ b.Host, b.Slug = "beta.go.tito.io", "beta"
+ for _, r := range []Record{a, b} {
+ if err := CreateRecord(ctx, s, r); err != nil {
+ t.Fatal(err)
+ }
+ }
+ reg := regWith(t)
+ g := &gateLog{refuse: map[string]error{"beta.go.tito.io": errors.New("cannot open")}}
+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ConfigGate: g.gate}
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ hasRoute(t, reg, "acme.go.tito.io", "an allowed record was held back by its neighbour's refusal")
+ noRoute(t, reg, "beta.go.tito.io", "a refused record was minted")
+
+ // Already minted, then refused: the route stays (removal is only for a
+ // retired record).
+ g.refuse["acme.go.tito.io"] = errors.New("now cannot open")
+ if err := Reconcile(ctx, s, reg, cfg); err != nil {
+ t.Fatal(err)
+ }
+ hasRoute(t, reg, "acme.go.tito.io", "a refusal unminted an existing route")
+}
+
+func TestSidecarReconcileDoesNotMintWhatTheGateRefuses(t *testing.T) {
+ f := sidecarSetup(t)
+ f.putManifest(t, "eleven", SidecarManifest{
+ Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}, Channel: "stable"})
+ f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"})
+ host := f.host("bot", "eleven")
+ g := &gateLog{refuse: map[string]error{host: errors.New("no key")}}
+
+ f.reconcile(t, ReconcileConfig{ConfigGate: g.gate})
+ noRoute(t, f.reg, host, "a refused sidecar was minted")
+ if len(g.asked) != 1 || g.asked[0].Kind != "sidecar" || g.asked[0].App != "eleven" {
+ t.Fatalf("gate asked %+v, want the would-be sidecar route", g.asked)
+ }
+
+ delete(g.refuse, host)
+ f.reconcile(t, ReconcileConfig{ConfigGate: g.gate})
+ hasRoute(t, f.reg, host, "sidecar not minted once the gate allowed it")
+}
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go
index ffef7f3a9..d6a55ef1b 100644
--- a/internal/provision/reconcile.go
+++ b/internal/provision/reconcile.go
@@ -75,6 +75,18 @@ type ReconcileConfig struct {
// path; the auto-purge that does still lives in domains, still refuses
// without a key.
ReleasePub ed25519.PublicKey
+ // ConfigGate answers whether this box can provide the config of the
+ // route a record would mint (amadan #213): nil error means yes, an error
+ // means do not mint it this pass. It is handed the route the mint would
+ // create (routeFor / sidecarRouteFor), so it reads App, Account,
+ // Environment and the shape off the one renderer, and it decides for
+ // itself which shapes need config at all. A refused record is retried
+ // every pass, so a box that gains the key mints on its next tick. The
+ // gate owns its own logging and operator-visible record; this pass only
+ // honours the answer. Existing routes are not unminted by a refusal: the
+ // wake-time gate (activator.ExecBackend.ConfigGate) is what stops one
+ // starting. Nil: no gate, every record mints as before.
+ ConfigGate func(ctx context.Context, rt registry.Route) error
}
// InboundAny and InboundNone are the closed vocabulary ReconcileConfig.Inbound
@@ -934,6 +946,17 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
// the additivity rule, pinned by TestRecordWireShapeIsAdditive and
// TestReconcileMints.
nr := routeFor(rec, cfg.DataDir, seed)
+ // A box that cannot provide this instance's config must not
+ // admit it: it would start without its secrets, and a keyed
+ // sibling would be left with a route that lost the race to one
+ // that was never going to work. Refused before the write, and
+ // before the dry-run line, so a dry run reports what a real pass
+ // would do.
+ if cfg.ConfigGate != nil {
+ if err := cfg.ConfigGate(ctx, nr); err != nil {
+ continue
+ }
+ }
if cfg.DryRun {
log.Printf("provision: dry-run: would mint %s route %s (app %q acct %q channel %q)",
nr.Kind, rec.Host, rec.App, rec.Account, rec.Channel)
diff --git a/internal/provision/sidecar_reconcile.go b/internal/provision/sidecar_reconcile.go
index d7f86c8f3..687dd9349 100644
--- a/internal/provision/sidecar_reconcile.go
+++ b/internal/provision/sidecar_reconcile.go
@@ -387,6 +387,14 @@ func ReconcileSidecars(ctx context.Context, s release.Store, reg *registry.Regis
continue
}
nr := sidecarRouteFor(rec, m, cfg.DataDir)
+ // Same admission rule as the instance pass (amadan #213): a
+ // sidecar that would start without its sealed config is not
+ // minted here. See ReconcileConfig.ConfigGate.
+ if cfg.ConfigGate != nil {
+ if err := cfg.ConfigGate(ctx, nr); err != nil {
+ continue
+ }
+ }
if cfg.DryRun {
log.Printf("provision: dry-run: would mint sidecar route %s (app %q acct %q channel %q args %s)",
rec.Host, rec.App, rec.Account, nr.Channel, nr.Args)
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go
index 2aac4256b..445720e6f 100644
--- a/internal/status/contract_test.go
+++ b/internal/status/contract_test.go
@@ -75,10 +75,14 @@ var allowedBoxKeys = map[string][]string{
"": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"},
"routes": {"static", "resident", "hibernating", "awake", "asleep"},
"replication": {"healthy", "lag_seconds"},
- "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams"},
+ "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "config_refusals", "config_refusals_total", "config_guard"},
"metrics.route_versions": {wildcardAnyKey},
"metrics.upstreams": {wildcardAnyKey},
"metrics.upstreams.*[]": {"addr", "up", "since", "fails", "last_error"},
+ // config_refusals (amadan #213) names hosts and a closed reason word, in
+ // the private block only. Pinned to the field level so a future field on
+ // the refusal is a publication decision too.
+ "metrics.config_refusals[]": {"host", "reason", "since", "last", "count"},
}
// allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as
@@ -289,20 +293,27 @@ func fixtureProbes() status.Probes {
}
return status.Probes{
- MachineID: "test-machine-id",
- Region: "us-west-1",
- Upstreams: upstreams,
- MemFrac: &memFrac,
- DiskFrac: &diskFrac,
- Listening: listening,
- LagSecs: &lagSecs,
- AdoptRunning: &adopting,
- MemTotalBytes: &memTotal,
- MemAvailBytes: &memAvail,
- Load1: &load1,
- DiskFreeBytes: &diskFree,
- Release: "contract-fixture-sha", // distinct from dryrun.go's "dryrun4242sha"
- Tunnel: "connected", // distinct from dryrun.go's "reconnecting"
+ MachineID: "test-machine-id",
+ Region: "us-west-1",
+ Upstreams: upstreams,
+ // One refusal so metrics.config_refusals is exercised by the
+ // allowlist walk and not merely permitted by it.
+ ConfigRefusals: []status.ConfigRefusal{
+ {Host: "contract-refused.internal", Reason: "cannot_open", Since: time.Now().Add(-time.Hour), Last: time.Now(), Count: 2},
+ },
+ ConfigRefusalsTotal: 1,
+ ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)",
+ MemFrac: &memFrac,
+ DiskFrac: &diskFrac,
+ Listening: listening,
+ LagSecs: &lagSecs,
+ AdoptRunning: &adopting,
+ MemTotalBytes: &memTotal,
+ MemAvailBytes: &memAvail,
+ Load1: &load1,
+ DiskFreeBytes: &diskFree,
+ Release: "contract-fixture-sha", // distinct from dryrun.go's "dryrun4242sha"
+ Tunnel: "connected", // distinct from dryrun.go's "reconnecting"
}
}
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go
index bf1f99eea..25e02899f 100644
--- a/internal/status/dryrun.go
+++ b/internal/status/dryrun.go
@@ -108,6 +108,14 @@ const (
var dryRunUpstreamNeedles = []string{dryRunUpstreamHost, dryRunUpstreamAddr}
+// dryRunConfigRefusalHost is the config guard's needle (amadan #213):
+// Box.Metrics.ConfigRefusals names the hosts a keyless box refuses, so the
+// same scope split applies. REQUIRED in status/boxes/*.json and BANNED in
+// every other published object.
+const dryRunConfigRefusalHost = "refused-secret.internal"
+
+var dryRunConfigRefusalNeedles = []string{dryRunConfigRefusalHost}
+
// ScanForLeaks reports which of banned's strings appear anywhere in j. It is
// the single implementation both DryRun's self-check and
// contract_test.go's scanForLeaks test helper run — the test package can't
@@ -189,6 +197,12 @@ func dryRunFixtures() ([]registry.Route, Probes) {
DiskFreeBytes: &diskFree,
Release: "dryrun4242sha",
Tunnel: "reconnecting",
+ // One refused host (amadan #213), so the scope check below has
+ // something to require in the private block and ban elsewhere.
+ ConfigRefusals: []ConfigRefusal{
+ {Host: dryRunConfigRefusalHost, Reason: "no_key", Since: time.Now().Add(-time.Hour), Last: time.Now(), Count: 2},
+ },
+ ConfigRefusalsTotal: 1,
// The probe table, keyed by a route host and carrying a backend
// address — see dryRunUpstreamNeedles. One up and one down upstream,
// so the down one's reason string is exercised too.
@@ -341,6 +355,9 @@ func DryRun(dir string, out io.Writer) error {
// upstreamKeySeen is the same positive half for the probe table
// (dryRunUpstreamNeedles) — Box.Metrics.Upstreams, Task 6.
upstreamKeySeen := false
+ // configRefusalKeySeen is the same positive half for the config guard's
+ // refusal list (dryRunConfigRefusalNeedles), amadan #213.
+ configRefusalKeySeen := false
for _, key := range keys {
body, _, err := s.Get(ctx, key)
@@ -400,6 +417,17 @@ func DryRun(dir string, out io.Writer) error {
return fmt.Errorf("dry run: %s leaked an upstreams fixture %v — Upstreams must never reach the public feed", key, leaks)
}
+ // dryRunConfigRefusalNeedles: the fourth scope split. The refusal list
+ // names real hosts, so status/boxes/*.json is its one private home.
+ if strings.HasPrefix(key, "status/boxes/") {
+ if leaks := ScanForLeaks(body, dryRunConfigRefusalNeedles); len(leaks) == 0 {
+ return fmt.Errorf("dry run: %s: expected config_refusals fixtures %v, found none — ConfigRefusals not attached?", key, dryRunConfigRefusalNeedles)
+ }
+ configRefusalKeySeen = true
+ } else if leaks := ScanForLeaks(body, dryRunConfigRefusalNeedles); len(leaks) > 0 {
+ return fmt.Errorf("dry run: %s leaked a config_refusals fixture %v — ConfigRefusals must never reach the public feed", key, leaks)
+ }
+
// Page assets (index.html, status.css) aren't JSON and simply fail
// to decode here — silently skipped, since the band-digit check
// only means anything for objects that actually carry Load/Capacity
@@ -424,6 +452,9 @@ func DryRun(dir string, out io.Writer) error {
if !upstreamKeySeen {
return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — upstreams scope check never ran")
}
+ if !configRefusalKeySeen {
+ return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — config_refusals scope check never ran")
+ }
fmt.Fprintln(out, "dry-run: no leaks")
return nil
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index 398c399d9..e1b76eaeb 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -133,6 +133,37 @@ type Probes struct {
// answer (no file yet, stat failed), means unknown and the host is left
// out of Metrics.RouteDBBytes rather than published as 0.
DBBytes func(dbPath string) *int64
+
+ // ConfigRefusals is the instances this box is CURRENTLY refusing to take
+ // because it cannot provide their config (amadan #213), as the host's
+ // config guard reports them; nil when it refuses nothing. They land in
+ // the private Metrics block only: they name hosts. ConfigRefusalsTotal is
+ // how many are refused in all when the guard capped the list.
+ ConfigRefusals []ConfigRefusal
+ ConfigRefusalsTotal int
+ // ConfigGuard is "" in the normal mode and a words-only note when the
+ // config guard's explicit bypass is in effect on this box (amadan #213).
+ ConfigGuard string
+}
+
+// ConfigRefusal is one host this box is refusing to take because it cannot
+// provide that instance's config: no CARLOS_SECRETS_KEY, one that does not
+// parse, one that does not open the sealed config, or a store read it could
+// not complete (a box that cannot tell refuses). Reason is a closed word
+// (no_key, key_unparseable, cannot_open, store_error), never error text, so
+// nothing from the store or the crypto reaches a snapshot. Since is when the
+// host was first refused in this stretch, Last the most recent refusal, and
+// Count how many wake or admission attempts were refused.
+//
+// PRIVATE, like everything under Box.Metrics: the host is a real route name.
+// It is declared here, with the other wire types of the snapshot, so the
+// readers (console, operators) need not import the host agent to decode it.
+type ConfigRefusal struct {
+ Host string `json:"host"`
+ Reason string `json:"reason"`
+ Since time.Time `json:"since"`
+ Last time.Time `json:"last"`
+ Count int `json:"count"`
}
// UpstreamState is one upstream's health as the edge's prober currently sees
@@ -258,6 +289,22 @@ type BoxMetrics struct {
// readers. A host whose file could not be sized is absent, never 0; the
// whole map is omitted when no route could be sized.
RouteDBBytes map[string]int64 `json:"route_db_bytes,omitempty"`
+ // ConfigRefusals is the hosts this box is refusing to take because it
+ // cannot provide their config (amadan #213). It is how an operator sees
+ // that a box is keyless, or holds the wrong key, without reading its
+ // journal: a keyed sibling takes these instances, and this list says why
+ // this box did not. Hostname-bearing, so it lives in this private block
+ // and nowhere else (BuildFleet never names Metrics). Omitted when empty.
+ ConfigRefusals []ConfigRefusal `json:"config_refusals,omitempty"`
+ // ConfigRefusalsTotal is the count of refused hosts when ConfigRefusals
+ // was capped, so a long list reads as long. Omitted when zero.
+ ConfigRefusalsTotal int `json:"config_refusals_total,omitempty"`
+ // ConfigGuard says the box's admission guard is bypassed, e.g.
+ // "bypassed (CARLOS_NO_SEALED_CONFIG)": it admits placed exec instances
+ // without checking their sealed config. Omitted in the normal mode, so an
+ // operator scanning snapshots sees only the boxes that are exposed. A
+ // fixed phrase, never a value from the environment.
+ ConfigGuard string `json:"config_guard,omitempty"`
}
// AliasReading is one alias hostname's status as CollectBox saw it this
@@ -650,7 +697,8 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" ||
p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil ||
len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 ||
- len(routeDBBytes) > 0 || p.Alerts != nil {
+ len(routeDBBytes) > 0 || len(p.ConfigRefusals) > 0 || p.ConfigGuard != "" ||
+ p.Alerts != nil {
box.Metrics = &BoxMetrics{
MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes,
Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes,
@@ -667,6 +715,10 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {
// alongside another known field above, and omitempty drops it.
Upstreams: p.Upstreams,
RouteDBBytes: routeDBBytes,
+ // A fresh copy: the guard's own slice is its to mutate.
+ ConfigRefusals: append([]ConfigRefusal(nil), p.ConfigRefusals...),
+ ConfigRefusalsTotal: p.ConfigRefusalsTotal,
+ ConfigGuard: p.ConfigGuard,
}
}
diff --git a/internal/status/status_test.go b/internal/status/status_test.go
index 4c6f12fbe..c9f6c9035 100644
--- a/internal/status/status_test.go
+++ b/internal/status/status_test.go
@@ -1444,3 +1444,79 @@ func TestCollectBoxCarriesRouteDBBytes(t *testing.T) {
t.Errorf("metrics = %+v with no DBBytes probe, want absent", b.Metrics)
}
}
+
+// TestCollectBoxCarriesConfigRefusals (amadan #213): a box that refuses to
+// take an instance because it cannot provide that instance's config says so
+// in its private Metrics block, where an operator can see which hosts and
+// why. The hostnames never reach the public fleet.json, and a box with no
+// refusal carries no key at all.
+func TestCollectBoxCarriesConfigRefusals(t *testing.T) {
+ since := time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC)
+ refusals := []status.ConfigRefusal{
+ {Host: "refused-secret.example.com", Reason: "no_key", Since: since, Last: since.Add(time.Minute), Count: 3},
+ }
+ routes := []registry.Route{{Host: "refused-secret.example.com", Kind: "instance", Hibernate: true}}
+ b := status.CollectBox(routes, status.Probes{MachineID: "m", ConfigRefusals: refusals, ConfigRefusalsTotal: 5}, time.Now())
+ if b.Metrics == nil {
+ t.Fatal("metrics block missing: a refusal alone should attach it")
+ }
+ if got := b.Metrics.ConfigRefusals; len(got) != 1 || got[0] != refusals[0] {
+ t.Fatalf("config_refusals = %+v, want %+v", got, refusals)
+ }
+ if b.Metrics.ConfigRefusalsTotal != 5 {
+ t.Errorf("config_refusals_total = %d, want 5", b.Metrics.ConfigRefusalsTotal)
+ }
+
+ s := teststore(t)
+ ctx := context.Background()
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fj, err := json.Marshal(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, needle := range []string{"refused-secret.example.com", "no_key", "config_refusals"} {
+ if strings.Contains(string(fj), needle) {
+ t.Errorf("fleet.json leaked %q: %s", needle, fj)
+ }
+ }
+
+ // Nothing refused: no Metrics block from this alone, no key on the wire.
+ quiet := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now())
+ if quiet.Metrics != nil {
+ t.Errorf("metrics = %+v with no refusal, want absent", quiet.Metrics)
+ }
+ raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now()))
+ if strings.Contains(string(raw), "config_refusals") {
+ t.Errorf("a box with no refusal publishes config_refusals: %s", raw)
+ }
+}
+
+// The config guard's bypass is visible in the private block, and only there.
+func TestCollectBoxCarriesTheConfigGuardBypass(t *testing.T) {
+ b := status.CollectBox(nil, status.Probes{MachineID: "m", ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)"}, time.Now())
+ if b.Metrics == nil || b.Metrics.ConfigGuard != "bypassed (CARLOS_NO_SEALED_CONFIG)" {
+ t.Fatalf("metrics = %+v, want the bypass word", b.Metrics)
+ }
+ s := teststore(t)
+ ctx := context.Background()
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ f, err := status.BuildFleet(ctx, s, nil, time.Now(), 5*time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fj, _ := json.Marshal(f)
+ if strings.Contains(string(fj), "bypass") || strings.Contains(string(fj), "NO_SEALED") {
+ t.Errorf("fleet.json carries the bypass: %s", fj)
+ }
+ if quiet := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); quiet.Metrics != nil {
+ t.Errorf("a box in the normal mode attached metrics: %+v", quiet.Metrics)
+ }
+}