| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ecdh" |
| 6 | "errors" |
| 7 | "log" |
| 8 | "sort" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "sync/atomic" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/appconfig" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | "github.com/carlosframework/platform/internal/status" |
| 18 | ) |
| 19 | |
| 20 | // The config guard (amadan #213). A box takes an instance's lease, or mints |
| 21 | // its record, only if it can provide that instance's config. |
| 22 | // |
| 23 | // The failure it ends: a box with no CARLOS_SECRETS_KEY (or the wrong one) |
| 24 | // still admitted and woke socket-backed shared-pool instances in its region. |
| 25 | // Its config sync loops are off without the key, and ExecBackend.instanceEnv |
| 26 | // treats a missing <host>.config.env as optional, so the instance started |
| 27 | // without its secrets or store config. kpmg was down 1h54m on 2026-10-01 |
| 28 | // because the UK box had no key. The console cannot prove what reconcile will |
| 29 | // admit, so the invariant lives here, at admission. |
| 30 | // |
| 31 | // One guard per process, built from CARLOS_SECRETS_KEY and the deploy store, |
| 32 | // and asked through Gate from three places, each BEFORE the side effect it |
| 33 | // guards: activator.ExecBackend.Acquire (before lease.Acquire), |
| 34 | // provision.Reconcile and provision.ReconcileSidecars (before a mint). The |
| 35 | // decision itself is appconfig.CanProvideConfig; this type adds what that |
| 36 | // pure function cannot: the box's key, the store, a short refusal cache, the |
| 37 | // once-per-host log and the list the status snapshot publishes. |
| 38 | |
| 39 | const ( |
| 40 | // configGuardRefuseTTL bounds how long a refusal is answered from memory. |
| 41 | // A request storm on a refused host would otherwise cost a set of bucket |
| 42 | // reads per request, and the reconcile pass asks about every unminted |
| 43 | // record every tick. It is also the retry interval after a store error: |
| 44 | // the box can tell again within this long of the store coming back. An |
| 45 | // admission is never cached: wakes are rare, and the answer must be |
| 46 | // current when it is taken. |
| 47 | configGuardRefuseTTL = 30 * time.Second |
| 48 | // configGuardListCap bounds the refusal list in the status snapshot; the |
| 49 | // total is published beside it so a capped list reads as long. |
| 50 | configGuardListCap = 100 |
| 51 | // configGuardRefusalMaxAge is how long a host stays in the published list |
| 52 | // without being asked about again. A wake-time refusal is only re-asked |
| 53 | // when someone requests the host, so this is what makes the list mean |
| 54 | // "currently refused" rather than "ever refused since boot". |
| 55 | configGuardRefusalMaxAge = 24 * time.Hour |
| 56 | ) |
| 57 | |
| 58 | // errNoDeployStore is openDeployStoreEnv's answer for a box with no |
| 59 | // deployment store configured at all: no bucket, no dir, no fleet identity. |
| 60 | // |
| 61 | // That is NOT proof that nothing is sealed. Config lives in the deployment |
| 62 | // store, but CARLOS_HIBERNATE_BUCKET enables leases independently of it, so a |
| 63 | // box can wake instances whose sealed config sits in a store it was never told |
| 64 | // about. The guard therefore refuses (no_config_source) unless the operator |
| 65 | // has said, with CARLOS_NO_SEALED_CONFIG=1 (or "true"), that this deployment |
| 66 | // has no sealed config: the config-free development setup. The flag only |
| 67 | // matters when there is no store; with one, config is read as usual. |
| 68 | var errNoDeployStore = errors.New("neither CARLOS_DEPLOYMENT_BUCKET nor CARLOS_DEPLOYMENT_DIR is set") |
| 69 | |
| 70 | type guardRefusal struct { |
| 71 | reason string |
| 72 | since time.Time |
| 73 | last time.Time |
| 74 | count int |
| 75 | } |
| 76 | |
| 77 | type guardVerdict struct { |
| 78 | err error |
| 79 | until time.Time |
| 80 | } |
| 81 | |
| 82 | type configGuard struct { |
| 83 | priv *ecdh.PrivateKey |
| 84 | keyState appconfig.KeyState |
| 85 | openStore func(context.Context) (release.Store, error) |
| 86 | now func() time.Time |
| 87 | // noSealedConfig is the operator's explicit "this deployment has no |
| 88 | // sealed config" (CARLOS_NO_SEALED_CONFIG), honoured only when there is |
| 89 | // no deployment store at all. |
| 90 | noSealedConfig bool |
| 91 | |
| 92 | mu sync.Mutex |
| 93 | bypassed map[string]bool // hosts already logged as admitted by the bypass |
| 94 | store release.Store |
| 95 | verdicts map[string]guardVerdict // host -> cached refusal |
| 96 | refused map[string]*guardRefusal |
| 97 | } |
| 98 | |
| 99 | // newConfigGuard reads CARLOS_SECRETS_KEY once, the way the config loops do. |
| 100 | // A key that does not parse is logged here, once, at arming; per-host lines |
| 101 | // say only that the host was refused. |
| 102 | func newConfigGuard(getenv func(string) string, openStore func(context.Context) (release.Store, error)) *configGuard { |
| 103 | g := &configGuard{ |
| 104 | openStore: openStore, |
| 105 | now: time.Now, |
| 106 | verdicts: map[string]guardVerdict{}, |
| 107 | refused: map[string]*guardRefusal{}, |
| 108 | } |
| 109 | g.noSealedConfig = noSealedConfigFlag(getenv) |
| 110 | raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")) |
| 111 | switch { |
| 112 | case raw == "": |
| 113 | g.keyState = appconfig.KeyAbsent |
| 114 | default: |
| 115 | priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY") |
| 116 | if err != nil { |
| 117 | log.Printf("agent: config guard: %v", err) |
| 118 | g.keyState = appconfig.KeyUnparseable |
| 119 | } else { |
| 120 | g.priv, g.keyState = priv, appconfig.KeyOK |
| 121 | } |
| 122 | } |
| 123 | return g |
| 124 | } |
| 125 | |
| 126 | // noSealedConfigFlag is the operator's CARLOS_NO_SEALED_CONFIG, read the |
| 127 | // one way the guard and the instance key's configuration reader share. |
| 128 | func noSealedConfigFlag(getenv func(string) string) bool { |
| 129 | switch strings.ToLower(strings.TrimSpace(getenv("CARLOS_NO_SEALED_CONFIG"))) { |
| 130 | case "1", "true": |
| 131 | return true |
| 132 | } |
| 133 | return false |
| 134 | } |
| 135 | |
| 136 | // needsBoxConfig reports whether a route is one this box would spawn with a |
| 137 | // config.env of its own: a socket-backed instance or a sidecar worker that |
| 138 | // belongs to an app. Static sites, addr-backed routes, Backing=unit routes |
| 139 | // (whose config the root adopt pass delivers), platform routes and app-less |
| 140 | // rows take no part: the guard must not touch them. |
| 141 | // |
| 142 | // The exclusion is Backing and nothing else, because Backing is what |
| 143 | // SplitBackend dispatches on. A socket-backed hibernating instance may carry a |
| 144 | // Unit name with an empty Backing (the record validator and `carlos route |
| 145 | // -unit` allow it) and still reaches ExecBackend.Acquire. |
| 146 | func needsBoxConfig(rt registry.Route) bool { |
| 147 | if rt.App == "" || rt.Platform || rt.Backing == registry.BackingUnit { |
| 148 | return false |
| 149 | } |
| 150 | switch { |
| 151 | case rt.IsSidecar(): |
| 152 | return true |
| 153 | case rt.Kind == "instance": |
| 154 | return rt.Socket != "" |
| 155 | } |
| 156 | return false |
| 157 | } |
| 158 | |
| 159 | // Gate is the seam both ExecBackend.ConfigGate and ReconcileConfig.ConfigGate |
| 160 | // are wired to. nil means this box can provide rt's config (or none is |
| 161 | // needed); an error means refuse, and is an *appconfig.ConfigRefusal. |
| 162 | func (g *configGuard) Gate(ctx context.Context, rt registry.Route) error { |
| 163 | if !needsBoxConfig(rt) { |
| 164 | return nil |
| 165 | } |
| 166 | now := g.now() |
| 167 | if err := g.cached(rt.Host, now); err != nil { |
| 168 | g.note(rt.Host, err, now) |
| 169 | return err |
| 170 | } |
| 171 | err := g.decide(ctx, rt) |
| 172 | if err != nil { |
| 173 | g.mu.Lock() |
| 174 | g.verdicts[rt.Host] = guardVerdict{err: err, until: now.Add(configGuardRefuseTTL)} |
| 175 | g.mu.Unlock() |
| 176 | g.note(rt.Host, err, now) |
| 177 | return err |
| 178 | } |
| 179 | g.clear(rt.Host) |
| 180 | return nil |
| 181 | } |
| 182 | |
| 183 | func (g *configGuard) cached(host string, now time.Time) error { |
| 184 | g.mu.Lock() |
| 185 | defer g.mu.Unlock() |
| 186 | v, ok := g.verdicts[host] |
| 187 | if !ok { |
| 188 | return nil |
| 189 | } |
| 190 | if !now.Before(v.until) { |
| 191 | delete(g.verdicts, host) |
| 192 | return nil |
| 193 | } |
| 194 | return v.err |
| 195 | } |
| 196 | |
| 197 | func (g *configGuard) decide(ctx context.Context, rt registry.Route) error { |
| 198 | s, err := g.deployStore(ctx) |
| 199 | if errors.Is(err, errNoDeployStore) { |
| 200 | if g.noSealedConfig { |
| 201 | // The operator said this deployment has none. Said loudly, once |
| 202 | // per host: this is the one path that admits without looking. |
| 203 | g.noteBypass(rt.Host) |
| 204 | return nil |
| 205 | } |
| 206 | return &appconfig.ConfigRefusal{Reason: appconfig.ReasonNoSource, Err: err} |
| 207 | } |
| 208 | if err != nil { |
| 209 | return &appconfig.ConfigRefusal{Reason: appconfig.ReasonStoreError, Err: err} |
| 210 | } |
| 211 | return appconfig.CanProvideConfig(ctx, s, g.priv, g.keyState, release.RouteAccount(rt), rt.App, rt.Environment, rt.Host) |
| 212 | } |
| 213 | |
| 214 | // configGuardBypassStatus is the private status word published while the |
| 215 | // CARLOS_NO_SEALED_CONFIG bypass is in effect. |
| 216 | const configGuardBypassStatus = "bypassed (CARLOS_NO_SEALED_CONFIG)" |
| 217 | |
| 218 | // noteBypass logs, once per host, that a placed instance was admitted without |
| 219 | // its sealed config being checked. |
| 220 | func (g *configGuard) noteBypass(host string) { |
| 221 | g.mu.Lock() |
| 222 | if g.bypassed == nil { |
| 223 | g.bypassed = map[string]bool{} |
| 224 | } |
| 225 | seen := g.bypassed[host] |
| 226 | g.bypassed[host] = true |
| 227 | g.mu.Unlock() |
| 228 | if !seen { |
| 229 | log.Printf("agent: config guard: BYPASS: admitting %s WITHOUT checking its sealed config, because CARLOS_NO_SEALED_CONFIG is set and this box has no deployment store. Never set it on a box that serves tenants", host) |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | // Bypassed reports whether the CARLOS_NO_SEALED_CONFIG bypass is in effect: |
| 234 | // the flag is set AND the box has no deployment store configured. A flag set |
| 235 | // beside a store changes nothing, so it does not count. |
| 236 | func (g *configGuard) Bypassed() bool { |
| 237 | if !g.noSealedConfig { |
| 238 | return false |
| 239 | } |
| 240 | _, err := g.deployStore(context.Background()) |
| 241 | return errors.Is(err, errNoDeployStore) |
| 242 | } |
| 243 | |
| 244 | // Status is the private status-block word for the guard's mode: the bypass |
| 245 | // words while it is in effect, "" otherwise. |
| 246 | func (g *configGuard) Status() string { |
| 247 | if g.Bypassed() { |
| 248 | return configGuardBypassStatus |
| 249 | } |
| 250 | return "" |
| 251 | } |
| 252 | |
| 253 | // Announce logs the guard's mode once at startup. The bypass is the only mode |
| 254 | // worth shouting about, and it is shouted whether or not it is in effect, so a |
| 255 | // flag copied into the wrong host.env is visible either way. |
| 256 | func (g *configGuard) Announce() { |
| 257 | switch { |
| 258 | case g.Bypassed(): |
| 259 | log.Printf("agent: config guard: BYPASSED. CARLOS_NO_SEALED_CONFIG is set and no deployment store is configured, so this box admits placed exec instances without checking their sealed config. Intended for config-free development only; it is listed in this box's status snapshot") |
| 260 | case g.noSealedConfig: |
| 261 | log.Printf("agent: config guard: CARLOS_NO_SEALED_CONFIG is set but a deployment store is configured, so it is ignored and sealed config is checked as usual") |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | // deployStore opens the store on first use and keeps it. A failure is not |
| 266 | // kept, so the next ask (after the refusal TTL) tries again. |
| 267 | func (g *configGuard) deployStore(ctx context.Context) (release.Store, error) { |
| 268 | g.mu.Lock() |
| 269 | s := g.store |
| 270 | g.mu.Unlock() |
| 271 | if s != nil { |
| 272 | return s, nil |
| 273 | } |
| 274 | s, err := g.openStore(ctx) |
| 275 | if err != nil { |
| 276 | return nil, err |
| 277 | } |
| 278 | g.mu.Lock() |
| 279 | g.store = s |
| 280 | g.mu.Unlock() |
| 281 | return s, nil |
| 282 | } |
| 283 | |
| 284 | // note records a refusal for the snapshot and logs it: once per host, and |
| 285 | // again only when the reason changes. The detail in the log is the operator's; |
| 286 | // the snapshot carries the closed reason word alone. |
| 287 | func (g *configGuard) note(host string, err error, now time.Time) { |
| 288 | reason := appconfig.ReasonStoreError |
| 289 | var r *appconfig.ConfigRefusal |
| 290 | if errors.As(err, &r) { |
| 291 | reason = r.Reason |
| 292 | } |
| 293 | g.mu.Lock() |
| 294 | e := g.refused[host] |
| 295 | fresh := e == nil || e.reason != reason |
| 296 | if e == nil { |
| 297 | e = &guardRefusal{since: now} |
| 298 | g.refused[host] = e |
| 299 | } |
| 300 | e.reason, e.last = reason, now |
| 301 | e.count++ |
| 302 | g.mu.Unlock() |
| 303 | if fresh { |
| 304 | log.Printf("agent: config guard: refusing %s: %v. A box that cannot open an instance's sealed config does not take its lease or mint its record, so a keyed sibling can; it is listed in this box's status snapshot", host, err) |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | // clear forgets a host that now passes, saying so once if it was refused. |
| 309 | func (g *configGuard) clear(host string) { |
| 310 | g.mu.Lock() |
| 311 | _, was := g.refused[host] |
| 312 | delete(g.refused, host) |
| 313 | delete(g.verdicts, host) |
| 314 | g.mu.Unlock() |
| 315 | if was { |
| 316 | log.Printf("agent: config guard: %s can be provided from this box again", host) |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | // Refusals is what the status snapshot publishes: the currently refused |
| 321 | // hosts, newest first and capped, plus how many there are in all. A host not |
| 322 | // asked about for configGuardRefusalMaxAge is dropped. |
| 323 | func (g *configGuard) Refusals() ([]status.ConfigRefusal, int) { |
| 324 | now := g.now() |
| 325 | g.mu.Lock() |
| 326 | defer g.mu.Unlock() |
| 327 | var out []status.ConfigRefusal |
| 328 | for host, e := range g.refused { |
| 329 | if now.Sub(e.last) > configGuardRefusalMaxAge { |
| 330 | delete(g.refused, host) |
| 331 | continue |
| 332 | } |
| 333 | out = append(out, status.ConfigRefusal{Host: host, Reason: e.reason, Since: e.since, Last: e.last, Count: e.count}) |
| 334 | } |
| 335 | total := len(out) |
| 336 | sort.Slice(out, func(i, j int) bool { |
| 337 | if !out[i].Last.Equal(out[j].Last) { |
| 338 | return out[i].Last.After(out[j].Last) |
| 339 | } |
| 340 | return out[i].Host < out[j].Host |
| 341 | }) |
| 342 | if len(out) > configGuardListCap { |
| 343 | out = out[:configGuardListCap] |
| 344 | } |
| 345 | return out, total |
| 346 | } |
| 347 | |
| 348 | // armedConfigGuard is the process's one guard, set by RunAgent before any |
| 349 | // loop that asks it starts. Nil is "not armed": every consumer treats that as |
| 350 | // today's behaviour, so tests and callers that never arm one are unaffected. |
| 351 | var armedConfigGuard atomic.Pointer[configGuard] |
| 352 | |
| 353 | func armConfigGuard(g *configGuard) { armedConfigGuard.Store(g) } |
| 354 | |
| 355 | // configGate is the armed guard's Gate, or nil when none is armed. |
| 356 | func configGate() func(context.Context, registry.Route) error { |
| 357 | if g := armedConfigGuard.Load(); g != nil { |
| 358 | return g.Gate |
| 359 | } |
| 360 | return nil |
| 361 | } |
| 362 | |
| 363 | // configGuardProbe feeds status.Probes.ConfigGuard: "" unless the bypass is |
| 364 | // in effect. |
| 365 | func configGuardProbe() string { |
| 366 | if g := armedConfigGuard.Load(); g != nil { |
| 367 | return g.Status() |
| 368 | } |
| 369 | return "" |
| 370 | } |
| 371 | |
| 372 | // configRefusalProbe feeds status.Probes.ConfigRefusals. |
| 373 | func configRefusalProbe() ([]status.ConfigRefusal, int) { |
| 374 | if g := armedConfigGuard.Load(); g != nil { |
| 375 | return g.Refusals() |
| 376 | } |
| 377 | return nil, 0 |
| 378 | } |
| 379 | |
amadan