amadan
activator, provision: a ConfigGate before the lease and before a mint (#213)
ExecBackend.Acquire asks the gate before lease.Acquire, the only place an instance or sidecar lease is taken, so every wake source is covered. A refusal is a quiet 503 with Retry-After 30. Reconcile and ReconcileSidecars ask it before minting a record. Nil gate is today's behaviour.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
6 files changed,
+273
−0
internal/activator/activator.go+23 −0internal/activator/backend_exec.go+17 −0internal/activator/config_gate_test.go+86 −0internal/provision/config_gate_test.go+116 −0internal/provision/reconcile.go+23 −0internal/provision/sidecar_reconcile.go+8 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index 16ef6357a..15802bcd1 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -574,6 +574,21 @@ func (e *errLeaseHeld) Error() string { |
| return fmt.Sprintf("instance lease held by %q (retry in %s)", e.holder, e.retryAfter) |
| } |
| +// errConfigRefused: this box cannot provide the instance's sealed config |
| +// (no key, the wrong key, or it could not tell), so it will not take the |
| +// lease (amadan #213). The condition outlives one request and a keyed |
| +// sibling may take the lease in the meantime, so the edge answers 503 with a |
| +// longer Retry-After than a lease contention gets. |
| +type errConfigRefused struct{ err error } |
| + |
| +func (e *errConfigRefused) Error() string { |
| + return "instance config unavailable on this box: " + e.err.Error() |
| +} |
| +func (e *errConfigRefused) Unwrap() error { return e.err } |
| + |
| +// configRefusedRetryAfter is the Retry-After (seconds) a refused wake sends. |
| +const configRefusedRetryAfter = 30 |
| + |
| var errWakeTimeout = errors.New("timed out waiting for instance wake") |
| // Service is the activator (Eleven's activatorSvc). It structurally satisfies |
| @@ -2850,9 +2865,17 @@ func (a *Service) serve(w http.ResponseWriter, r *http.Request, rt registry.Rout |
| release, err := a.begin(r.Context(), rt, stamp) |
| if err != nil { |
| var held *errLeaseHeld |
| + var refused *errConfigRefused |
| switch { |
| case errors.Is(err, ErrFenced): |
| maintenanceResponse(w, r) |
| + case errors.As(err, &refused): |
| + // Quiet by design: the config guard logs once per host and |
| + // publishes the refusal in the box's status snapshot. A line per |
| + // request here would bury that one. |
| + w.Header().Set("Retry-After", strconv.Itoa(configRefusedRetryAfter)) |
| + edge.MarkAnswered(r) |
| + http.Error(w, "instance failed to wake", http.StatusServiceUnavailable) |
| case errors.As(err, &held): |
| retry := int(held.retryAfter/time.Second) + 1 |
| w.Header().Set("Retry-After", strconv.Itoa(retry)) |
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go| index 114ee0987..c75513e80 100644 |
| --- a/internal/activator/backend_exec.go |
| +++ b/internal/activator/backend_exec.go |
| @@ -142,6 +142,14 @@ type ExecBackend struct { |
| // gate (every unit test and any box that predates the check). Wired in |
| // production by Task 6. |
| NestedOK func() error |
| + // ConfigGate answers whether this box can provide rt's config (amadan |
| + // #213): nil means yes, an error means refuse. Acquire asks it BEFORE |
| + // lease.Acquire, so a box that would start the instance without its |
| + // secrets never holds the lease, and a keyed sibling takes it instead. |
| + // The gate owns its own policy, logging and operator-visible record of |
| + // the refusal; this backend only honours the answer. Nil: no gate (every |
| + // unit test, and a box with no deploy store to read config from). |
| + ConfigGate func(ctx context.Context, rt registry.Route) error |
| // ObservePoll is how often the exit observer (unitproc.go) and a |
| // seed's cgroup-empty wait re-check unit state. 0: one second. A field |
| // so tests can shrink the wait without a package-global var shared |
| @@ -227,6 +235,15 @@ func (b *ExecBackend) LiveVersion(host string) (string, error) { |
| // Retry-After that tracks the holder's expiry. |
| func (b *ExecBackend) Acquire(ctx context.Context, rt registry.Route) (Lease, error) { |
| host := rt.Host |
| + // Admission before the lease: the instance's config has to be |
| + // providable from THIS box, or it must not be the one that starts it. |
| + // Here, not in doWake, because this is the only place an instance or |
| + // sidecar lease is taken; every wake source reaches it. |
| + if b.ConfigGate != nil { |
| + if err := b.ConfigGate(ctx, rt); err != nil { |
| + return nil, &errConfigRefused{err: err} |
| + } |
| + } |
| held, err := lease.Acquire(ctx, b.Cfg.Bucket, b.Cfg.hibernatePrefix(host), b.Cfg.Region) |
| if err != nil { |
| var he *lease.HeldError |
diff --git a/internal/activator/config_gate_test.go b/internal/activator/config_gate_test.go| new file mode 100644 |
| index 000000000..51ba62fed |
| --- /dev/null |
| +++ b/internal/activator/config_gate_test.go |
| @@ -0,0 +1,86 @@ |
| +package activator |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "net/http" |
| + "net/http/httptest" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// Issue #213: a box that cannot provide an instance's config must not take |
| +// its lease. The gate runs inside ExecBackend.Acquire, before lease.Acquire, |
| +// so every wake source (request, tick, sweeper, worker boot) is covered by |
| +// the one place a lease is taken. |
| + |
| +// The gate is asked about the route being woken, and a refusal means the |
| +// lease is never attempted. Cfg.Bucket is empty here, so a call that fell |
| +// through to lease.Acquire would fail with a different error. |
| +func TestExecBackendAcquireRefusesBeforeTheLease(t *testing.T) { |
| + var asked []registry.Route |
| + boom := errors.New("cannot open sealed config") |
| + b := &ExecBackend{ConfigGate: func(_ context.Context, rt registry.Route) error { |
| + asked = append(asked, rt) |
| + return boom |
| + }} |
| + rt := registry.Route{Host: "kpmg.example.com", Kind: "instance", App: "shop", Hibernate: true} |
| + |
| + lease, err := b.Acquire(context.Background(), rt) |
| + if lease != nil { |
| + t.Fatalf("lease = %v, want nil", lease) |
| + } |
| + var refused *errConfigRefused |
| + if !errors.As(err, &refused) { |
| + t.Fatalf("err = %v, want *errConfigRefused", err) |
| + } |
| + if !errors.Is(err, boom) { |
| + t.Errorf("err %v does not wrap the gate's reason", err) |
| + } |
| + if len(asked) != 1 || asked[0].Host != rt.Host { |
| + t.Errorf("gate asked about %v, want exactly %s", asked, rt.Host) |
| + } |
| +} |
| + |
| +// The refusal is the member's 503 with a Retry-After, answered quietly (the |
| +// guard logs once per host; a log line per request would drown it), and the |
| +// failed wake leaves no entry behind so the next request asks again. |
| +func TestActivatorConfigRefusal503(t *testing.T) { |
| + a, b, _, reg := testActivator(t, time.Minute) |
| + b.acquireErr = &errConfigRefused{err: errors.New("no key")} |
| + |
| + if err := reg.Upsert(registry.Route{Host: "nokey.example.com", Kind: "instance", Socket: "/tmp/nokey.sock", Hibernate: true}); err != nil { |
| + t.Fatalf("upsert: %v", err) |
| + } |
| + rt, err := newTestRouter(reg, a) |
| + if err != nil { |
| + t.Fatalf("new router: %v", err) |
| + } |
| + front := httptest.NewServer(rt) |
| + t.Cleanup(front.Close) |
| + |
| + req, _ := http.NewRequest("GET", front.URL+"/", nil) |
| + req.Host = "nokey.example.com" |
| + resp, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatalf("request: %v", err) |
| + } |
| + defer resp.Body.Close() |
| + if resp.StatusCode != http.StatusServiceUnavailable { |
| + t.Fatalf("status %d, want 503", resp.StatusCode) |
| + } |
| + if ra := resp.Header.Get("Retry-After"); ra != "30" { |
| + t.Fatalf("Retry-After %q, want 30", ra) |
| + } |
| + if a.entryCount() != 0 { |
| + t.Fatal("refused wake left an entry behind") |
| + } |
| + // Nothing past Acquire ran: no restore, no spawn. |
| + b.mu.Lock() |
| + defer b.mu.Unlock() |
| + if b.restores != 0 || b.starts != 0 { |
| + t.Fatalf("refused wake went on: restores=%d starts=%d", b.restores, b.starts) |
| + } |
| +} |
diff --git a/internal/provision/config_gate_test.go b/internal/provision/config_gate_test.go| new file mode 100644 |
| index 000000000..a13eecbf1 |
| --- /dev/null |
| +++ b/internal/provision/config_gate_test.go |
| @@ -0,0 +1,116 @@ |
| +package provision |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// Issue #213: a box mints an instance's record only if it can provide that |
| +// instance's config. ReconcileConfig.ConfigGate is the seam; the host agent |
| +// wires it to the same guard the activator asks before taking a lease. |
| + |
| +type gateLog struct { |
| + asked []registry.Route |
| + refuse map[string]error // host -> refusal |
| +} |
| + |
| +func (g *gateLog) gate(_ context.Context, rt registry.Route) error { |
| + g.asked = append(g.asked, rt) |
| + return g.refuse[rt.Host] |
| +} |
| + |
| +func TestReconcileDoesNotMintWhatTheGateRefuses(t *testing.T) { |
| + s, sqid := setup(t) |
| + if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg := regWith(t) |
| + g := &gateLog{refuse: map[string]error{"acme.go.tito.io": errors.New("no key")}} |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ConfigGate: g.gate} |
| + |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + noRoute(t, reg, "acme.go.tito.io", "a refused record was minted") |
| + if len(g.asked) != 1 { |
| + t.Fatalf("gate asked %d times, want 1", len(g.asked)) |
| + } |
| + // The gate sees the route that WOULD be minted: the activator's own gate |
| + // reads App, Account, Environment off it. |
| + if got := g.asked[0]; got.App != "tito" || got.Account != sqid || got.Kind != "instance" || got.Socket == "" { |
| + t.Errorf("gate saw %+v, want the would-be minted socket-backed instance route", got) |
| + } |
| + |
| + // Retried next pass: once the box can provide the config, it mints. |
| + delete(g.refuse, "acme.go.tito.io") |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + hasRoute(t, reg, "acme.go.tito.io", "not minted once the gate allowed it") |
| +} |
| + |
| +// A nil gate is today's behaviour exactly. |
| +func TestReconcileNilGateMintsAsBefore(t *testing.T) { |
| + s, sqid := setup(t) |
| + if err := CreateRecord(ctx, s, acmeRecord(sqid)); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg := regWith(t) |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + hasRoute(t, reg, "acme.go.tito.io", "nil gate stopped a mint") |
| +} |
| + |
| +// The gate is asked about every minted shape and decides itself which need |
| +// config; a refusal of one record leaves the others minted, and an existing |
| +// route is never unminted by a refusal (the wake gate is what protects it). |
| +func TestReconcileGateRefusesOnlyTheRecordItNames(t *testing.T) { |
| + s, sqid := setup(t) |
| + a := acmeRecord(sqid) |
| + b := acmeRecord(sqid) |
| + b.Host, b.Slug = "beta.go.tito.io", "beta" |
| + for _, r := range []Record{a, b} { |
| + if err := CreateRecord(ctx, s, r); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + reg := regWith(t) |
| + g := &gateLog{refuse: map[string]error{"beta.go.tito.io": errors.New("cannot open")}} |
| + cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", ConfigGate: g.gate} |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + hasRoute(t, reg, "acme.go.tito.io", "an allowed record was held back by its neighbour's refusal") |
| + noRoute(t, reg, "beta.go.tito.io", "a refused record was minted") |
| + |
| + // Already minted, then refused: the route stays (removal is only for a |
| + // retired record). |
| + g.refuse["acme.go.tito.io"] = errors.New("now cannot open") |
| + if err := Reconcile(ctx, s, reg, cfg); err != nil { |
| + t.Fatal(err) |
| + } |
| + hasRoute(t, reg, "acme.go.tito.io", "a refusal unminted an existing route") |
| +} |
| + |
| +func TestSidecarReconcileDoesNotMintWhatTheGateRefuses(t *testing.T) { |
| + f := sidecarSetup(t) |
| + f.putManifest(t, "eleven", SidecarManifest{ |
| + Schema: 1, App: "eleven", Args: []string{"sidecar", "run"}, Channel: "stable"}) |
| + f.putRecord(t, SidecarRecord{Schema: 1, Name: "bot", App: "eleven", Channel: "stable"}) |
| + host := f.host("bot", "eleven") |
| + g := &gateLog{refuse: map[string]error{host: errors.New("no key")}} |
| + |
| + f.reconcile(t, ReconcileConfig{ConfigGate: g.gate}) |
| + noRoute(t, f.reg, host, "a refused sidecar was minted") |
| + if len(g.asked) != 1 || g.asked[0].Kind != "sidecar" || g.asked[0].App != "eleven" { |
| + t.Fatalf("gate asked %+v, want the would-be sidecar route", g.asked) |
| + } |
| + |
| + delete(g.refuse, host) |
| + f.reconcile(t, ReconcileConfig{ConfigGate: g.gate}) |
| + hasRoute(t, f.reg, host, "sidecar not minted once the gate allowed it") |
| +} |
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go| index ffef7f3a9..d6a55ef1b 100644 |
| --- a/internal/provision/reconcile.go |
| +++ b/internal/provision/reconcile.go |
| @@ -75,6 +75,18 @@ type ReconcileConfig struct { |
| // path; the auto-purge that does still lives in domains, still refuses |
| // without a key. |
| ReleasePub ed25519.PublicKey |
| + // ConfigGate answers whether this box can provide the config of the |
| + // route a record would mint (amadan #213): nil error means yes, an error |
| + // means do not mint it this pass. It is handed the route the mint would |
| + // create (routeFor / sidecarRouteFor), so it reads App, Account, |
| + // Environment and the shape off the one renderer, and it decides for |
| + // itself which shapes need config at all. A refused record is retried |
| + // every pass, so a box that gains the key mints on its next tick. The |
| + // gate owns its own logging and operator-visible record; this pass only |
| + // honours the answer. Existing routes are not unminted by a refusal: the |
| + // wake-time gate (activator.ExecBackend.ConfigGate) is what stops one |
| + // starting. Nil: no gate, every record mints as before. |
| + ConfigGate func(ctx context.Context, rt registry.Route) error |
| } |
| // InboundAny and InboundNone are the closed vocabulary ReconcileConfig.Inbound |
| @@ -934,6 +946,17 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // the additivity rule, pinned by TestRecordWireShapeIsAdditive and |
| // TestReconcileMints. |
| nr := routeFor(rec, cfg.DataDir, seed) |
| + // A box that cannot provide this instance's config must not |
| + // admit it: it would start without its secrets, and a keyed |
| + // sibling would be left with a route that lost the race to one |
| + // that was never going to work. Refused before the write, and |
| + // before the dry-run line, so a dry run reports what a real pass |
| + // would do. |
| + if cfg.ConfigGate != nil { |
| + if err := cfg.ConfigGate(ctx, nr); err != nil { |
| + continue |
| + } |
| + } |
| if cfg.DryRun { |
| log.Printf("provision: dry-run: would mint %s route %s (app %q acct %q channel %q)", |
| nr.Kind, rec.Host, rec.App, rec.Account, rec.Channel) |
diff --git a/internal/provision/sidecar_reconcile.go b/internal/provision/sidecar_reconcile.go| index d7f86c8f3..687dd9349 100644 |
| --- a/internal/provision/sidecar_reconcile.go |
| +++ b/internal/provision/sidecar_reconcile.go |
| @@ -387,6 +387,14 @@ func ReconcileSidecars(ctx context.Context, s release.Store, reg *registry.Regis |
| continue |
| } |
| nr := sidecarRouteFor(rec, m, cfg.DataDir) |
| + // Same admission rule as the instance pass (amadan #213): a |
| + // sidecar that would start without its sealed config is not |
| + // minted here. See ReconcileConfig.ConfigGate. |
| + if cfg.ConfigGate != nil { |
| + if err := cfg.ConfigGate(ctx, nr); err != nil { |
| + continue |
| + } |
| + } |
| if cfg.DryRun { |
| log.Printf("provision: dry-run: would mint sidecar route %s (app %q acct %q channel %q args %s)", |
| rec.Host, rec.App, rec.Account, nr.Channel, nr.Args) |