amadan

carlos / platform Public

console: pause native sidecar enable and create outside an allowlist

Native sidecars run on the flat tier as uid carlos with no mount
namespace, so on their box they can read and write every nested
tenant's /data/instances/<iid>. Until nested sidecars land (#370),
Config.SidecarAccounts names the accounts that may still enable or
create one. Set, every other account gets 403 sidecars-paused from
handleAPISidecarEnable and handleAPISidecarCreate, after the auth,
membership and validation checks (the non-member 404 is unchanged) and
before any bucket write, both create arms included. List, get, wake and
delete stay ungated; existing sidecars keep running. Empty is no gate.

The refusal sentence is a single constant, pending copy review.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 263150299008c264197f5ca169b1372d999d8bef parent 65ca4ac
4 files changed, +277 −1
  • internal/console/apisidecars.go +13 −1
  • internal/console/server.go +7 −0
  • internal/console/sidecarsgate.go +64 −0
  • internal/console/sidecarsgate_test.go +193 −0

Intent

full ledger: sec-sidecar-create-gate

diff --git a/internal/console/apisidecars.go b/internal/console/apisidecars.go
index 698b7daa4..17c8aa12d 100644
--- a/internal/console/apisidecars.go
+++ b/internal/console/apisidecars.go
@@ -198,7 +198,8 @@ func (s *Server) handleAPISidecarGet(w http.ResponseWriter, r *http.Request) {
// handleAPISidecarCreate is the point of the feature: one create-only write
// to the bucket, in the family's refusal order — gate, budget, readiness
-// (instanceGate), manifest, name, region, channel, shape, write.
+// (instanceGate), manifest, name, region, channel, shape, pause
+// (sidecarsgate.go), write.
//
// Every field policy governs is stamped from the MANIFEST, never from the
// request. The body has no pool field to lie in, no host field (the host is
@@ -311,6 +312,12 @@ func (s *Server) handleAPISidecarCreate(w http.ResponseWriter, r *http.Request)
jsonError(w, http.StatusBadRequest, err.Error())
return
}
+ // The sidecar pause (sidecarsgate.go): after every check above, so a
+ // stranger still meets the 404 and a bad request its 400, and before
+ // the cap's listing, the leftover rail and either write arm.
+ if s.refuseSidecarsPaused(w, acct, app, "create") {
+ return
+ }
// Plans 3a: the plan caps workers per app, under the account lock.
defer s.planLock(acct)()
if err := s.checkSidecars(r.Context(), acct, app); err != nil {
@@ -586,6 +593,11 @@ func (s *Server) handleAPISidecarEnable(w http.ResponseWriter, r *http.Request)
return
}
}
+ // The sidecar pause (sidecarsgate.go): after authentication, membership
+ // and validation, before the manifest read and its write.
+ if s.refuseSidecarsPaused(w, acct, app, "enable") {
+ return
+ }
ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
defer cancel()
// Read WITH the etag and write guarded by it below: a read-modify-write
diff --git a/internal/console/server.go b/internal/console/server.go
index e7c940087..0c186db2f 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -855,6 +855,13 @@ type Config struct {
// wherever the two differ by anything but that label; the relay's DNS
// names are operator-managed and this console cannot discover them.
PlatformDomain string
+ // SidecarAccounts is the operator allowlist of account sqids that may
+ // still enable and create native sidecars (CARLOS_SIDECAR_ACCOUNTS,
+ // normalized by cmd/console: trimmed, lower-cased, each a valid sqid).
+ // Empty is no gate at all, every deployment's shape before the list
+ // existed. Set, every other account meets sidecarsPaused on the two
+ // doors that bring a sidecar into being (sidecarsgate.go).
+ SidecarAccounts []string
}
// Server is the console's http.Handler.
diff --git a/internal/console/sidecarsgate.go b/internal/console/sidecarsgate.go
new file mode 100644
index 000000000..54354cc65
--- /dev/null
+++ b/internal/console/sidecarsgate.go
@@ -0,0 +1,64 @@
+// The sidecar pause: an operator allowlist in front of the two doors that
+// bring a native sidecar into being.
+//
+// Why it exists: a native sidecar runs on the flat tier
+// (activator.ExecBackend.StartWorkerStaged) as the box's carlos uid with no
+// mount namespace, so on its box it can read and write every nested
+// tenant's /data/instances/<iid>. The fix is nested sidecars (amadan
+// discussion #370). Until that lands, an operator lists the accounts that
+// may still enable or create one (CARLOS_SIDECAR_ACCOUNTS) and every other
+// account is refused.
+//
+// The doors, and why there are only two. A sidecar runs only from a record
+// at apps/<sqid>/<app>/sidecars/<name>.json, which the box's sidecar
+// reconcile pass (or, under host claim, the root claim path) turns into a
+// Kind "sidecar" route, and it takes its argv and ceilings from
+// apps/<sqid>/<app>/sidecars.json. The console writes the record in exactly
+// one place (handleAPISidecarCreate, both the CreateSidecarRecord and the
+// host-claim beginClaimCreate arms) and the manifest in exactly one place
+// (handleAPISidecarEnable). Both call refuseSidecarsPaused. Ship and promote
+// publish releases and channel pointers, never a sidecar manifest; restore
+// clears an app's trash marker and brings back only sidecars that already
+// existed; carlos run needs a nested instance. Store mode (`carlos sidecars
+// enable` against the bucket) is the operator's own bucket credential and
+// is not a console door.
+//
+// What it leaves alone, on purpose: list, get, wake and delete. Existing
+// sidecars keep running, and an account taken off the list must still be
+// able to retire its own.
+package console
+
+import (
+ "log"
+ "net/http"
+ "slices"
+)
+
+// sidecarsPausedMsg is the one sentence a refused account reads.
+// copy: pending review.
+const sidecarsPausedMsg = "sidecars are paused for this account. existing sidecars keep running, and you can still delete them"
+
+// sidecarsPausedCode is the refusal's machine-readable code, beside the
+// sentence the way plan caps carry "cap".
+const sidecarsPausedCode = "sidecars-paused"
+
+// sidecarsPaused reports whether acct is refused new sidecars: the
+// allowlist is set and does not name the account's sqid.
+func (s *Server) sidecarsPaused(acct *Account) bool {
+ return len(s.cfg.SidecarAccounts) > 0 && !slices.Contains(s.cfg.SidecarAccounts, acct.Sqid)
+}
+
+// refuseSidecarsPaused answers 403 sidecars-paused and reports true when
+// acct may not enable or create a sidecar. Call it AFTER the door's
+// authentication, membership and validation checks, so a non-member still
+// gets the byte-identical 404 and the gate never confirms that an account
+// exists, and BEFORE any bucket write. Request-path refusals are not
+// audited in this console (the plan cap's is not either), so it logs.
+func (s *Server) refuseSidecarsPaused(w http.ResponseWriter, acct *Account, app, verb string) bool {
+ if !s.sidecarsPaused(acct) {
+ return false
+ }
+ log.Printf("console: api sidecars %s %s/%s: refused, account not in CARLOS_SIDECAR_ACCOUNTS", verb, acct.Sqid, app)
+ writeJSON(w, http.StatusForbidden, map[string]any{"error": sidecarsPausedMsg, "code": sidecarsPausedCode})
+ return true
+}
diff --git a/internal/console/sidecarsgate_test.go b/internal/console/sidecarsgate_test.go
new file mode 100644
index 000000000..7f0f90dd0
--- /dev/null
+++ b/internal/console/sidecarsgate_test.go
@@ -0,0 +1,193 @@
+package console
+
+// The sidecar pause (sidecarsgate.go), through the real Server, its router
+// and the cliRoute auth stack the rest of apisidecars_test.go uses. The
+// properties, in the order they would hurt if lost: with the allowlist set,
+// an account it does not name cannot enable or create a sidecar by any
+// credential that reaches those doors, and nothing is written to the bucket
+// when it is refused; a stranger still gets the byte-identical 404, so the
+// gate never confirms an account exists; existing sidecars stay listable,
+// wakeable and deletable; a named account is unaffected; and with the
+// allowlist unset nothing changes at all.
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/svcrole"
+)
+
+// sidecarAppKeys is every bucket key under the fixture app's prefix, sorted: the
+// refused calls must leave it exactly as it was.
+func sidecarAppKeys(t *testing.T, fx *consoleFixture) []string {
+ t.Helper()
+ keys, err := fx.rel.List(context.Background(), release.AppPrefix("bab", "titogo"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ slices.Sort(keys)
+ return keys
+}
+
+func assertSidecarsPaused(t *testing.T, what string, status int, body string) {
+ t.Helper()
+ if status != http.StatusForbidden {
+ t.Fatalf("%s = %d %s, want 403", what, status, body)
+ }
+ var got struct{ Error, Code string }
+ if err := json.Unmarshal([]byte(body), &got); err != nil {
+ t.Fatalf("%s body %q: %v", what, body, err)
+ }
+ if got.Code != "sidecars-paused" || got.Error != sidecarsPausedMsg {
+ t.Fatalf("%s body = %+v, want code sidecars-paused and the paused sentence", what, got)
+ }
+}
+
+// TestSidecarsPausedRefusesEnableAndCreate: the fixture's account (bab) is
+// not on the list, so enable and create are refused for the member, the
+// legacy app token and an instance-role service principal, and the bucket
+// is untouched.
+func TestSidecarsPausedRefusesEnableAndCreate(t *testing.T) {
+ fx, member := instanceFixture(t, "code-side-paused")
+ fx.srv.cfg.SidecarAccounts = []string{"cdef"}
+ u := sidecarsURL(fx, "bab", "titogo")
+
+ // Enable with no manifest yet: refused, and no manifest appears.
+ before := sidecarAppKeys(t, fx)
+ status, _, out := rawReq(t, http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), member, `{"channel":"beta"}`)
+ assertSidecarsPaused(t, "paused enable, no manifest", status, out)
+ if after := sidecarAppKeys(t, fx); !slices.Equal(before, after) {
+ t.Fatalf("a refused enable changed the bucket: %v -> %v", before, after)
+ }
+
+ // An operator-written manifest (store mode, not a console door) is
+ // there; enable still may not rewrite it.
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ manifest, _, err := fx.rel.Get(context.Background(), provision.SidecarManifestKey("bab", "titogo"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ status, _, out = rawReq(t, http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), member, `{"channel":"beta","args":["evil"]}`)
+ assertSidecarsPaused(t, "paused enable over a manifest", status, out)
+ if now, _, err := fx.rel.Get(context.Background(), provision.SidecarManifestKey("bab", "titogo")); err != nil || string(now) != string(manifest) {
+ t.Fatalf("a refused enable rewrote the manifest: %s -> %s (%v)", manifest, now, err)
+ }
+
+ // Create, by every credential the create door admits.
+ appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ svc := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
+ before = sidecarAppKeys(t, fx)
+ for _, c := range []struct{ what, token string }{
+ {"member create", member},
+ {"app token create", appTok},
+ {"instance service principal create", svc},
+ } {
+ status, _, out := rawReq(t, http.MethodPost, u, c.token, `{"name":"bot","region":"ie"}`)
+ assertSidecarsPaused(t, c.what, status, out)
+ }
+ if after := sidecarAppKeys(t, fx); !slices.Equal(before, after) {
+ t.Fatalf("a refused create changed the bucket: %v -> %v", before, after)
+ }
+ if keys, err := fx.rel.List(context.Background(), provision.SidecarsPrefix("bab", "titogo")); err != nil || len(keys) != 0 {
+ t.Fatalf("refused creates left records %v (%v)", keys, err)
+ }
+
+ // The gate comes after membership: a stranger and an unknown account
+ // still get the byte-identical 404, never the 403 that would confirm
+ // the account exists and is paused.
+ want := canonicalNotFound(t)
+ eve := cliToken(t, fx, "eve@keymail.dev", "code-side-paused-eve")
+ for _, c := range []struct{ what, method, url, token, body string }{
+ {"non-member enable", http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), eve, `{}`},
+ {"non-member create", http.MethodPost, u, eve, `{"name":"bot","region":"ie"}`},
+ {"unknown account create", http.MethodPost, sidecarsURL(fx, "zzzz", "titogo"), member, `{"name":"bot","region":"ie"}`},
+ {"unclaimed app create", http.MethodPost, sidecarsURL(fx, "bab", "ghost-app"), member, `{"name":"bot","region":"ie"}`},
+ } {
+ status, h, out := rawReq(t, c.method, c.url, c.token, c.body)
+ assertNotFound(t, c.what, want, status, h, out)
+ }
+ // And validation still answers first for a member.
+ if status, _, out := rawReq(t, http.MethodPost, u, member, `{"name":"-nope-","region":"ie"}`); status != http.StatusBadRequest {
+ t.Fatalf("paused create with a bad name = %d %s, want the 400 first", status, out)
+ }
+}
+
+// TestSidecarsPausedLeavesExistingSidecarsAlone: a sidecar created before the
+// pause is still listed, read, woken and deleted by the paused account, by
+// the member and by the instance service principal.
+func TestSidecarsPausedLeavesExistingSidecarsAlone(t *testing.T) {
+ fx, member := instanceFixture(t, "code-side-paused-existing")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+ for _, name := range []string{"bot", "bot2"} {
+ if status, _, out := rawReq(t, http.MethodPost, u, member, `{"name":"`+name+`","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("create %s before the pause = %d %s", name, status, out)
+ }
+ }
+ fx.srv.cfg.SidecarAccounts = []string{"cdef"}
+ svc := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
+
+ for _, c := range []struct{ what, method, url string }{
+ {"list", http.MethodGet, u},
+ {"get", http.MethodGet, u + "/bot"},
+ {"wake", http.MethodPost, u + "/bot/wake"},
+ } {
+ if status, _, out := rawReq(t, c.method, c.url, member, ""); status != http.StatusOK {
+ t.Fatalf("paused %s = %d %s, want 200", c.what, status, out)
+ }
+ }
+ for _, c := range []struct{ name, token string }{{"bot", member}, {"bot2", svc}} {
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/"+c.name, c.token, ""); status != http.StatusOK {
+ t.Fatalf("paused delete of %s = %d %s, want 200", c.name, status, out)
+ }
+ if _, _, err := fx.rel.Get(context.Background(), provision.SidecarRecordKey("bab", "titogo", c.name)); err == nil {
+ t.Fatalf("the record for %s survived its delete", c.name)
+ }
+ }
+}
+
+// TestSidecarsPauseAllowlistAndUnset: an account the list names, and a
+// console with no list at all, enable and create exactly as before.
+func TestSidecarsPauseAllowlistAndUnset(t *testing.T) {
+ for _, c := range []struct {
+ what string
+ accts []string
+ }{
+ {"allowlisted", []string{"cdef", "bab"}},
+ {"unset", nil},
+ } {
+ t.Run(c.what, func(t *testing.T) {
+ fx, member := instanceFixture(t, "code-side-allow-"+c.what)
+ fx.srv.cfg.SidecarAccounts = c.accts
+ status, _, out := rawReq(t, http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), member, `{"channel":"stable"}`)
+ if status != http.StatusOK {
+ t.Fatalf("enable = %d %s, want 200", status, out)
+ }
+ // The pool and regions are operator policy the enable door never
+ // sets; put them as the operator would so create can place.
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+ if status, _, out := rawReq(t, http.MethodPost, u, member, `{"name":"bot","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("member create = %d %s, want 201", status, out)
+ }
+ svc := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
+ if status, _, out := rawReq(t, http.MethodPost, u, svc, `{"name":"bot2","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("service principal create = %d %s, want 201", status, out)
+ }
+ for _, name := range []string{"bot", "bot2"} {
+ if _, err := provision.GetSidecarRecord(context.Background(), fx.rel, "bab", "titogo", name); err != nil {
+ t.Fatalf("record %s: %v", name, err)
+ }
+ }
+ })
+ }
+}