amadan

carlos / platform Public

Add the console's sidecar verbs, stamping a channel on every record

Paul Campbell pushed by paul@keymail.dev ab7e01f5b6db274d8ba56add4bfb7400a35dc01f parent 90e89ee
3 files changed, +1428 −0
  • internal/console/apisidecars.go +551 −0
  • internal/console/apisidecars_test.go +856 −0
  • internal/console/authcli.go +21 −0
diff --git a/internal/console/apisidecars.go b/internal/console/apisidecars.go
new file mode 100644
index 000000000..b8637359d
--- /dev/null
+++ b/internal/console/apisidecars.go
@@ -0,0 +1,551 @@
+// The CLI's sidecar provisioning API: the verbs that turn "run a member's
+// bot" into a data write in the deployment bucket, so an app process holding
+// no AWS credentials and no box access can provision one per enrolled member
+// in seconds. The record it writes is the whole authorization; the agent's
+// sidecar reconcile pass (internal/provision/sidecar_reconcile.go) is what
+// turns it into a route, on its own box only.
+//
+// Everything structural here is the instance family's, deliberately: the
+// GATE is instanceGate verbatim (apiinstances.go), so the SAME two bearer
+// shapes reach it — a member's user token, or an APP token bound to one
+// account and one app at mint time — with the same ordering (principal,
+// budget, readiness) and the same byte-identical 404 on every refusal before
+// the principal is proven. That reuse is the point rather than a shortcut:
+// Eleven's process calls this door with the CARLOS_INSTANCE_TOKEN it already
+// holds, and a second hand-rolled gate would be a second place for the
+// ordering to drift. `apidomains.go` shares it for the identical reason.
+//
+// Three things are NOT the instance family's, and each is a consequence of
+// what a sidecar is (design: sidecar members):
+//
+// - There is no host in the request and no domain in the manifest. A
+// sidecar's host is DERIVED — provision.SidecarHost(name, app, sqid),
+// under the reserved, never-resolvable .sidecar.internal suffix — so
+// there is no namespace for a caller to reach into and no sibling-apex
+// hazard to refuse: the app and account are IN the host. The only thing
+// the request names is one DNS label.
+// - There is no PATCH. A sidecar has no upstream set to repoint, which is
+// the one thing an instance record's update verb exists for. The family
+// is create-only plus delete, which is also what makes CreateSidecarRecord's
+// lost race a 409 rather than a silent overwrite.
+// - CHANNEL is stamped on every record, always. Task 9's reconcile pass
+// PROTECTS a channel-less sidecar record — it is deliberately never
+// minted, because a route with no channel cannot resolve a release and
+// so has no build to exec. A record written without one would be
+// silently inert: minted by nothing, with no error anywhere. That makes
+// this door load-bearing for it (see handleAPISidecarCreate).
+//
+// Spec: docs/superpowers/specs/2026-08-17-sidecar-members-design.md §11 (the
+// verbs), §4 (the shapes), §7.4 (the wake), §8 (placement).
+package console
+
+import (
+ "context"
+ "errors"
+ "log"
+ "net/http"
+ "os"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// sidecarListMax caps one list answer, the instance family's number for the
+// same reason: a sidecar fleet is one row per enrolled member and can be
+// large, and the response says `truncated` rather than silently serving a
+// prefix.
+const sidecarListMax = 500
+
+// The manifest's argv bounds (design §4.1). Args becomes the argv the
+// activator execs the app's OWN binary with, so the bound here is manifest
+// hygiene rather than privilege — the caller already controls that binary
+// via ship — but a manifest is read forever, and an unbounded argv written
+// once would be permanent. Same numbers as instanceSeedMaxArgs for the same
+// reason, kept as their own constants so a change to one family's bound is
+// never silently a change to the other's.
+const (
+ sidecarArgsMax = 16
+ sidecarArgMaxLen = 256
+)
+
+// sidecarMaxSeconds bounds the two duration knobs. Thirty days is far past
+// any honest idle or max-run and short of the values that read as "never" —
+// a manifest is read forever, so a nonsense value written once outlives
+// whoever typed it.
+const sidecarMaxSeconds = 30 * 24 * 60 * 60
+
+// sidecarManifest reads the app's sidecar manifest and answers the caller
+// itself when it cannot be had. ok=false means a response has already been
+// written.
+//
+// CREATE is its only caller, exactly as instanceManifest's is: the manifest
+// is policy about what may be BROUGHT INTO existence, and reads, wakes and
+// deletes are about what already exists, which no version of it can retract.
+//
+// The two failures are kept apart on purpose. ABSENCE is an answer — the
+// feature does not exist for this app — while a read failure is the bucket
+// being unreachable. Reporting a hiccup as "sidecars are not enabled" would
+// tell an app its feature had been switched off. The 409 names SIDECARS' own
+// enable verb: an app with instances enabled and sidecars not is the ordinary
+// state, so pointing at the other family's verb would be actively wrong.
+func (s *Server) sidecarManifest(ctx context.Context, w http.ResponseWriter, acct *Account, app string) (*provision.SidecarManifest, bool) {
+ m, err := provision.GetSidecarManifest(ctx, s.cfg.Releases, acct.Sqid, app)
+ switch {
+ case err == nil:
+ return m, true
+ case errors.Is(err, os.ErrNotExist):
+ // Membership (or binding) is already proven, so this may name its
+ // cause and the act that fixes it.
+ jsonError(w, http.StatusConflict,
+ "sidecars are not enabled for "+app+" — enable them first (carlos sidecars enable -app "+app+")")
+ return nil, false
+ default:
+ log.Printf("console: api sidecars manifest %s/%s: %v", acct.Sqid, app, err)
+ jsonError(w, http.StatusBadGateway, "could not read sidecar settings")
+ return nil, false
+ }
+}
+
+// sidecarListRecord wraps provision.SidecarRecord with the console's own
+// running-version join, kept OFF the record for the same reason
+// instanceListRecord keeps staged_version off provision.Record: the record is
+// the wire shape CreateSidecarRecord persists to the bucket, this field is
+// never written there, and giving the record the field would risk a future
+// write path serializing a stale value into stored bytes the join was never
+// meant to touch.
+type sidecarListRecord struct {
+ provision.SidecarRecord
+ // RunningVersion is what the sidecar's WORKER last reported, read from
+ // whichever box carries its route via that box's own
+ // status/boxes/<label>.json (loadBoxJoin, which drops a box that has gone
+ // stale rather than letting it answer as current). For a sidecar this is
+ // specifically the version stamped by the worker's own /v1/ready call
+ // (activator's SetVersions) — the adopt pass never stamps a sidecar row,
+ // because release.Adoptable excludes them — so a sidecar that has never
+ // reported reads EMPTY here and must be rendered "unknown", never as the
+ // channel's desired version. A stamp that overclaims is worse than an
+ // absent one.
+ RunningVersion string `json:"running_version"`
+}
+
+// handleAPISidecarList answers `carlos sidecars ls`: this app's records,
+// name-sorted, capped and honest about the cap, each carrying its
+// best-effort running version.
+func (s *Server) handleAPISidecarList(w http.ResponseWriter, r *http.Request) {
+ _, acct, app, ok := s.instanceGate(w, r)
+ if !ok {
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
+ defer cancel()
+ recs, truncated, err := provision.ListSidecarRecords(ctx, s.cfg.Releases, acct.Sqid, app, sidecarListMax)
+ if err != nil {
+ log.Printf("console: api sidecars list %s/%s: %v", acct.Sqid, app, err)
+ jsonError(w, http.StatusBadGateway, "could not list sidecars")
+ return
+ }
+ join := loadBoxJoin(ctx, s.cfg.Releases, time.Now())
+ out := make([]sidecarListRecord, 0, len(recs))
+ for _, rec := range recs {
+ out = append(out, sidecarListRecord{SidecarRecord: rec, RunningVersion: join.versions[rec.Host]})
+ }
+ writeJSON(w, http.StatusOK, map[string]any{"sidecars": out, "truncated": truncated})
+}
+
+// handleAPISidecarGet answers one record. A name with no record is the
+// byte-identical 404 — a sidecar's existence is not a thing to probe for
+// either, and this way "no such sidecar" and "no such app" read alike.
+func (s *Server) handleAPISidecarGet(w http.ResponseWriter, r *http.Request) {
+ _, acct, app, ok := s.instanceGate(w, r)
+ if !ok {
+ return
+ }
+ name := r.PathValue("name")
+ // Shape-checked before the key builder, not after. A {name} segment
+ // arrives percent-decoded, so "..%2f..%2fsidecars.json" reaches this
+ // handler as a path with separators in it; SidecarRecordKey would happily
+ // join it. A name that cannot name a record is simply not found — the
+ // same answer a name that names no record gets.
+ if !provision.ValidSidecarName(name) {
+ http.NotFound(w, r)
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
+ defer cancel()
+ rec, err := provision.GetSidecarRecord(ctx, s.cfg.Releases, acct.Sqid, app, name)
+ if errors.Is(err, os.ErrNotExist) {
+ http.NotFound(w, r)
+ return
+ }
+ if err != nil {
+ log.Printf("console: api sidecars get %s/%s/%s: %v", acct.Sqid, app, name, err)
+ jsonError(w, http.StatusBadGateway, "could not read the sidecar")
+ return
+ }
+ writeJSON(w, http.StatusOK, rec)
+}
+
+// handleAPISidecarCreate is the point of the feature: one create-only write
+// to the bucket, in the family's refusal order — gate, budget, readiness
+// (instanceGate), manifest, name, region, channel, shape, write.
+//
+// Every field policy governs is stamped from the MANIFEST, never from the
+// request. The body has no pool field to lie in, no host field (the host is
+// derived), and no placement field — that last one deliberately, and it is
+// worth stating why: the reconcile pass resolves placement record-FIRST
+// (sidecarPlacement), so a record that restated the manifest's placement
+// would pin itself against a later change to it, and Task 9's placement
+// evacuation — flip the manifest to no-inbound and the rows move — would
+// stop working for every record this door ever wrote. Leaving the field
+// empty keeps the manifest authoritative, which is what app-wide policy
+// means. It also closes the widening an app token could otherwise reach for:
+// stating "any" on a record under a no-inbound manifest would place a bot on
+// boxes the operator meant to keep it off.
+func (s *Server) handleAPISidecarCreate(w http.ResponseWriter, r *http.Request) {
+ principal, acct, app, ok := s.instanceGate(w, r)
+ if !ok {
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
+ defer cancel()
+ m, ok := s.sidecarManifest(ctx, w, acct, app)
+ if !ok {
+ return
+ }
+ var req struct {
+ Name string `json:"name"`
+ Region string `json:"region"`
+ Channel string `json:"channel"`
+ }
+ if !readJSON(w, r, &req) {
+ return
+ }
+ name := strings.TrimSpace(req.Name)
+ // Path-traversal defence stated as such: this name becomes a bucket key
+ // segment here and, on the box, a segment of the derived host — which is
+ // itself the <host>.db and cgroup path segment. Nothing below runs until
+ // it is one DNS label.
+ if !provision.ValidSidecarName(name) {
+ jsonError(w, http.StatusBadRequest,
+ "invalid name — a sidecar name is a single DNS label (lowercase a-z, 0-9, hyphen, 1-63 chars)")
+ return
+ }
+ region := strings.TrimSpace(req.Region)
+ // No fleet-placement override here, unlike handleAPIInstanceCreate.
+ // appPlacement answers "fleet/<name>", which provision.ValidateSidecarShape
+ // refuses as a region (ValidRegion has no "/" in its charset) — so
+ // inheriting it would write a record that fails its own shape check at
+ // the next door. Where a sidecar runs is the sidecar family's own scope
+ // axes: the manifest's regions/pool, and placement.
+ if !provision.ValidRegion(region) || !instanceRegionAllowed(m.Regions, region) {
+ msg := "invalid region"
+ if len(m.Regions) > 0 {
+ msg = "region " + region + " is not one this app's sidecars run in — allowed: " + strings.Join(m.Regions, ", ")
+ }
+ jsonError(w, http.StatusBadRequest, msg)
+ return
+ }
+ // The channel, resolved request -> manifest -> documented default, and
+ // NEVER left empty. See this file's package comment: the reconcile pass
+ // protects a channel-less record and never mints it, so an empty channel
+ // here is a sidecar that silently never runs.
+ channel := strings.TrimSpace(req.Channel)
+ if channel == "" {
+ if channel = strings.TrimSpace(m.Channel); channel == "" {
+ channel = "stable"
+ }
+ }
+ if !release.ValidChannel(channel) {
+ jsonError(w, http.StatusBadRequest, "invalid channel name")
+ return
+ }
+ rec := provision.SidecarRecord{
+ Schema: 1,
+ Name: name,
+ Host: provision.SidecarHost(name, app, acct.Sqid),
+ App: app,
+ Account: acct.Sqid,
+ Pool: m.Pool,
+ Region: region,
+ Channel: channel,
+ // UTC, like every other timestamp written to the bucket: these
+ // records are read on boxes in other regions.
+ CreatedAt: time.Now().UTC(),
+ CreatedBy: principal,
+ }
+ // The shape rails, at the door — the same door the box's reconcile pass
+ // re-checks them at, because neither side trusts the other about what is
+ // in the bucket. The message is ValidateSidecarShape's own.
+ if err := provision.ValidateSidecarShape(rec); err != nil {
+ jsonError(w, http.StatusBadRequest, err.Error())
+ return
+ }
+ if err := provision.CreateSidecarRecord(ctx, s.cfg.Releases, rec); err != nil {
+ if errors.Is(err, release.ErrCASConflict) {
+ // A lost race against a second create for the same name, or a
+ // plain duplicate. Either way the winner's record stands.
+ jsonError(w, http.StatusConflict, "sidecar "+name+" already exists")
+ return
+ }
+ log.Printf("console: api sidecars create %s/%s/%s: %v", acct.Sqid, app, name, err)
+ jsonError(w, http.StatusBadGateway, "could not create the sidecar — try again")
+ return
+ }
+ s.audit(acct, app, principal, "sidecar-create", "created sidecar "+name+" (cli)")
+ // The record is what a box's sidecar reconcile leg mints a route from, so
+ // this is a box-consumed write (carlos-deploy §3): announce it, and the
+ // sidecar is running seconds after signup instead of on the hour.
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusCreated, rec)
+}
+
+// handleAPISidecarDelete retires a sidecar's provisioning authorization. The
+// record is deleted; the parked database and blobs are deliberately
+// untouched, exactly as an instance delete leaves them — retiring a member's
+// bot is not destroying that member's data. Idempotent: the caller's goal is
+// "not there", and a second delete has reached it.
+func (s *Server) handleAPISidecarDelete(w http.ResponseWriter, r *http.Request) {
+ principal, acct, app, ok := s.instanceGate(w, r)
+ if !ok {
+ return
+ }
+ name := r.PathValue("name")
+ ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
+ defer cancel()
+ // SHAPE ONLY, always — and the manifest is not read here at all. Policy
+ // is CREATE-time ("may this app have sidecars"), and deletion has to
+ // outlive every version of it: an app that retired its manifest still has
+ // records, and those are exactly the records that must be deletable, or
+ // provisioned rows are stranded with nothing left able to retire them.
+ // What deletion actually needs from validation is the traversal defence:
+ // a {name} arrives percent-decoded, so "..%2f..%2fsidecars.json" reaches
+ // here with separators in it and SidecarRecordKey would happily join it.
+ if !provision.ValidSidecarName(name) {
+ jsonError(w, http.StatusBadRequest, "invalid name")
+ return
+ }
+ if err := provision.DeleteSidecarRecord(ctx, s.cfg.Releases, acct.Sqid, app, name); err != nil {
+ log.Printf("console: api sidecars delete %s/%s/%s: %v", acct.Sqid, app, name, err)
+ jsonError(w, http.StatusBadGateway, "could not delete the sidecar — try again")
+ return
+ }
+ // The wake object is this record's, and it goes with it — best effort,
+ // AFTER the authorization is gone and never able to fail the request. An
+ // app shaped like Eleven's has one sidecar per enrolled member, so
+ // leaving these behind would accrue one object per retired member in the
+ // bucket forever. A leftover is harmless in itself (the box's first sight
+ // of a wake object is recorded silently and never replayed —
+ // host/sidecarwake.go), which is exactly why a failure here is logged
+ // rather than reported: the caller's goal is reached either way.
+ if err := s.cfg.Releases.Delete(ctx, release.SidecarWakeKey(acct.Sqid, app, name)); err != nil {
+ log.Printf("console: api sidecars delete %s/%s/%s: wake object: %v", acct.Sqid, app, name, err)
+ }
+ // Audited whether or not an object was actually there to remove:
+ // idempotency makes "deleted" and "already gone" indistinguishable from
+ // the store, and the fact worth recording is that this principal asked
+ // for this sidecar to stop existing.
+ s.audit(acct, app, principal, "sidecar-delete", "deleted sidecar "+name+" (cli)")
+ // Retiring the row is reconcile's work too, and the direction people want
+ // FAST: the announcement is what stops the process in seconds.
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusOK, map[string]any{"result": "deleted"})
+}
+
+// handleAPISidecarWake asks every box serving this sidecar to wake it
+// (design §7.4's second wake source). A sidecar has no socket and nothing
+// dials it, so unlike an instance it has no "next request" to wake on: the
+// signal has to be an object a pull-based box notices on its own.
+//
+// The write goes through release.TouchSidecarWake and nowhere else. Two
+// writers for one generation counter is how that counter ends up written two
+// different ways; the object is CAS-guarded, a lost race is success (the
+// signal has already fired), and change detection downstream is by ETAG, not
+// by the counter's value.
+//
+// A name with no record is the byte-identical 404, and the record read is
+// what stands between an app token and an unbounded write: without it, any
+// name at all would mint a wake object for a sidecar that does not exist.
+func (s *Server) handleAPISidecarWake(w http.ResponseWriter, r *http.Request) {
+ principal, acct, app, ok := s.instanceGate(w, r)
+ if !ok {
+ return
+ }
+ name := r.PathValue("name")
+ if !provision.ValidSidecarName(name) {
+ http.NotFound(w, r)
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
+ defer cancel()
+ rec, err := provision.GetSidecarRecord(ctx, s.cfg.Releases, acct.Sqid, app, name)
+ if errors.Is(err, os.ErrNotExist) {
+ http.NotFound(w, r)
+ return
+ }
+ if err != nil {
+ log.Printf("console: api sidecars wake read %s/%s/%s: %v", acct.Sqid, app, name, err)
+ jsonError(w, http.StatusBadGateway, "could not read the sidecar")
+ return
+ }
+ if err := release.TouchSidecarWake(ctx, s.cfg.Releases, acct.Sqid, app, name,
+ principal, "carlos sidecars wake"); err != nil {
+ log.Printf("console: api sidecars wake %s/%s/%s: %v", acct.Sqid, app, name, err)
+ jsonError(w, http.StatusBadGateway, "could not request the wake — try again")
+ return
+ }
+ s.audit(acct, app, principal, "sidecar-wake", "requested a wake for sidecar "+name+" (cli)")
+ // The wake object is watched on the fast convergence path, which a serial
+ // move is what drives: without this the box would not look until its next
+ // ordinary tick.
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusOK, map[string]any{
+ "result": "waking", "name": name, "host": rec.Host,
+ })
+}
+
+// handleAPISidecarEnable is the member-facing half of `carlos sidecars
+// enable`: the app's opt-in to sidecar provisioning, and the policy every
+// record is checked against.
+//
+// Gated by apiAccountApp DIRECTLY, not the instance gate: enabling is
+// policy, a human act — an app token meets cliUser's 401 here, exactly as it
+// does at the instance token mint and the instances-manifest door. Budgeted
+// on the address-keyed write limiter for the same reason: the principal an
+// enable belongs to is a person.
+//
+// It has NO domain to pin, which is the one security check its instances
+// counterpart is mostly made of. A sidecar host is derived
+// (provision.SidecarHost) under a reserved suffix nothing resolves, so there
+// is no hostname namespace for this door to hand out and no operator-written
+// custom domain for it to overwrite. What it can still do wrong is rewrite
+// operator policy, so POOL and REGIONS ride through untouched in both
+// directions — store mode (`carlos sidecars enable` against the bucket) is
+// where those are set, exactly as with instances.
+func (s *Server) handleAPISidecarEnable(w http.ResponseWriter, r *http.Request) {
+ addr, acct, app, ok := s.apiAccountApp(w, r)
+ if !ok {
+ return
+ }
+ if !s.cli.write.allow(addr, time.Now()) {
+ jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again")
+ return
+ }
+ if s.cfg.Releases == nil {
+ jsonError(w, http.StatusServiceUnavailable,
+ "no deployment bucket on this host — the console is running without one")
+ return
+ }
+ // Every field is a POINTER because "absent" and "empty" are different
+ // requests on the wire (the apiConfigApply precedent): an omitted field
+ // preserves what the manifest already carries, an explicit value replaces
+ // it — including args' explicit [] meaning "clear" and hibernate's
+ // explicit false, which must win over HibernateOrDefault's true.
+ var req struct {
+ Channel *string `json:"channel"`
+ Args *[]string `json:"args"`
+ Placement *string `json:"placement"`
+ IdleSeconds *int `json:"idle_seconds"`
+ MaxRunSeconds *int `json:"max_run_seconds"`
+ Hibernate *bool `json:"hibernate"`
+ }
+ if !readJSON(w, r, &req) {
+ return
+ }
+ if req.Channel != nil && !release.ValidChannel(*req.Channel) {
+ jsonError(w, http.StatusBadRequest, "invalid channel name")
+ return
+ }
+ if req.Placement != nil &&
+ !slices.Contains([]string{"", provision.PlacementAny, provision.PlacementNoInbound}, *req.Placement) {
+ jsonError(w, http.StatusBadRequest,
+ `invalid placement — want "any" or "no-inbound"`)
+ return
+ }
+ if req.Args != nil {
+ if len(*req.Args) > sidecarArgsMax {
+ jsonError(w, http.StatusBadRequest, "args takes at most 16 arguments")
+ return
+ }
+ for _, arg := range *req.Args {
+ if arg == "" || len(arg) > sidecarArgMaxLen {
+ jsonError(w, http.StatusBadRequest, "each argument must be 1-256 bytes")
+ return
+ }
+ }
+ }
+ for _, d := range []struct {
+ what string
+ v *int
+ }{{"idle_seconds", req.IdleSeconds}, {"max_run_seconds", req.MaxRunSeconds}} {
+ if d.v != nil && (*d.v < 0 || *d.v > sidecarMaxSeconds) {
+ jsonError(w, http.StatusBadRequest, d.what+" must be between 0 and 2592000 (30 days); 0 means the default")
+ return
+ }
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout)
+ defer cancel()
+ // Read WITH the etag and write guarded by it below: a read-modify-write
+ // through the upsert (PutSidecarManifest re-reads inside) would silently
+ // swallow an operator's store-mode write landing in the same window,
+ // which is exactly the pool/regions policy this door promises not to
+ // touch.
+ m, etag, err := provision.GetSidecarManifestETag(ctx, s.cfg.Releases, acct.Sqid, app)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ m, etag = &provision.SidecarManifest{Schema: 1, App: app}, ""
+ case err != nil:
+ log.Printf("console: api sidecars enable read %s/%s: %v", acct.Sqid, app, err)
+ jsonError(w, http.StatusBadGateway, "could not read sidecar settings")
+ return
+ }
+ // Recorded debt, the same the instances enable door carries: fields ADDED
+ // to schema 1 after this binary would be dropped by this
+ // read-modify-write, so an additive manifest field needs the console
+ // upgraded before members enable through it.
+ if req.Channel != nil {
+ m.Channel = *req.Channel
+ }
+ if req.Args != nil {
+ m.Args = *req.Args
+ }
+ if req.Placement != nil {
+ m.Placement = *req.Placement
+ }
+ if req.IdleSeconds != nil {
+ m.IdleSeconds = *req.IdleSeconds
+ }
+ if req.MaxRunSeconds != nil {
+ m.MaxRunSeconds = *req.MaxRunSeconds
+ }
+ if req.Hibernate != nil {
+ m.Hibernate = req.Hibernate
+ }
+ // App is stamped from the PATH, never from the body — there is no field
+ // for it above, and a manifest whose App disagreed with its own key would
+ // be a record family that lies about its identity.
+ m.Schema, m.App = 1, app
+ if err := provision.PutSidecarManifestIfMatch(ctx, s.cfg.Releases, acct.Sqid, app, *m, etag); err != nil {
+ if errors.Is(err, release.ErrCASConflict) {
+ jsonError(w, http.StatusConflict,
+ "sidecar settings changed while this request was in flight — try again")
+ return
+ }
+ log.Printf("console: api sidecars enable %s/%s: %v", acct.Sqid, app, err)
+ jsonError(w, http.StatusBadGateway, "could not enable sidecars — try again")
+ return
+ }
+ s.audit(acct, app, addr, "sidecar-enable", "enabled sidecars for "+app+" (cli)")
+ // The manifest is what the reconcile pass converges args, ceilings,
+ // hibernate and placement from on every pass, so a change here changes
+ // what boxes do — announce it.
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusOK, map[string]any{
+ "app": app, "pool": m.Pool, "channel": m.Channel, "args": m.Args,
+ "placement": m.PlacementOrDefault(), "regions": m.Regions,
+ "idle_seconds": m.IdleSeconds, "max_run_seconds": m.MaxRunSeconds,
+ "hibernate": m.HibernateOrDefault(),
+ })
+}
diff --git a/internal/console/apisidecars_test.go b/internal/console/apisidecars_test.go
new file mode 100644
index 000000000..476ebc7ba
--- /dev/null
+++ b/internal/console/apisidecars_test.go
@@ -0,0 +1,856 @@
+package console
+
+// The sidecar API (apisidecars.go): the console's provisioning verbs for a
+// member that nothing can dial, and the app-held token that may speak to
+// them. The properties that matter, in the order they'd hurt if lost: every
+// refusal before the principal is proven is the instance family's
+// byte-identical 404 (a probe learns nothing — not whether an account
+// exists, not whether an app does, not whether a SIDECAR does), an app token
+// is contained to its own account+app and can enable nothing, every created
+// record carries a non-empty channel (a channel-less record is protected and
+// never minted by the reconcile pass — silently inert, with no error
+// anywhere), no name reaches a bucket key without passing the label check,
+// and the rate limiter is consulted only AFTER the gate so a 429 can never
+// confirm what a 404 hides.
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// sidecarsURL and sidecarEnableURL are the two prefixes this family answers
+// on: the collection, and the literal manifest sibling beside it.
+func sidecarsURL(fx *consoleFixture, acct, app string) string {
+ return fx.ts.URL + "/api/cli/apps/" + acct + "/" + app + "/sidecars"
+}
+
+func sidecarEnableURL(fx *consoleFixture, acct, app string) string {
+ return fx.ts.URL + "/api/cli/apps/" + acct + "/" + app + "/sidecars-manifest"
+}
+
+// enableSidecars writes the app's sidecar manifest directly — the operator
+// half of `carlos sidecars enable`, whose ABSENCE is what makes the feature
+// not exist for an app.
+func enableSidecars(t *testing.T, fx *consoleFixture, app string, m provision.SidecarManifest) {
+ t.Helper()
+ if err := provision.PutSidecarManifest(context.Background(), fx.rel, release.OpsAccount, app, m); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func titoSidecarManifest() provision.SidecarManifest {
+ return provision.SidecarManifest{Schema: 1, App: "titogo", Pool: "tito",
+ Channel: "stable", Regions: []string{"ie"}, Args: []string{"sidecar", "run"}}
+}
+
+// botHost is the synthetic host the door derives for a sidecar of the
+// fixture's app. Spelled out rather than computed so a change to
+// SidecarHost's shape fails a test rather than silently agreeing with itself.
+const botHost = "bot.titogo.bab.sidecar.internal"
+
+// TestAPISidecarRefusalMatrix: every way of not being allowed here — a
+// stranger's token, an app token pointed at a neighbouring account or a
+// neighbouring app, an account that does not exist, an app that was never
+// claimed — answers with the SAME bytes http.NotFound writes, on every verb
+// including wake. Asserted against the stdlib's own output.
+func TestAPISidecarRefusalMatrix(t *testing.T) {
+ fx, member := instanceFixture(t, "code-side-refusal")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ want := canonicalNotFound(t)
+
+ now := time.Now()
+ if err := fx.st.CreateAccount(42, "cdef", "widgets", "bob@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fx.st.CreateAppIn(42, "titogo", "bob@keymail.dev", now); err != nil {
+ t.Fatal(err)
+ }
+ appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ eve := cliToken(t, fx, "eve@keymail.dev", "code-side-eve")
+
+ body := `{"name":"bot","region":"ie"}`
+ cases := []struct {
+ what string
+ method string
+ url string
+ token string
+ body string
+ }{
+ {"non-member list", http.MethodGet, sidecarsURL(fx, "bab", "titogo"), eve, ""},
+ {"non-member create", http.MethodPost, sidecarsURL(fx, "bab", "titogo"), eve, body},
+ {"non-member get", http.MethodGet, sidecarsURL(fx, "bab", "titogo") + "/bot", eve, ""},
+ {"non-member delete", http.MethodDelete, sidecarsURL(fx, "bab", "titogo") + "/bot", eve, ""},
+ {"non-member wake", http.MethodPost, sidecarsURL(fx, "bab", "titogo") + "/bot/wake", eve, ""},
+ {"app token, foreign account", http.MethodGet, sidecarsURL(fx, "cdef", "titogo"), appTok, ""},
+ {"app token, foreign account create", http.MethodPost, sidecarsURL(fx, "cdef", "titogo"), appTok, body},
+ {"app token, foreign account wake", http.MethodPost, sidecarsURL(fx, "cdef", "titogo") + "/bot/wake", appTok, ""},
+ {"app token, foreign app", http.MethodGet, sidecarsURL(fx, "bab", "other"), appTok, ""},
+ {"app token, foreign app create", http.MethodPost, sidecarsURL(fx, "bab", "other"), appTok, body},
+ // The account segment resolves by NAME for a user token; an app
+ // token holds no memberships to resolve one against, so its own
+ // account's name is not a way in either.
+ {"app token, own account by name", http.MethodGet, sidecarsURL(fx, "ops", "titogo"), appTok, ""},
+ {"unknown account", http.MethodGet, sidecarsURL(fx, "zzzz", "titogo"), member, ""},
+ {"unclaimed app", http.MethodGet, sidecarsURL(fx, "bab", "ghost-app"), member, ""},
+ {"unclaimed app create", http.MethodPost, sidecarsURL(fx, "bab", "ghost-app"), member, body},
+ {"unclaimed app wake", http.MethodPost, sidecarsURL(fx, "bab", "ghost-app") + "/bot/wake", member, ""},
+ }
+ for _, c := range cases {
+ status, h, out := rawReq(t, c.method, c.url, c.token, c.body)
+ assertNotFound(t, c.what, want, status, h, out)
+ }
+
+ // A sidecar's EXISTENCE is not a thing to probe for either: a member of
+ // the account asking for a name that was never created gets the same
+ // bytes a stranger gets for the whole app.
+ for _, c := range []struct{ what, method, url string }{
+ {"member, unknown sidecar get", http.MethodGet, sidecarsURL(fx, "bab", "titogo") + "/ghost"},
+ {"member, unknown sidecar wake", http.MethodPost, sidecarsURL(fx, "bab", "titogo") + "/ghost/wake"},
+ {"member, traversal name get", http.MethodGet, sidecarsURL(fx, "bab", "titogo") + "/..%2f..%2fsidecars.json"},
+ {"member, traversal name wake", http.MethodPost, sidecarsURL(fx, "bab", "titogo") + "/..%2f..%2fsidecars.json/wake"},
+ } {
+ status, h, out := rawReq(t, c.method, c.url, member, "")
+ assertNotFound(t, c.what, want, status, h, out)
+ }
+
+ // None of the refused calls wrote anything.
+ if _, _, err := fx.rel.Get(context.Background(),
+ provision.SidecarRecordKey("bab", "titogo", "bot")); err == nil {
+ t.Fatalf("a record exists after only refused creates")
+ }
+ if _, _, err := fx.rel.Get(context.Background(),
+ provision.SidecarRecordKey("cdef", "titogo", "bot")); err == nil {
+ t.Fatalf("a record exists in the neighbouring account after only refused creates")
+ }
+ // And no wake object was minted by a refused wake — the wake write is
+ // strictly behind the gate and behind the record's own existence.
+ if _, _, err := fx.rel.Get(context.Background(),
+ release.SidecarWakeKey("bab", "titogo", "bot")); err == nil {
+ t.Fatalf("a wake object exists after only refused wakes")
+ }
+ if _, _, err := fx.rel.Get(context.Background(),
+ release.SidecarWakeKey("bab", "titogo", "ghost")); err == nil {
+ t.Fatalf("a wake object exists for a sidecar that was never created")
+ }
+}
+
+// TestAPISidecarEnableRefusalMatrix: the enable door is apiAccountApp's, so
+// its refusals are that gate's — the byte-identical 404 for a stranger, and
+// a plain 401 for an app token, which CLITokenAddress does not answer for at
+// all. Enabling is a human act.
+func TestAPISidecarEnableRefusalMatrix(t *testing.T) {
+ fx, member := instanceFixture(t, "code-side-enable-ref")
+ want := canonicalNotFound(t)
+ eve := cliToken(t, fx, "eve@keymail.dev", "code-side-enable-eve")
+ appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range []struct{ what, acct, app, token string }{
+ {"non-member", "bab", "titogo", eve},
+ {"unknown account", "zzzz", "titogo", member},
+ {"unclaimed app", "bab", "ghost-app", member},
+ } {
+ status, h, out := rawReq(t, http.MethodPut, sidecarEnableURL(fx, c.acct, c.app), c.token, `{}`)
+ assertNotFound(t, "sidecars enable: "+c.what, want, status, h, out)
+ }
+ if status, _, out := rawReq(t, http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), appTok, `{}`); status != http.StatusUnauthorized {
+ t.Fatalf("app token enabling sidecars = %d %s, want 401", status, out)
+ }
+ if _, _, err := fx.rel.Get(context.Background(),
+ provision.SidecarManifestKey("bab", "titogo")); err == nil {
+ t.Fatalf("a manifest exists after only refused enables")
+ }
+}
+
+// TestAPISidecarNoToken: no credential at all is the 401 every CLI door
+// gives, on all six routes.
+func TestAPISidecarNoToken(t *testing.T) {
+ fx, _ := instanceFixture(t, "code-side-notoken")
+ u := sidecarsURL(fx, "bab", "titogo")
+ for _, c := range []struct{ method, url, body string }{
+ {http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), `{}`},
+ {http.MethodGet, u, ""},
+ {http.MethodPost, u, `{"name":"bot"}`},
+ {http.MethodGet, u + "/bot", ""},
+ {http.MethodDelete, u + "/bot", ""},
+ {http.MethodPost, u + "/bot/wake", ""},
+ } {
+ if status, _, out := rawReq(t, c.method, c.url, "", c.body); status != http.StatusUnauthorized {
+ t.Errorf("%s %s with no token = %d %s, want 401", c.method, c.url, status, out)
+ }
+ }
+}
+
+// TestAPISidecarAppTokenContainment: the app token is contained in both
+// directions. It cannot speak to the verbs a person's token speaks to
+// (promote, minting, ENABLING), and it CAN do exactly the five record verbs
+// on its own app — which is the whole "apps provision sidecars easily"
+// requirement: Eleven's process, holding no AWS credentials and no box
+// access, creates a bot per enrolled member.
+func TestAPISidecarAppTokenContainment(t *testing.T) {
+ fx, _ := instanceFixture(t, "code-side-contain")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ u := sidecarsURL(fx, "bab", "titogo")
+
+ // The human-only doors.
+ if status, _, out := rawReq(t, http.MethodPost,
+ fx.ts.URL+"/api/cli/apps/bab/titogo/promote", appTok,
+ `{"version":"1.0.0","channel":"canary/a"}`); status != http.StatusUnauthorized {
+ t.Fatalf("app token on promote = %d %s, want 401", status, out)
+ }
+ if status, _, out := rawReq(t, http.MethodPost,
+ fx.ts.URL+"/api/cli/apps/bab/titogo/instance-token", appTok, `{}`); status != http.StatusUnauthorized {
+ t.Fatalf("app token minting a token = %d %s, want 401", status, out)
+ }
+ if status, _, out := rawReq(t, http.MethodPut,
+ sidecarEnableURL(fx, "bab", "titogo"), appTok, `{"channel":"beta"}`); status != http.StatusUnauthorized {
+ t.Fatalf("app token enabling sidecars = %d %s, want 401", status, out)
+ }
+
+ // Its own app's verbs work, all five.
+ status, _, out := rawReq(t, http.MethodPost, u, appTok, `{"name":"bot","region":"ie"}`)
+ if status != http.StatusCreated {
+ t.Fatalf("app token create on its own app = %d %s, want 201", status, out)
+ }
+ var rec provision.SidecarRecord
+ if err := json.Unmarshal([]byte(out), &rec); err != nil {
+ t.Fatalf("create body %s: %v", out, err)
+ }
+ if rec.CreatedBy != "app:bab/titogo" {
+ t.Fatalf("created_by = %q, want the app principal app:bab/titogo", rec.CreatedBy)
+ }
+ if status, _, out := rawReq(t, http.MethodGet, u, appTok, ""); status != http.StatusOK {
+ t.Fatalf("app token list = %d %s, want 200", status, out)
+ }
+ if status, _, out := rawReq(t, http.MethodGet, u+"/bot", appTok, ""); status != http.StatusOK {
+ t.Fatalf("app token get = %d %s, want 200", status, out)
+ }
+ if status, _, out := rawReq(t, http.MethodPost, u+"/bot/wake", appTok, ""); status != http.StatusOK {
+ t.Fatalf("app token wake = %d %s, want 200", status, out)
+ }
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/bot", appTok, ""); status != http.StatusOK {
+ t.Fatalf("app token delete = %d %s, want 200", status, out)
+ }
+}
+
+// TestAPISidecarCreateWithoutManifestNamesTheEnableVerb: membership is
+// already proven at this point, so the refusal may name its cause and the
+// act that fixes it — and it must name SIDECARS' own verb, never the
+// instance family's.
+func TestAPISidecarCreateWithoutManifestNamesTheEnableVerb(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-nomanifest")
+ status, _, out := rawReq(t, http.MethodPost, sidecarsURL(fx, "bab", "titogo"), token, `{"name":"bot"}`)
+ if status != http.StatusConflict {
+ t.Fatalf("create with no manifest = %d %s, want 409", status, out)
+ }
+ if !strings.Contains(out, "carlos sidecars enable") {
+ t.Fatalf("no-manifest refusal = %s, want it to name `carlos sidecars enable`", out)
+ }
+ if strings.Contains(out, "instances enable") {
+ t.Fatalf("no-manifest refusal names another family's verb: %s", out)
+ }
+ // An instances manifest is not a sidecars manifest: enabling one must
+ // not enable the other.
+ enableInstances(t, fx, "titogo", titoManifest())
+ if status, _, out := rawReq(t, http.MethodPost, sidecarsURL(fx, "bab", "titogo"), token,
+ `{"name":"bot"}`); status != http.StatusConflict {
+ t.Fatalf("create with only an INSTANCES manifest = %d %s, want 409", status, out)
+ }
+}
+
+// TestAPISidecarCreateRefusalOrder walks the family's order with a member
+// token: membership is already proven, so from here on refusals may name
+// their cause — and each one must be the RIGHT cause, in the right order.
+func TestAPISidecarCreateRefusalOrder(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-create")
+ u := sidecarsURL(fx, "bab", "titogo")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+
+ for _, c := range []struct{ what, body string }{
+ {"empty name", `{"name":"","region":"ie"}`},
+ {"bad name shape", `{"name":"-nope-","region":"ie"}`},
+ {"dotted name", `{"name":"bot.eleven","region":"ie"}`},
+ {"traversal name", `{"name":"../../etc/passwd","region":"ie"}`},
+ {"uppercase name", `{"name":"Bot","region":"ie"}`},
+ {"region outside the allow-list", `{"name":"bot","region":"us"}`},
+ {"malformed region", `{"name":"bot","region":"IE/../x"}`},
+ {"bad channel", `{"name":"bot","region":"ie","channel":"../etc"}`},
+ } {
+ status, _, out := rawReq(t, http.MethodPost, u, token, c.body)
+ if status != http.StatusBadRequest {
+ t.Errorf("%s = %d %s, want 400", c.what, status, out)
+ }
+ }
+ // Nothing above reached the bucket.
+ keys, err := fx.rel.List(context.Background(), provision.SidecarsPrefix("bab", "titogo"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(keys) != 0 {
+ t.Fatalf("refused creates left %v in the bucket", keys)
+ }
+
+ // The happy path: the host is DERIVED (nothing the body says can name
+ // it), pool comes from the manifest, channel defaults from it, and the
+ // record comes back whole.
+ status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie","pool":"attacker","host":"evil.example"}`)
+ if status != http.StatusCreated {
+ t.Fatalf("create = %d %s, want 201", status, out)
+ }
+ var rec provision.SidecarRecord
+ if err := json.Unmarshal([]byte(out), &rec); err != nil {
+ t.Fatalf("create body %s: %v", out, err)
+ }
+ if rec.Schema != 1 || rec.Name != "bot" || rec.Host != botHost ||
+ rec.App != "titogo" || rec.Account != "bab" || rec.Pool != "tito" ||
+ rec.Region != "ie" || rec.Channel != "stable" {
+ t.Fatalf("record = %+v", rec)
+ }
+ if rec.CreatedBy != "alice@keymail.dev" {
+ t.Fatalf("created_by = %q, want the member's address", rec.CreatedBy)
+ }
+ // Placement stays EMPTY on the record: it is app-wide manifest policy,
+ // and a record that restated it would pin itself against a later
+ // manifest change (the reconcile pass prefers the record's spelling, so
+ // stamping it here would break placement evacuation).
+ if rec.Placement != "" {
+ t.Fatalf("placement = %q, want empty — placement is the manifest's to state", rec.Placement)
+ }
+
+ // It is in the bucket at exactly the key the reconciler will look for.
+ stored, err := provision.GetSidecarRecord(context.Background(), fx.rel, "bab", "titogo", "bot")
+ if err != nil {
+ t.Fatalf("stored record: %v", err)
+ }
+ if stored.CreatedBy != "alice@keymail.dev" || stored.Host != botHost {
+ t.Fatalf("stored record = %+v", stored)
+ }
+ if stored.CreatedAt.IsZero() {
+ t.Fatalf("stored record has no created_at: %+v", stored)
+ }
+
+ // An explicit channel is honoured over the manifest's default.
+ status, _, out = rawReq(t, http.MethodPost, u, token, `{"name":"beta","region":"ie","channel":"canary/beta"}`)
+ if status != http.StatusCreated {
+ t.Fatalf("create with channel = %d %s, want 201", status, out)
+ }
+ if err := json.Unmarshal([]byte(out), &rec); err != nil {
+ t.Fatal(err)
+ }
+ if rec.Channel != "canary/beta" {
+ t.Fatalf("channel = %q, want the requested canary/beta", rec.Channel)
+ }
+}
+
+// TestAPISidecarCreateAlwaysStampsAChannel is the load-bearing one. Task 9's
+// reconcile pass PROTECTS a channel-less sidecar record — it is deliberately
+// never minted, because a route with no channel cannot resolve a release and
+// so has no build to exec. A record created without a channel would be
+// silently inert: minted by nothing, with no error anywhere. This door is
+// therefore the thing that has to stamp it, from the request if valid, else
+// the manifest, else the documented default.
+func TestAPISidecarCreateAlwaysStampsAChannel(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-channel")
+ u := sidecarsURL(fx, "bab", "titogo")
+
+ // A manifest carrying NO channel at all — the shape that would leave a
+ // record channel-less if the door only copied the manifest through.
+ m := titoSidecarManifest()
+ m.Channel = ""
+ m.Regions = nil
+ enableSidecars(t, fx, "titogo", m)
+
+ for _, c := range []struct{ what, body, want string }{
+ {"no channel anywhere", `{"name":"bot"}`, "stable"},
+ {"explicit channel", `{"name":"beta","channel":"canary/beta"}`, "canary/beta"},
+ {"blank channel falls through to the default", `{"name":"blank","channel":" "}`, "stable"},
+ } {
+ status, _, out := rawReq(t, http.MethodPost, u, token, c.body)
+ if status != http.StatusCreated {
+ t.Fatalf("%s: create = %d %s, want 201", c.what, status, out)
+ }
+ var rec provision.SidecarRecord
+ if err := json.Unmarshal([]byte(out), &rec); err != nil {
+ t.Fatalf("%s: create body %s: %v", c.what, out, err)
+ }
+ if rec.Channel != c.want {
+ t.Errorf("%s: channel = %q, want %q", c.what, rec.Channel, c.want)
+ }
+ }
+ // The manifest's channel is used when it has one.
+ m.Channel = "canary/tito"
+ enableSidecars(t, fx, "titogo", m)
+ status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"inherit"}`)
+ if status != http.StatusCreated {
+ t.Fatalf("create = %d %s, want 201", status, out)
+ }
+ var rec provision.SidecarRecord
+ if err := json.Unmarshal([]byte(out), &rec); err != nil {
+ t.Fatal(err)
+ }
+ if rec.Channel != "canary/tito" {
+ t.Errorf("channel = %q, want the manifest's canary/tito", rec.Channel)
+ }
+
+ // EVERY record this door wrote carries a non-empty channel, read back
+ // from the bucket — the reconcile pass reads these bytes, not the
+ // response body.
+ recs, _, err := provision.ListSidecarRecords(context.Background(), fx.rel, "bab", "titogo", 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(recs) != 4 {
+ t.Fatalf("stored records = %d, want 4", len(recs))
+ }
+ for _, r := range recs {
+ if r.Channel == "" {
+ t.Errorf("record %s was stored with no channel — the reconcile pass protects it and never mints it", r.Name)
+ }
+ }
+}
+
+// TestAPISidecarCreateIsCreateOnly: the same name twice is a lost race, not
+// a silent overwrite — the winner's record stands, byte for byte.
+func TestAPISidecarCreateIsCreateOnly(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-createonly")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+ if status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("create = %d %s", status, out)
+ }
+ first, err := provision.GetSidecarRecord(context.Background(), fx.rel, "bab", "titogo", "bot")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie","channel":"canary/other"}`)
+ if status != http.StatusConflict {
+ t.Fatalf("duplicate create = %d %s, want 409", status, out)
+ }
+ if !strings.Contains(out, "already exists") {
+ t.Fatalf("duplicate refusal = %s, want it to say the sidecar already exists", out)
+ }
+ after, err := provision.GetSidecarRecord(context.Background(), fx.rel, "bab", "titogo", "bot")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.Channel != first.Channel || !after.CreatedAt.Equal(first.CreatedAt) {
+ t.Fatalf("the losing create changed the winner's record: %+v then %+v", first, after)
+ }
+}
+
+// TestAPISidecarDeleteIsIdempotent: the caller's goal is "not there", and a
+// second delete has reached it. Deletion also outlives POLICY — an app that
+// retired its sidecar manifest must still be able to clean up the records it
+// left behind, or provisioned rows are stranded with nothing able to retire
+// them.
+func TestAPISidecarDeleteIsIdempotent(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-delete")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+
+ // A name that never existed: still 200.
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/ghost", token, ""); status != http.StatusOK {
+ t.Fatalf("delete of an absent name = %d %s, want an idempotent 200", status, out)
+ }
+ // A hostile name is refused before the key builder, and deletes nothing.
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/..%2f..%2fsidecars.json", token, ""); status != http.StatusBadRequest {
+ t.Fatalf("delete of a traversal name = %d %s, want 400", status, out)
+ }
+ if _, _, err := fx.rel.Get(context.Background(), provision.SidecarManifestKey("bab", "titogo")); err != nil {
+ t.Fatalf("the manifest was destroyed by a traversal delete: %v", err)
+ }
+
+ if status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("create = %d %s", status, out)
+ }
+ // Give it a wake object, so the delete's cleanup has something to do.
+ if status, _, out := rawReq(t, http.MethodPost, u+"/bot/wake", token, ""); status != http.StatusOK {
+ t.Fatalf("wake = %d %s", status, out)
+ }
+ status, _, out := rawReq(t, http.MethodDelete, u+"/bot", token, "")
+ if status != http.StatusOK || !strings.Contains(out, "deleted") {
+ t.Fatalf("delete = %d %s, want 200 deleted", status, out)
+ }
+ if _, _, err := fx.rel.Get(context.Background(),
+ provision.SidecarRecordKey("bab", "titogo", "bot")); err == nil {
+ t.Fatalf("the record survived its delete")
+ }
+ // The wake object goes with it: a per-member sidecar fleet would
+ // otherwise leave one object per retired member in the bucket forever.
+ if _, _, err := fx.rel.Get(context.Background(),
+ release.SidecarWakeKey("bab", "titogo", "bot")); err == nil {
+ t.Fatalf("the wake object outlived the record it belongs to")
+ }
+ // Again: still 200.
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/bot", token, ""); status != http.StatusOK {
+ t.Fatalf("second delete = %d %s, want an idempotent 200", status, out)
+ }
+
+ // Deletion outlives the manifest.
+ if status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"beta","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("create = %d %s", status, out)
+ }
+ if err := fx.rel.Delete(context.Background(), provision.SidecarManifestKey("bab", "titogo")); err != nil {
+ t.Fatal(err)
+ }
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/beta", token, ""); status != http.StatusOK {
+ t.Fatalf("delete with the manifest retired = %d %s, want 200", status, out)
+ }
+ // And the shape check still applies with no manifest to read.
+ if status, _, out := rawReq(t, http.MethodDelete, u+"/..%2f..%2fchannels", token, ""); status != http.StatusBadRequest {
+ t.Fatalf("traversal delete with no manifest = %d %s, want 400", status, out)
+ }
+}
+
+// TestAPISidecarWakeBumpsTheGeneration: two wakes move the generation 1 then
+// 2, the serial moves both times (that is what makes a box act within
+// seconds), and the write goes through release.TouchSidecarWake — the one
+// writer for this object — rather than a second hand-rolled counter.
+func TestAPISidecarWakeBumpsTheGeneration(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-wake")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+ if status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("create = %d %s", status, out)
+ }
+
+ readGen := func() release.SidecarWake {
+ t.Helper()
+ raw, _, err := fx.rel.Get(context.Background(), release.SidecarWakeKey("bab", "titogo", "bot"))
+ if err != nil {
+ t.Fatalf("wake object: %v", err)
+ }
+ var w release.SidecarWake
+ if err := json.Unmarshal(raw, &w); err != nil {
+ t.Fatal(err)
+ }
+ return w
+ }
+ serial := func() int64 {
+ t.Helper()
+ doc, _, err := release.ReadSerial(context.Background(), fx.rel)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return doc.Serial
+ }
+
+ before := serial()
+ if status, _, out := rawReq(t, http.MethodPost, u+"/bot/wake", token, ""); status != http.StatusOK {
+ t.Fatalf("first wake = %d %s, want 200", status, out)
+ }
+ w := readGen()
+ if w.Schema != 1 || w.Generation != 1 {
+ t.Fatalf("after one wake = %+v, want schema 1 generation 1", w)
+ }
+ if w.By != "alice@keymail.dev" {
+ t.Fatalf("wake by = %q, want the principal that asked", w.By)
+ }
+ if w.At == "" || w.Reason == "" {
+ t.Fatalf("wake object carries no at/reason for a human debugging it: %+v", w)
+ }
+ afterFirst := serial()
+ if afterFirst <= before {
+ t.Fatalf("serial %d -> %d: the first wake did not announce itself", before, afterFirst)
+ }
+
+ if status, _, out := rawReq(t, http.MethodPost, u+"/bot/wake", token, ""); status != http.StatusOK {
+ t.Fatalf("second wake = %d %s, want 200", status, out)
+ }
+ if w := readGen(); w.Generation != 2 {
+ t.Fatalf("after two wakes generation = %d, want 2", w.Generation)
+ }
+ if s := serial(); s <= afterFirst {
+ t.Fatalf("serial %d -> %d: the second wake did not announce itself", afterFirst, s)
+ }
+}
+
+// TestAPISidecarEnableChain: the member-facing half of `carlos sidecars
+// enable`. It writes the app's policy, updates preserve the operator-only
+// fields (pool and regions), and its refusals are cheap and named.
+func TestAPISidecarEnableChain(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-enable")
+ e := sidecarEnableURL(fx, "bab", "titogo")
+
+ status, _, out := rawReq(t, http.MethodPut, e, token,
+ `{"channel":"beta","args":["sidecar","run"],"placement":"no-inbound","idle_seconds":300,"max_run_seconds":3600,"hibernate":false}`)
+ if status != http.StatusOK {
+ t.Fatalf("enable = %d %s, want 200", status, out)
+ }
+ m, err := provision.GetSidecarManifest(context.Background(), fx.rel, "bab", "titogo")
+ if err != nil {
+ t.Fatalf("manifest after enable: %v", err)
+ }
+ if m.Schema != 1 || m.App != "titogo" || m.Channel != "beta" ||
+ m.Placement != provision.PlacementNoInbound || m.IdleSeconds != 300 || m.MaxRunSeconds != 3600 {
+ t.Fatalf("manifest = %+v", m)
+ }
+ if len(m.Args) != 2 || m.Args[0] != "sidecar" || m.Args[1] != "run" {
+ t.Fatalf("manifest args = %v", m.Args)
+ }
+ if m.Hibernate == nil || *m.Hibernate {
+ t.Fatalf("hibernate = %v, want an explicit false (absent and false are different facts)", m.Hibernate)
+ }
+
+ // Operator policy rides through an update untouched, and an omitted
+ // field preserves what the manifest already carries.
+ m.Pool, m.Regions = "tito", []string{"ie"}
+ enableSidecars(t, fx, "titogo", *m)
+ if status, _, out := rawReq(t, http.MethodPut, e, token, `{"channel":"stable"}`); status != http.StatusOK {
+ t.Fatalf("update = %d %s, want 200", status, out)
+ }
+ m2, err := provision.GetSidecarManifest(context.Background(), fx.rel, "bab", "titogo")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if m2.Pool != "tito" || len(m2.Regions) != 1 || m2.Regions[0] != "ie" {
+ t.Fatalf("operator policy was rewritten by a member update: %+v", m2)
+ }
+ if m2.Channel != "stable" || m2.Placement != provision.PlacementNoInbound ||
+ m2.IdleSeconds != 300 || len(m2.Args) != 2 || m2.Hibernate == nil || *m2.Hibernate {
+ t.Fatalf("an omitted field did not survive the update: %+v", m2)
+ }
+
+ // The create door can now be used, and the enable announced itself.
+ if status, _, out := rawReq(t, http.MethodPost, sidecarsURL(fx, "bab", "titogo"), token,
+ `{"name":"bot","region":"ie"}`); status != http.StatusCreated {
+ t.Fatalf("create after enable = %d %s, want 201", status, out)
+ }
+
+ // Cheap validation refusals, after the gate.
+ for _, c := range []struct{ what, body string }{
+ {"bad channel", `{"channel":"NOT a channel"}`},
+ {"unknown placement", `{"placement":"anywhere"}`},
+ {"too many args", `{"args":[` + strings.Repeat(`"a",`, 16) + `"a"]}`},
+ {"empty arg", `{"args":[""]}`},
+ {"oversized arg", `{"args":["` + strings.Repeat("a", 257) + `"]}`},
+ {"negative idle", `{"idle_seconds":-1}`},
+ {"negative max run", `{"max_run_seconds":-1}`},
+ {"absurd idle", `{"idle_seconds":999999999}`},
+ } {
+ if status, _, out := rawReq(t, http.MethodPut, e, token, c.body); status != http.StatusBadRequest {
+ t.Errorf("enable %s = %d %s, want 400", c.what, status, out)
+ }
+ }
+}
+
+// TestAPISidecarListAndGet: the list is name-sorted, capped honestly, and
+// carries the best-effort running version joined from the fleet's own box
+// snapshots — a sidecar that has never reported reads empty, never the
+// desired version, because a stamp that overclaims is worse than an absent
+// one.
+func TestAPISidecarListAndGet(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-list")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+ for _, n := range []string{"gamma", "alpha", "beta"} {
+ if status, _, out := rawReq(t, http.MethodPost, u, token,
+ fmt.Sprintf(`{"name":%q,"region":"ie"}`, n)); status != http.StatusCreated {
+ t.Fatalf("create %s = %d %s", n, status, out)
+ }
+ }
+
+ // A fresh box reporting alpha's version, and a STALE one reporting
+ // beta's, which must drop out of the join rather than answer as current.
+ fresh := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
+ fresh.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{
+ "alpha.titogo.bab.sidecar.internal": "abc1234"}}
+ if err := status.WriteBox(context.Background(), fx.rel, fresh); err != nil {
+ t.Fatal(err)
+ }
+ stale := status.CollectBox(nil, status.Probes{MachineID: "box2"}, time.Now().Add(-time.Hour))
+ stale.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{
+ "beta.titogo.bab.sidecar.internal": "deadbeef"}}
+ if err := status.WriteBox(context.Background(), fx.rel, stale); err != nil {
+ t.Fatal(err)
+ }
+
+ code, _, out := rawReq(t, http.MethodGet, u, token, "")
+ if code != http.StatusOK {
+ t.Fatalf("list = %d %s, want 200", code, out)
+ }
+ var list struct {
+ Sidecars []struct {
+ Name string `json:"name"`
+ Host string `json:"host"`
+ Channel string `json:"channel"`
+ RunningVersion string `json:"running_version"`
+ } `json:"sidecars"`
+ Truncated bool `json:"truncated"`
+ }
+ if err := json.Unmarshal([]byte(out), &list); err != nil {
+ t.Fatalf("list body %s: %v", out, err)
+ }
+ if list.Truncated {
+ t.Fatalf("three records reported truncated")
+ }
+ if len(list.Sidecars) != 3 {
+ t.Fatalf("list = %+v, want three records", list.Sidecars)
+ }
+ if list.Sidecars[0].Name != "alpha" || list.Sidecars[2].Name != "gamma" {
+ t.Fatalf("list is not name-sorted: %+v", list.Sidecars)
+ }
+ got := map[string]string{}
+ for _, r := range list.Sidecars {
+ if r.Channel == "" {
+ t.Errorf("%s listed with no channel", r.Name)
+ }
+ got[r.Name] = r.RunningVersion
+ }
+ if got["alpha"] != "abc1234" {
+ t.Errorf("alpha running_version = %q, want abc1234 (its box is fresh)", got["alpha"])
+ }
+ if got["beta"] != "" {
+ t.Errorf("beta running_version = %q, want empty — its only reporting box is stale", got["beta"])
+ }
+ if got["gamma"] != "" {
+ t.Errorf("gamma running_version = %q, want empty — no box has reported it", got["gamma"])
+ }
+
+ // GET one.
+ code, _, out = rawReq(t, http.MethodGet, u+"/alpha", token, "")
+ if code != http.StatusOK {
+ t.Fatalf("get = %d %s, want 200", code, out)
+ }
+ var rec provision.SidecarRecord
+ if err := json.Unmarshal([]byte(out), &rec); err != nil {
+ t.Fatal(err)
+ }
+ if rec.Name != "alpha" || rec.Host != "alpha.titogo.bab.sidecar.internal" {
+ t.Fatalf("get = %+v", rec)
+ }
+}
+
+// TestAPISidecarNoStore: a console running without a deployment bucket
+// answers 503 on every verb rather than panicking on a nil store.
+func TestAPISidecarNoStore(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-nostore")
+ fx.srv.cfg.Releases = nil
+ u := sidecarsURL(fx, "bab", "titogo")
+ for _, c := range []struct{ what, method, url, body string }{
+ {"enable", http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"), `{}`},
+ {"list", http.MethodGet, u, ""},
+ {"create", http.MethodPost, u, `{"name":"bot","region":"ie"}`},
+ {"get", http.MethodGet, u + "/bot", ""},
+ {"delete", http.MethodDelete, u + "/bot", ""},
+ {"wake", http.MethodPost, u + "/bot/wake", ""},
+ } {
+ if status, _, out := rawReq(t, c.method, c.url, token, c.body); status != http.StatusServiceUnavailable {
+ t.Errorf("%s with no bucket = %d %s, want 503", c.what, status, out)
+ }
+ }
+}
+
+// unlistableStore is a bucket whose prefix listing fails — the one store
+// call the list door makes on its own behalf.
+type unlistableStore struct{ release.Store }
+
+func (u unlistableStore) List(context.Context, string) ([]string, error) {
+ return nil, errors.New(brokenStoreErr)
+}
+
+// TestAPISidecarStoreFailure: a bucket that cannot be read or written is a
+// 502, and its error text — bucket names, request ids — never reaches the
+// caller. A read failure is NOT reported as "sidecars are not enabled".
+func TestAPISidecarStoreFailure(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-broken")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+
+ fx.srv.cfg.Releases = brokenStore{fx.rel}
+ status, _, out := rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie"}`)
+ if status != http.StatusBadGateway {
+ t.Fatalf("create against an unwritable bucket = %d %s, want 502", status, out)
+ }
+ if strings.Contains(out, "carlos-prod-secret-bucket") || strings.Contains(out, "ABC123") {
+ t.Fatalf("refusal leaks the store's error text: %s", out)
+ }
+
+ fx.srv.cfg.Releases = unreadableStore{fx.rel}
+ status, _, out = rawReq(t, http.MethodPost, u, token, `{"name":"bot","region":"ie"}`)
+ if status != http.StatusBadGateway {
+ t.Fatalf("create against an unreadable bucket = %d %s, want 502", status, out)
+ }
+ if strings.Contains(out, "sidecars enable") {
+ t.Fatalf("a read failure was reported as the feature being disabled: %s", out)
+ }
+ // A bucket that cannot even be LISTED is the list's own 502. (A bucket
+ // that lists but whose bodies cannot be read answers an empty list, not
+ // an error: provision.ListSidecarRecords skips an unreadable record
+ // rather than failing the page, the same rule ListRecords applies for
+ // instances. Recorded here so the difference is deliberate.)
+ fx.srv.cfg.Releases = unlistableStore{fx.rel}
+ status, _, out = rawReq(t, http.MethodGet, u, token, "")
+ if status != http.StatusBadGateway {
+ t.Fatalf("list against an unlistable bucket = %d %s, want 502", status, out)
+ }
+ if strings.Contains(out, "carlos-prod-secret-bucket") {
+ t.Fatalf("list refusal leaks the store's error text: %s", out)
+ }
+
+ fx.srv.cfg.Releases = undeletableStore{fx.rel}
+ status, _, out = rawReq(t, http.MethodDelete, u+"/bot", token, "")
+ if status != http.StatusBadGateway {
+ t.Fatalf("delete against a bucket that cannot delete = %d %s, want 502", status, out)
+ }
+ if strings.Contains(out, "carlos-prod-secret-bucket") {
+ t.Fatalf("delete refusal leaks the store's error text: %s", out)
+ }
+}
+
+// TestAPISidecarRateLimit: the budget is spent AFTER the gate, so a
+// stranger's refusal never changes shape however many times they ask —
+// which is the gate-before-budget ordering asserted from outside. A limiter
+// consulted first would start answering 429 where the 404 belongs, and that
+// is a difference a prober can measure.
+func TestAPISidecarRateLimit(t *testing.T) {
+ fx, token := instanceFixture(t, "code-side-ratelimit")
+ enableSidecars(t, fx, "titogo", titoSidecarManifest())
+ u := sidecarsURL(fx, "bab", "titogo")
+
+ for i := 0; i < cliInstanceBurst; i++ {
+ body := fmt.Sprintf(`{"name":"bot%d","region":"ie"}`, i)
+ if status, _, out := rawReq(t, http.MethodPost, u, token, body); status != http.StatusCreated {
+ t.Fatalf("create %d = %d %s, want 201 inside the budget", i, status, out)
+ }
+ }
+ if status, _, out := rawReq(t, http.MethodPost, u, token,
+ `{"name":"over","region":"ie"}`); status != http.StatusTooManyRequests {
+ t.Fatalf("create past the budget = %d %s, want 429", status, out)
+ }
+
+ eve := cliToken(t, fx, "eve@keymail.dev", "code-side-rl-eve")
+ want := canonicalNotFound(t)
+ for i := 0; i < cliInstanceBurst+2; i++ {
+ status, h, out := rawReq(t, http.MethodPost, u, eve, `{"name":"x","region":"ie"}`)
+ assertNotFound(t, fmt.Sprintf("non-member probe %d", i), want, status, h, out)
+ if t.Failed() {
+ break
+ }
+ }
+ // The write budget the ENABLE door uses is a different one: the member
+ // above spent the record budget, not this.
+ if status, _, out := rawReq(t, http.MethodPut, sidecarEnableURL(fx, "bab", "titogo"),
+ token, `{"channel":"stable"}`); status != http.StatusOK {
+ t.Fatalf("enable after spending the record budget = %d %s, want 200", status, out)
+ }
+}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 78b7de62c..1da104df5 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -284,6 +284,27 @@ func (s *Server) registerCLIRoutes() {
s.mux.HandleFunc("PATCH /api/cli/apps/{account}/{app}/instances/{host}", s.handleAPIInstanceUpdate)
s.mux.HandleFunc("DELETE /api/cli/apps/{account}/{app}/instances/{host}", s.handleAPIInstanceDelete)
s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/instance-token", s.handleAPIInstanceToken)
+ // The sidecar API (apisidecars.go) — the only door onto sidecars/ from
+ // outside a box, sharing the instance family's gate exactly
+ // (instanceGate), so an app process provisions a sidecar per enrolled
+ // member with the CARLOS_INSTANCE_TOKEN it already holds.
+ //
+ // `sidecars-manifest` is the member-facing half of `carlos sidecars
+ // enable`: a literal sibling of sidecars/, never under it, so a sidecar
+ // named "sidecars-manifest" could still never reach it — and it is
+ // gated by apiAccountApp directly, because enabling is a human act.
+ // `wake` is a literal segment UNDER {name}; the mux prefers a literal
+ // regardless of registration order, and provision.ValidSidecarName is
+ // one DNS label, so no sidecar name can ever be "wake"'s path either.
+ s.mux.HandleFunc("PUT /api/cli/apps/{account}/{app}/sidecars-manifest", s.handleAPISidecarEnable)
+ s.mux.HandleFunc("GET /api/cli/apps/{account}/{app}/sidecars", s.handleAPISidecarList)
+ s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/sidecars", s.handleAPISidecarCreate)
+ s.mux.HandleFunc("GET /api/cli/apps/{account}/{app}/sidecars/{name}", s.handleAPISidecarGet)
+ s.mux.HandleFunc("DELETE /api/cli/apps/{account}/{app}/sidecars/{name}", s.handleAPISidecarDelete)
+ // No PATCH: a sidecar has no upstream set to repoint, which is the one
+ // thing an instance record's update verb exists for. The family is
+ // create-only plus delete.
+ s.mux.HandleFunc("POST /api/cli/apps/{account}/{app}/sidecars/{name}/wake", s.handleAPISidecarWake)
// The extra-hostname API (apidomains.go, issue #151) — the only door onto
// domains/ from outside a box, sharing the instance family's gate exactly
// (instanceGate), so titogo's router attaches its customers' hostnames