amadan
relay: console doors and carlos relay enable|status|disable|adopted|rotate (spec §5)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 files changed,
+1428
−19
apimode.go+138 −0docs/site/cli.md+24 −9internal/console/apirelay.go+578 −0internal/console/apirelay_test.go+349 −0internal/console/authcli.go+17 −0relay.go+322 −10
diff --git a/apimode.go b/apimode.go| index fbb91f813..868db69c5 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -1912,6 +1912,144 @@ func (s *apiSession) storeRotate(ctx context.Context, a apiAcct, app string, fin |
| return out, nil |
| } |
| +// ---- the STUN/TURN relay (relay.go's member and operator verbs) ----------- |
| +// |
| +// The reply shapes mirror internal/console/apirelay.go's JSON exactly, and |
| +// they are redeclared here rather than imported for apiStoreReply's own |
| +// reason: this binary decodes what the console sent, and sharing a struct |
| +// would turn a server-side field rename into a client compile error instead |
| +// of the wire question it actually is. |
| +// |
| +// NOTHING KEY-SHAPED CROSSES THESE. The only identity in the reply is the |
| +// username suffix the console issued per region — account, app, incarnation |
| +// and kid — which is the same public half the app's own CARLOS_RELAY carries |
| +// and the thing a member matches a failing call against. The key exists only |
| +// sealed in the app's config bundle, and this CLI never sees it. |
| + |
| +type apiRelayRoute struct { |
| + Host string `json:"host"` |
| + Shape string `json:"shape"` // unit | socket-exec | tcp-exec-file |
| + Delivery string `json:"delivery"` |
| + AckedAt string `json:"acked_at,omitempty"` |
| + Cutover bool `json:"cutover"` |
| +} |
| + |
| +type apiRelayRegion struct { |
| + Region string `json:"region"` |
| + Box string `json:"box"` |
| + HeartbeatAge string `json:"heartbeat_age"` |
| + Current bool `json:"current"` |
| + LeaseLive bool `json:"lease_live"` |
| + Listeners []string `json:"listeners"` |
| + Kids []string `json:"kids"` |
| + Stale bool `json:"stale"` |
| +} |
| + |
| +type apiRelayReply struct { |
| + App string `json:"app"` |
| + Enabled bool `json:"enabled"` |
| + Incarnation string `json:"incarnation,omitempty"` |
| + Regions []string `json:"regions,omitempty"` |
| + Op int64 `json:"op"` |
| + Phase string `json:"phase,omitempty"` |
| + Issued map[string]string `json:"issued,omitempty"` |
| + Routes []apiRelayRoute `json:"routes,omitempty"` |
| + RegionStatus []apiRelayRegion `json:"region_status,omitempty"` |
| + DisablePending []string `json:"disable_pending,omitempty"` |
| +} |
| + |
| +// apiRelayRotateReply is rotate's own answer: a region and a kid, not an |
| +// app's record. |
| +type apiRelayRotateReply struct { |
| + Region string `json:"region"` |
| + Kid string `json:"kid"` |
| + Apps int `json:"apps"` |
| + RetireNoEarlierThan string `json:"retire_no_earlier_than"` |
| +} |
| + |
| +// relayCall is the four app-scoped verbs' one round trip — they all answer |
| +// with the same record shape, so writing the decode four times would only |
| +// create four places for it to drift. in == nil is the GET. |
| +func (s *apiSession) relayCall(ctx context.Context, verb, path string, in any) (apiRelayReply, error) { |
| + var ( |
| + status int |
| + body []byte |
| + err error |
| + ) |
| + if in == nil { |
| + status, body, err = s.getJSON(ctx, path) |
| + } else { |
| + status, body, err = s.postJSON(ctx, path, in) |
| + } |
| + if err != nil { |
| + return apiRelayReply{}, err |
| + } |
| + if status != http.StatusOK { |
| + return apiRelayReply{}, apiErrorFromBody(status, body) |
| + } |
| + var out apiRelayReply |
| + if err := json.Unmarshal(body, &out); err != nil { |
| + return apiRelayReply{}, fmt.Errorf("%s: unreadable response: %w", verb, err) |
| + } |
| + return out, nil |
| +} |
| + |
| +// relayEnable turns the relay on for app, in regions or — with none named — |
| +// every region the deployment arms, in the console's own order. The list is |
| +// omitted entirely when empty, so a bare `carlos relay enable` sends the |
| +// literal `{}` the door documents as its default rather than a null the |
| +// server would have to guess at. |
| +func (s *apiSession) relayEnable(ctx context.Context, a apiAcct, app string, regions []string) (apiRelayReply, error) { |
| + in := struct { |
| + Regions []string `json:"regions,omitempty"` |
| + }{regions} |
| + return s.relayCall(ctx, "relay enable", apiPath(a, app, "/relay"), in) |
| +} |
| + |
| +// relayGet is `carlos relay status`. Its 404 is a real diagnosis ("the relay |
| +// has never been enabled..."), not the membership gate's byte-identical one, |
| +// and apiErrorFromBody surfaces it verbatim like any other server sentence. |
| +func (s *apiSession) relayGet(ctx context.Context, a apiAcct, app string) (apiRelayReply, error) { |
| + return s.relayCall(ctx, "relay status", apiPath(a, app, "/relay"), nil) |
| +} |
| + |
| +func (s *apiSession) relayDisable(ctx context.Context, a apiAcct, app string) (apiRelayReply, error) { |
| + return s.relayCall(ctx, "relay disable", apiPath(a, app, "/relay/disable"), struct{}{}) |
| +} |
| + |
| +// relayAdopted tells the console that host — a TCP-address route this |
| +// platform cannot restart — now runs the identities it last issued. |
| +func (s *apiSession) relayAdopted(ctx context.Context, a apiAcct, app, host string) (apiRelayReply, error) { |
| + in := struct { |
| + Host string `json:"host"` |
| + }{host} |
| + return s.relayCall(ctx, "relay adopted", apiPath(a, app, "/relay/adopted"), in) |
| +} |
| + |
| +// relayRotate is the operator's verb, and the ONE call in this file with no |
| +// account or app in its path: it moves a REGION's issuer pointer. The path is |
| +// therefore a literal, not an apiPath — there is no account segment for |
| +// apiPath to escape into it. A caller who is not on CARLOS_RELAY_OPERATORS |
| +// meets the mux's own 404 rather than a 403, so this client cannot tell |
| +// "not allowed" from "no such console door", and does not try to. |
| +func (s *apiSession) relayRotate(ctx context.Context, region string) (apiRelayRotateReply, error) { |
| + in := struct { |
| + Region string `json:"region"` |
| + }{region} |
| + status, body, err := s.postJSON(ctx, "/api/cli/relay/rotate", in) |
| + if err != nil { |
| + return apiRelayRotateReply{}, err |
| + } |
| + if status != http.StatusOK { |
| + return apiRelayRotateReply{}, apiErrorFromBody(status, body) |
| + } |
| + var out apiRelayRotateReply |
| + if err := json.Unmarshal(body, &out); err != nil { |
| + return apiRelayRotateReply{}, fmt.Errorf("relay rotate: unreadable response: %w", err) |
| + } |
| + return out, nil |
| +} |
| + |
| // ledgerAppend asks the console to CAS-append one entry to name's chain, |
| // stamping writer from the caller's own authenticated identity server-side |
| // (never client-supplied — apiledger.go). A 503 here is contention (too many |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index b1c35a30f..a549e4e00 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -666,23 +666,38 @@ switch is `carlos system antivirus` — see [carlos system](#carlos-system). |
| ### carlos relay |
| -The platform's STUN and TURN relay — the thing that lets a browser-to-browser |
| -call connect across the internet rather than only across a LAN. Two verbs |
| -exist so far, and neither is one an app developer types. |
| +A STUN and TURN relay the platform runs for your app's WebRTC calls, so a |
| +browser can find its public address and, when two networks cannot reach |
| +each other directly, relay through the deployment instead of a third party. |
| ```sh |
| -carlos relay keygen |
| +carlos relay enable --app meet |
| +carlos relay status --app meet |
| ``` |
| +`enable` turns it on for one app and delivers `CARLOS_RELAY` to the app's |
| +config: a base64-wrapped JSON list of regions, each with its ICE server URLs, |
| +a key and a username suffix. The app mints a TURN credential per call from |
| +the key (see the relay design spec, §2). `status` shows what was issued, |
| +which routes have picked it up, and the health of each region's relay, |
| +none of which needs box access. `disable` removes it and ends every |
| +allocation the app holds. `adopted --host <route>` is how a member tells the |
| +platform that a TCP-address route it restarted by hand now runs the new |
| +key. `rotate --region <r>` is the operator's verb after staging a new |
| +master; `keygen` prints one. `serve` is the unit's entry point and not a |
| +thing to run by hand. |
| + |
| `keygen` prints one line — a key id, a key, and `-` for "no retirement date" |
| — for a region's master key. Stage that line in `/etc/carlos/relay-masters` |
| on the region's boxes, mode 0640, root-owned, group `carlos`. The console |
| needs the same key in a different shape: set its |
| -`CARLOS_RELAY_MASTERS_<REGION>` secret to `<kid>:<key>`, or to a |
| -comma-separated list, `<kid>:<key>,<kid>:<key>`, while a rotation has two |
| -live. The region name is uppercased with hyphens as underscores, since |
| -variable names refuse hyphens. The key is printed once and stored nowhere by |
| -this command. |
| +`CARLOS_RELAY_MASTERS_<REGION>` secret to `<kid>:<key>[,<kid>:<key>]`, the |
| +list form being what a rotation needs while two masters are live. The region |
| +name is uppercased with hyphens as underscores, since variable names refuse |
| +hyphens. The key is printed once and stored nowhere by this command. Adding |
| +a master to that secret issues nothing on its own — `rotate` is what moves |
| +the region onto it, and it refuses until every live relay there reports the |
| +new key id, so a rotation cannot mint credentials nothing can verify. |
| `serve` is what `carlos-relay.service` execs. The unit only starts on a box |
| that has a masters file, so a box with no key never listens; a box with one |
diff --git a/internal/console/apirelay.go b/internal/console/apirelay.go| new file mode 100644 |
| index 000000000..13d6e960d |
| --- /dev/null |
| +++ b/internal/console/apirelay.go |
| @@ -0,0 +1,578 @@ |
| +// The CLI's doors onto the platform STUN/TURN relay (relay spec §5): |
| +// `carlos relay enable|status|disable|adopted` for a member, and |
| +// `carlos relay rotate` for a deployment operator. Every write here is a |
| +// thin shell over internal/console/relayops.go's operations — the record, |
| +// the lease, the walk and the reconcile all live there; this file's whole |
| +// job is the door: gate the caller, decode the request, translate the |
| +// operation's sentinel errors into HTTP, and answer with the ONE reply |
| +// shape every member-facing door shares (storeReply's pattern, apistore.go). |
| +// |
| +// FOUR DOORS ARE APP-SCOPED, ONE IS NOT. enable, status, disable and adopted |
| +// sit under .../apps/{account}/{app}/relay, behind apiAccountApp exactly |
| +// like the store doors: an unknown account, someone else's account and an |
| +// app that was never claimed are one byte-identical 404 (apiAccountApp's own |
| +// doc comment). rotate has no account or app in its path at all — it moves a |
| +// REGION's issuer pointer and re-issues every app enabled there, so its gate |
| +// is apiRelayOperator, not apiAccountApp, and its refusal is a byte-identical |
| +// 404 rather than the 403 the store's grant door gives a non-operator MEMBER |
| +// (apistore.go's handleAPIStoreGrant): that 403 is safe to speak only because |
| +// apiAccountApp has already proven the caller belongs to the account it |
| +// named, so it teaches them only what THEY lack. rotate proves no membership |
| +// at all before answering, so a 403 here would confirm to any valid token |
| +// that this deployment runs a relay a caller might rotate — the same |
| +// reasoning apiSystemOwner's family already applies to `carlos system`. |
| +// |
| +// DELIVERY IS DELIBERATELY THIN FOR NOW. Spec §5 describes five delivery |
| +// states a route can be in, most of them read from a per-route config-ack |
| +// (`control/config-ack/<box>/<route host>.json`) that Task 12 has not written |
| +// yet. This file reports only what a route's shape and the record's own |
| +// Cutovers can say today — "adopted by member at <time>" once a member has |
| +// acknowledged the identities that are issued NOW, "written, awaiting |
| +// delivery" otherwise — and marks the place Task 12 fills in with a TODO. It |
| +// does not invent the ack's shape. |
| +// |
| +// Spec: docs/superpowers/specs/2026-09-07-relay-stun-turn-design.md §2, §5. |
| +package console |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "maps" |
| + "net/http" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/relay" |
| +) |
| + |
| +// relayTimeout bounds a single relay round trip: one GET status, one adopted |
| +// ack. The same 15s storeTimeout uses, for the same reason (apistore.go). |
| +const relayTimeout = 15 * time.Second |
| + |
| +// relayOpTimeout bounds enable and disable: each is relayops.go's three-step |
| +// walk (record, config, aggregate) rather than one object read, so it gets |
| +// storeGrantTimeout's ceiling rather than storeTimeout's. |
| +const relayOpTimeout = 60 * time.Second |
| + |
| +// relayRotateTimeout bounds rotate: the issuer pointer move plus a |
| +// re-issue of every app enabled in the region, one at a time. |
| +const relayRotateTimeout = 60 * time.Second |
| + |
| +// relayNoBucket is the 503 every relay door gives when this console has no |
| +// deployment bucket — storeNoBucket's exact reasoning (apistore.go): the |
| +// record IS a bucket object, and answering as if the relay were simply off |
| +// would be a lie a member could act on. |
| +func relayNoBucket(w http.ResponseWriter) { |
| + jsonError(w, http.StatusServiceUnavailable, errRelayNoBucket.Error()) |
| +} |
| + |
| +// relayDoorError turns one of relayops.go's errors into the door's answer. |
| +// The sentinels each name a configuration fact or a genuine 404; anything |
| +// else is logged (a relay error can carry bucket paths and request ids, the |
| +// same reasoning storeGrant's own %v log line gives) and answered with a |
| +// generic retry. |
| +// |
| +// A few of relayops.go's refusals are not sentinels — relayResolveRegions |
| +// and relayRotate build fmt.Errorf sentences for "no such region" and "the |
| +// staged master is not loaded yet" because they always name the specific |
| +// region or kid at fault, which a sentinel compared by value cannot do. They |
| +// are still the CALLER's mistake or something for the operator to fix by |
| +// re-running the runbook, not this console failing, so they are matched by |
| +// their own stable wording rather than left to read as a 502. |
| +func relayDoorError(w http.ResponseWriter, verb string, err error) { |
| + switch { |
| + case errors.Is(err, errRelayNoRegions), errors.Is(err, errRelayNoSealKey), |
| + errors.Is(err, errRelayNotHeld), errors.Is(err, errRelayNoBucket): |
| + jsonError(w, http.StatusServiceUnavailable, err.Error()) |
| + case errors.Is(err, errRelayUnknownHost), errors.Is(err, errRelayNotEnabled): |
| + jsonError(w, http.StatusNotFound, err.Error()) |
| + case errors.Is(err, errRelayUndelivered), relayLooksLikeClientError(err): |
| + jsonError(w, http.StatusBadRequest, err.Error()) |
| + default: |
| + log.Printf("console: api relay %s: %v", verb, err) |
| + jsonError(w, http.StatusBadGateway, "the relay could not complete that — try again") |
| + } |
| +} |
| + |
| +// relayLooksLikeClientError matches the handful of relayResolveRegions' and |
| +// relayRotate's own sentences that name something the CALLER typed wrong or |
| +// the OPERATOR has not finished staging — never this console's own failure. |
| +func relayLooksLikeClientError(err error) bool { |
| + msg := err.Error() |
| + for _, s := range []string{ |
| + "runs no relay in", // relayResolveRegions, relayRotate: unarmed region named |
| + "no new master is staged", // relayRotate: nothing new in CARLOS_RELAY_MASTERS_<REGION> |
| + "has not loaded kid", // relayRotate: a box has not picked up the staged master yet |
| + "has not heartbeated in", // relayRotate: no live relay in the region at all |
| + } { |
| + if strings.Contains(msg, s) { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// ---- the reply shape --------------------------------------------------------- |
| + |
| +// apiRelayRoute is one of an app's routes as `carlos relay status` reports |
| +// it: which of spec §5's delivery shapes it is, and what the console can say |
| +// about whether its key has arrived. |
| +type apiRelayRoute struct { |
| + Host string `json:"host"` |
| + Shape string `json:"shape"` // "unit" | "socket-exec" | "tcp-exec-file" |
| + Delivery string `json:"delivery"` |
| + // AckedAt and Cutover are both empty/false until a member has run |
| + // `carlos relay adopted --host` for this route (relay.RouteCutover). |
| + AckedAt string `json:"acked_at,omitempty"` |
| + Cutover bool `json:"cutover"` |
| +} |
| + |
| +// apiRelayRegion is one region's health as this app's status sees it, built |
| +// from a single armed relay's own heartbeat (control/relay-ack/<box>.json). |
| +// A region with several relays gets one row per box — Box is the reader's |
| +// only way to tell them apart, since Region alone is not unique across them. |
| +type apiRelayRegion struct { |
| + Region string `json:"region"` |
| + Box string `json:"box"` |
| + HeartbeatAge string `json:"heartbeat_age"` // "never", or a Go duration string |
| + Current bool `json:"current"` // this box's authority copy is the latest published |
| + LeaseLive bool `json:"lease_live"` // relay.Lease still armed to serve (15-minute ceiling) |
| + Listeners []string `json:"listeners"` |
| + Kids []string `json:"kids"` |
| + // Stale mirrors relayRotate's own gate: a heartbeat whose last successful |
| + // authority read is older than relayHeartbeatFresh (five minutes) cannot |
| + // be read as current, so a dead relay cannot look alive by silence. |
| + Stale bool `json:"stale"` |
| +} |
| + |
| +// apiRelayReply is the one answer shape enable, status, disable and adopted |
| +// all give — storeReply's pattern (apistore.go): describing the SAME record |
| +// from four different doors in one function is what keeps them from |
| +// describing it differently. rotate is not app-scoped and does not produce |
| +// one of these at all; see apiRelayRotateReply below. |
| +type apiRelayReply struct { |
| + App string `json:"app"` |
| + Enabled bool `json:"enabled"` |
| + Incarnation string `json:"incarnation,omitempty"` |
| + Regions []string `json:"regions,omitempty"` |
| + Op int64 `json:"op"` |
| + Phase string `json:"phase,omitempty"` |
| + // Issued is the username suffix the console last issued per region — |
| + // relay.Record.IssuedSuffixes, echoed verbatim. Not a secret: it is the |
| + // same account/app/incarnation/kid identity §2's CARLOS_RELAY carries, |
| + // never the key. |
| + Issued map[string]string `json:"issued,omitempty"` |
| + Routes []apiRelayRoute `json:"routes,omitempty"` |
| + RegionStatus []apiRelayRegion `json:"region_status,omitempty"` |
| + // DisablePending names the regions where the aggregate publish that |
| + // removed this app has not yet been confirmed by every armed relay's own |
| + // heartbeat (spec §5, "when a disable is complete") — present only while |
| + // the record is disabled and at least one region has not caught up. |
| + DisablePending []string `json:"disable_pending,omitempty"` |
| + // There is deliberately NO retire-no-earlier-than here. That time is a |
| + // property of a ROTATION, not of an app: relayRotate computes it from the |
| + // moment the issuer pointer moved and hands it to the operator who moved |
| + // it (apiRelayRotateReply), and nothing persists it. Reporting a per-region |
| + // retirement date on an app's status would mean either inventing state the |
| + // console does not keep or recomputing it from a clock, which would tell a |
| + // member a retirement deadline that drifts every time they ask. |
| +} |
| + |
| +// apiRelayRotateReply is what `carlos relay rotate` answers with — a region |
| +// and a kid, not an app's record, so it does not try to fit apiRelayReply's |
| +// shape. |
| +type apiRelayRotateReply struct { |
| + Region string `json:"region"` |
| + Kid string `json:"kid"` |
| + Apps int `json:"apps"` |
| + RetireNoEarlierThan string `json:"retire_no_earlier_than"` // RFC3339 |
| +} |
| + |
| +// relayRouteShape names which of spec §5's three delivery shapes a route is, |
| +// in relayDeliverable's OWN precedence (relayops.go), branch for branch: a |
| +// unit wins over a socket, an `instance` route is the platform's own — the |
| +// config sync restarts it whatever else the record says — and only then does |
| +// a bare TCP address read as the shape a member has to adopt by hand. |
| +// |
| +// The instance branch sits ABOVE the address one for the same reason |
| +// relayDeliverable's does: an instance record carries the address the router |
| +// dials, and reading that address as "a TCP process this platform does not |
| +// own" would tell a member to run `carlos relay adopted` for a route the |
| +// platform restarts itself. |
| +func relayRouteShape(rt relayRoute) string { |
| + switch { |
| + case rt.Unit != "": |
| + return "unit" |
| + case rt.Socket != "": |
| + return "socket-exec" |
| + case rt.Kind == "" || rt.Kind == "instance": |
| + return "unit" |
| + case rt.Addr != "": |
| + return "tcp-exec-file" |
| + default: |
| + return "unit" |
| + } |
| +} |
| + |
| +// relayCutoverFor finds host's cutover in rec, nil if a member has never |
| +// acknowledged it. |
| +func relayCutoverFor(rec *relay.Record, host string) *relay.RouteCutover { |
| + for i := range rec.Cutovers { |
| + if rec.Cutovers[i].Host == host { |
| + return &rec.Cutovers[i] |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// relayReply assembles apiRelayReply from a record that has already been |
| +// read or just been written. rec is never nil here — the callers that could |
| +// see one (handleAPIRelayGet) answer their own 404 first. |
| +func (s *Server) relayReply(ctx context.Context, acct *Account, app string, rec *relay.Record) (apiRelayReply, error) { |
| + out := apiRelayReply{ |
| + App: app, Enabled: rec.Enabled, Incarnation: rec.Incarnation, |
| + Regions: rec.Regions, Op: rec.Op, Phase: rec.Phase, Issued: rec.IssuedSuffixes, |
| + } |
| + |
| + routes, err := s.relayAppRoutes(ctx, acct.Sqid, app) |
| + if err != nil { |
| + return apiRelayReply{}, err |
| + } |
| + for _, rt := range routes { |
| + if rt.Kind == "static" { |
| + // A static route runs no process at all — relayDeliverable |
| + // (relayops.go) ignores it for the same reason: there is nothing |
| + // here for a relay key to ever reach. |
| + continue |
| + } |
| + row := apiRelayRoute{Host: rt.Host, Shape: relayRouteShape(rt), Delivery: "written, awaiting delivery"} |
| + if !rec.Enabled { |
| + // A disabled record has no key to deliver: relayDisable removed |
| + // the variable from the bundle. "Written, awaiting delivery" |
| + // would name a write that has been UNDONE, and a member reading |
| + // it would go looking for a key that is on its way out. |
| + row.Delivery = "the relay key was removed from this app's config" |
| + } |
| + if cut := relayCutoverFor(rec, rt.Host); cut != nil && rec.Enabled { |
| + row.AckedAt = cut.At.UTC().Format(time.RFC3339) |
| + // An ack counts only for the identities it NAMED. relayAdopted |
| + // stamps the record's issued suffixes into the cutover for |
| + // exactly this comparison: a rotation since then re-issued the |
| + // app under a new kid, and reporting the older acknowledgement as |
| + // current would say a route runs a key it has never seen. |
| + if maps.Equal(cut.Suffixes, rec.IssuedSuffixes) { |
| + row.Cutover = true |
| + row.Delivery = "adopted by member at " + row.AckedAt |
| + } else { |
| + row.Delivery = "acknowledged at " + row.AckedAt + |
| + ", but the identities have changed since — re-run `carlos relay adopted --host " + rt.Host + "`" |
| + } |
| + } |
| + // TODO(task 12): once internal/appconfig/sync.go writes |
| + // control/config-ack/<box>/<route host>.json, read it here and |
| + // report the fuller set of states spec §5 describes for a unit or |
| + // socket-exec route ("adopted at <time>, restarted", "restart |
| + // failed", "hibernating, delivered on wake") instead of the two |
| + // states above, which is all the console can see without it. |
| + out.Routes = append(out.Routes, row) |
| + } |
| + |
| + hbs, err := s.relayHeartbeats(ctx) |
| + if err != nil { |
| + return apiRelayReply{}, err |
| + } |
| + auth, _, err := relay.ReadAuthority(ctx, s.cfg.Releases) |
| + if err != nil { |
| + return apiRelayReply{}, err |
| + } |
| + now := s.relayTime() |
| + for _, region := range rec.Regions { |
| + for _, hb := range hbs { |
| + if hb.Region != region { |
| + continue |
| + } |
| + row := apiRelayRegion{ |
| + Region: region, Box: hb.Box, Listeners: hb.Listeners, Kids: hb.Kids, |
| + LeaseLive: !hb.ReadAt.IsZero() && now.Sub(hb.ReadAt) <= relay.LeaseMaxAge, |
| + Stale: hb.ReadAt.IsZero() || now.Sub(hb.ReadAt) > relayHeartbeatFresh, |
| + Current: auth != nil && !hb.WrittenAt.IsZero() && !hb.WrittenAt.Before(auth.WrittenAt), |
| + } |
| + if hb.ReadAt.IsZero() { |
| + row.HeartbeatAge = "never" |
| + } else { |
| + row.HeartbeatAge = now.Sub(hb.ReadAt).Round(time.Second).String() |
| + } |
| + out.RegionStatus = append(out.RegionStatus, row) |
| + } |
| + } |
| + |
| + // "When a disable is complete" (spec §5): the app is out of a region's |
| + // authority once that region's own armed relay has heartbeated an |
| + // authority record written at or after the disable. rec.DisabledAt is |
| + // the record's own timestamp for that publish (relayDisable, relayops.go) |
| + // — close enough to the aggregate's own written_at for this purpose, |
| + // since the publish is the very next step in the same walk. |
| + if !rec.Enabled && !rec.DisabledAt.IsZero() { |
| + for _, region := range rec.Regions { |
| + revoked := false |
| + for _, hb := range hbs { |
| + if hb.Region == region && !hb.WrittenAt.IsZero() && !hb.WrittenAt.Before(rec.DisabledAt) { |
| + revoked = true |
| + break |
| + } |
| + } |
| + if !revoked { |
| + out.DisablePending = append(out.DisablePending, region) |
| + } |
| + } |
| + } |
| + return out, nil |
| +} |
| + |
| +// ---- the app-scoped doors: enable, status, disable, adopted ----------------- |
| + |
| +// handleAPIRelayEnable answers `carlos relay enable`. The doors, in order, |
| +// mirror handleAPIStoreCreate's (apistore.go): token 401 (cliRoute), then |
| +// apiAccountApp's byte-identical 404, then Releases nil (503), the write |
| +// budget, and only then the body — a malformed request from a stranger must |
| +// not distinguish itself from one that was never a member to begin with. |
| +func (s *Server) handleAPIRelayEnable(w http.ResponseWriter, r *http.Request) { |
| + addr, acct, app, ok := s.apiAccountApp(w, r) |
| + if !ok { |
| + return |
| + } |
| + if s.cfg.Releases == nil { |
| + relayNoBucket(w) |
| + return |
| + } |
| + p, _ := principalFrom(r.Context()) |
| + if !s.allowWrite(p, time.Now()) { |
| + jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again") |
| + return |
| + } |
| + var req struct { |
| + Regions []string `json:"regions,omitempty"` |
| + } |
| + if !readJSON(w, r, &req) { |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), relayOpTimeout) |
| + defer cancel() |
| + rec, err := s.relayEnable(ctx, acct, app, req.Regions, addr) |
| + if err != nil { |
| + relayDoorError(w, "enable", err) |
| + return |
| + } |
| + s.audit(acct, app, addr, "relay-enable", |
| + "enabled the relay in "+strings.Join(rec.Regions, ",")+" (cli)") |
| + out, err := s.relayReply(ctx, acct, app, rec) |
| + if err != nil { |
| + log.Printf("console: api relay reply %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, |
| + "the relay was enabled, but its status could not be read — try `carlos relay status`") |
| + return |
| + } |
| + writeJSON(w, http.StatusOK, out) |
| +} |
| + |
| +// handleAPIRelayGet answers `carlos relay status` — the one sight a member |
| +// with no box access has of delivery per route and health per region, |
| +// handleAPIStoreGet's exact reasoning (apistore.go): the 404 here speaks only |
| +// after apiAccountApp has already proven membership, so it teaches a |
| +// stranger nothing. Not rate limited, matching every other read door. |
| +func (s *Server) handleAPIRelayGet(w http.ResponseWriter, r *http.Request) { |
| + _, acct, app, ok := s.apiAccountApp(w, r) |
| + if !ok { |
| + return |
| + } |
| + if s.cfg.Releases == nil { |
| + relayNoBucket(w) |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), relayTimeout) |
| + defer cancel() |
| + rec, _, err := relay.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app) |
| + if err != nil { |
| + log.Printf("console: api relay status %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, "could not read the relay record — try again") |
| + return |
| + } |
| + if rec == nil { |
| + // A state, not a failure — objstore's same choice (handleAPIStoreGet): |
| + // "never enabled" and "enabled with nothing filled in" would |
| + // otherwise be indistinguishable to the CLI. |
| + jsonError(w, http.StatusNotFound, "the relay has never been enabled for this app — run `carlos relay enable`") |
| + return |
| + } |
| + out, err := s.relayReply(ctx, acct, app, rec) |
| + if err != nil { |
| + log.Printf("console: api relay delivery %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, "could not read the relay's delivery status — try again") |
| + return |
| + } |
| + writeJSON(w, http.StatusOK, out) |
| +} |
| + |
| +// handleAPIRelayDisable answers `carlos relay disable`. |
| +func (s *Server) handleAPIRelayDisable(w http.ResponseWriter, r *http.Request) { |
| + addr, acct, app, ok := s.apiAccountApp(w, r) |
| + if !ok { |
| + return |
| + } |
| + if s.cfg.Releases == nil { |
| + relayNoBucket(w) |
| + return |
| + } |
| + p, _ := principalFrom(r.Context()) |
| + if !s.allowWrite(p, time.Now()) { |
| + jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again") |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), relayOpTimeout) |
| + defer cancel() |
| + rec, err := s.relayDisable(ctx, acct, app, addr) |
| + if err != nil { |
| + relayDoorError(w, "disable", err) |
| + return |
| + } |
| + s.audit(acct, app, addr, "relay-disable", "disabled the relay (cli)") |
| + out, err := s.relayReply(ctx, acct, app, rec) |
| + if err != nil { |
| + log.Printf("console: api relay reply %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, |
| + "the relay was disabled, but its status could not be read — try `carlos relay status`") |
| + return |
| + } |
| + writeJSON(w, http.StatusOK, out) |
| +} |
| + |
| +// handleAPIRelayAdopted answers `carlos relay adopted --host <route>`: a |
| +// member telling the platform that a TCP-address route they restarted by |
| +// hand now runs the identities the console last issued (spec §5, step 5). |
| +func (s *Server) handleAPIRelayAdopted(w http.ResponseWriter, r *http.Request) { |
| + addr, acct, app, ok := s.apiAccountApp(w, r) |
| + if !ok { |
| + return |
| + } |
| + if s.cfg.Releases == nil { |
| + relayNoBucket(w) |
| + return |
| + } |
| + p, _ := principalFrom(r.Context()) |
| + if !s.allowWrite(p, time.Now()) { |
| + jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again") |
| + return |
| + } |
| + var req struct { |
| + Host string `json:"host"` |
| + } |
| + if !readJSON(w, r, &req) { |
| + return |
| + } |
| + host := strings.TrimSpace(req.Host) |
| + if host == "" { |
| + jsonError(w, http.StatusBadRequest, "host is required") |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), relayTimeout) |
| + defer cancel() |
| + rec, err := s.relayAdopted(ctx, acct, app, host, addr) |
| + if err != nil { |
| + relayDoorError(w, "adopted", err) |
| + return |
| + } |
| + s.audit(acct, app, addr, "relay-adopted", "acknowledged the relay key on "+host+" (cli)") |
| + out, err := s.relayReply(ctx, acct, app, rec) |
| + if err != nil { |
| + log.Printf("console: api relay reply %s/%s: %v", acct.Sqid, app, err) |
| + jsonError(w, http.StatusBadGateway, |
| + "the acknowledgement was recorded, but status could not be read — try `carlos relay status`") |
| + return |
| + } |
| + writeJSON(w, http.StatusOK, out) |
| +} |
| + |
| +// ---- the operator door: rotate ----------------------------------------------- |
| + |
| +// apiRelayOperator is rotate's whole gate. RelayRegions/RelayOperators name a |
| +// deployment-wide authority (RelayOperators' own doc comment, server.go) with |
| +// no account or app to check membership against — POST /api/cli/relay/rotate |
| +// carries neither segment, the same shape `carlos system`'s doors answer for |
| +// (apiSystemOwner, apisystemupdate.go; apiSystemOperator, apisystem.go). |
| +// |
| +// Unlike those two — and unlike the store's grant door, which gives a |
| +// member-but-not-operator caller a 403 (apistore.go's storeOperator) — a |
| +// failed check here is a byte-identical 404, because there is no membership |
| +// proof for a 403 to be safe AFTER: this door proves nothing about the |
| +// caller before answering, so a 403 naming CARLOS_RELAY_OPERATORS would tell |
| +// any valid token holder that this deployment runs a relay at all. A |
| +// non-person principal (a service credential, or a request that never |
| +// reached cliRoute) gets the identical answer, never a distinguishing one. |
| +func (s *Server) apiRelayOperator(w http.ResponseWriter, r *http.Request) (string, bool) { |
| + p, ok := principalFrom(r.Context()) |
| + if !ok || p.Kind != PrincipalUser || !s.relayOperator(p.Address) { |
| + http.NotFound(w, r) |
| + return "", false |
| + } |
| + return p.Address, true |
| +} |
| + |
| +// relayOperator reports whether addr may drive `carlos relay rotate` — |
| +// storeOperator's exact-match rule, verbatim (apistore.go's doc comment |
| +// explains why EqualFold is wrong for a gate like this one: the ſ homoglyph |
| +// folds to a lowercase "s" that already survived every ToLower on the way |
| +// in). Its own list rather than StoreOperators', for RelayOperators' own |
| +// reason (server.go): rotating a region's master is authority over the |
| +// deployment's key material, not over spending its cloud bill, and a |
| +// deployment may grant the two to different people. |
| +func (s *Server) relayOperator(addr string) bool { |
| + for _, op := range s.cfg.RelayOperators { |
| + if strings.ToLower(strings.TrimSpace(op)) == addr { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// handleAPIRelayRotate answers `carlos relay rotate --region <r>` (spec §5, |
| +// step 4): move the region's issuer pointer to the master the operator |
| +// staged, and re-issue every app enabled there. Not rate limited on the |
| +// address the way the member doors are — apiRelayOperator's own 404 already |
| +// bounds who reaches this far, and a rotation the caller is entitled to |
| +// start must not be delayed by a shared write budget spent by other doors. |
| +func (s *Server) handleAPIRelayRotate(w http.ResponseWriter, r *http.Request) { |
| + addr, ok := s.apiRelayOperator(w, r) |
| + if !ok { |
| + return |
| + } |
| + if s.cfg.Releases == nil { |
| + relayNoBucket(w) |
| + return |
| + } |
| + var req struct { |
| + Region string `json:"region"` |
| + } |
| + if !readJSON(w, r, &req) { |
| + return |
| + } |
| + region := strings.TrimSpace(req.Region) |
| + if region == "" { |
| + jsonError(w, http.StatusBadRequest, "region is required") |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(r.Context(), relayRotateTimeout) |
| + defer cancel() |
| + res, err := s.relayRotate(ctx, region) |
| + if err != nil { |
| + relayDoorError(w, "rotate", err) |
| + return |
| + } |
| + s.auditSystem(addr, "relay-rotate", |
| + fmt.Sprintf("rotated %s to kid %s (%d app(s) re-issued, cli)", res.Region, res.Kid, res.Apps)) |
| + writeJSON(w, http.StatusOK, apiRelayRotateReply{ |
| + Region: res.Region, Kid: res.Kid, Apps: res.Apps, |
| + RetireNoEarlierThan: res.RetireNoEarlierThan.UTC().Format(time.RFC3339), |
| + }) |
| +} |
diff --git a/internal/console/apirelay_test.go b/internal/console/apirelay_test.go| new file mode 100644 |
| index 000000000..8ad8777aa |
| --- /dev/null |
| +++ b/internal/console/apirelay_test.go |
| @@ -0,0 +1,349 @@ |
| +package console |
| + |
| +// The relay's doors (relay spec §5), through the real mux and a real bearer |
| +// token. What is pinned here, in the order it would hurt if lost: a stranger |
| +// probing the relay learns nothing — its refusal is byte-for-byte the STORE |
| +// door's refusal, so the two cannot be told apart by anything but the path; |
| +// a deployment with no armed region refuses to enable and NAMES the variable |
| +// an operator would set, rather than minting a key nothing can verify; status |
| +// tells the truth about a region whose relay has gone quiet, because "stale" |
| +// is the only way a member with no box access finds out; and rotate — the one |
| +// door with no account in its path — answers a caller who is not on |
| +// CARLOS_RELAY_OPERATORS with the same 404 an unknown account gets, never a |
| +// 403 that would confirm this deployment runs a relay at all. |
| + |
| +import ( |
| + "context" |
| + "net/http" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/relay" |
| +) |
| + |
| +// relayURL is the app-scoped family's root: enable POSTs here, status GETs |
| +// here, and disable/adopted hang off it. |
| +func relayURL(fx *consoleFixture, app string) string { |
| + return fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/" + app + "/relay" |
| +} |
| + |
| +// seedHeartbeat writes one armed relay's heartbeat as the box itself would, |
| +// with its last SUCCESSFUL authority read `age` in the past — the clock |
| +// status reads, and the one a dead relay cannot move. |
| +func seedHeartbeat(t *testing.T, fx *consoleFixture, box, region string, age time.Duration, kids ...string) { |
| + t.Helper() |
| + now := time.Now() |
| + hb := relay.Heartbeat{ |
| + Box: box, Region: region, ReadAt: now.Add(-age), WrittenAt: now.Add(-age), |
| + Listeners: []string{"udp:3478", "tcp:3478"}, Kids: kids, |
| + } |
| + if err := relay.WriteHeartbeat(context.Background(), fx.rel, hb); err != nil { |
| + t.Fatalf("seed heartbeat: %v", err) |
| + } |
| +} |
| + |
| +// region pulls one row out of a status reply's region_status list. |
| +func regionRow(t *testing.T, out map[string]any, i int) map[string]any { |
| + t.Helper() |
| + rows, ok := out["region_status"].([]any) |
| + if !ok || len(rows) <= i { |
| + t.Fatalf("region_status = %#v, want at least %d row(s)", out["region_status"], i+1) |
| + } |
| + row, ok := rows[i].(map[string]any) |
| + if !ok { |
| + t.Fatalf("region_status[%d] = %#v, want an object", i, rows[i]) |
| + } |
| + return row |
| +} |
| + |
| +// ---- enable, status, disable ------------------------------------------------- |
| + |
| +// TestAPIRelayEnableThenStatusThenDisable is the member's whole path through |
| +// the doors: turn it on, read what was issued and how the region looks, turn |
| +// it off. Each assertion is on something only the door can get wrong — the |
| +// operations themselves are pinned in relayops_test.go. |
| +func TestAPIRelayEnableThenStatusThenDisable(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-door") |
| + |
| + status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, cliToken(t, fx, "alice@keymail.dev", "code-relay-door-2")) |
| + if status != http.StatusOK { |
| + t.Fatalf("enable = %d %v, want 200", status, out) |
| + } |
| + if out["enabled"] != true { |
| + t.Fatalf("enabled = %v, want true", out["enabled"]) |
| + } |
| + inc := str(t, out, "incarnation") |
| + if len(inc) != 16 { |
| + t.Fatalf("incarnation = %q, want 16 hex characters", inc) |
| + } |
| + if got := strList(t, out, "regions"); len(got) != 1 || got[0] != "eu-west-1" { |
| + t.Fatalf("regions = %v, want the one armed region", got) |
| + } |
| + // The username suffix is echoed, the key never is: the suffix is the same |
| + // public identity the app's own CARLOS_RELAY carries, and a member needs |
| + // it to match a call's credential against a rotation. |
| + issued, ok := out["issued"].(map[string]any) |
| + if !ok || !strings.Contains(issued["eu-west-1"].(string), inc) { |
| + t.Fatalf("issued = %#v, want a suffix naming the incarnation", out["issued"]) |
| + } |
| + // Nothing else: no key, no envelope, no variable. The record holds the |
| + // digest of what was sealed and the console cannot open it anyway, but a |
| + // door that grew a "value" field would be the one place a key could leak |
| + // back out of this system. |
| + for _, k := range []string{"value", "key", "envelope", "secret", "master"} { |
| + if _, ok := out[k]; ok { |
| + t.Fatalf("the enable reply carries %q — nothing key-shaped may cross this door", k) |
| + } |
| + } |
| + |
| + // A relay that read the authority six minutes ago is STALE, and the lease |
| + // it serves on is still live (fifteen minutes) — two different clocks, and |
| + // a member debugging a failing call needs both. |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-door-3") |
| + seedHeartbeat(t, fx, "box1", "eu-west-1", 6*time.Minute, "k1") |
| + status, out = getJSON(t, relayURL(fx, "titogo"), token) |
| + if status != http.StatusOK { |
| + t.Fatalf("status = %d %v, want 200", status, out) |
| + } |
| + row := regionRow(t, out, 0) |
| + if row["region"] != "eu-west-1" || row["box"] != "box1" { |
| + t.Fatalf("region row = %#v, want the seeded box", row) |
| + } |
| + if row["stale"] != true { |
| + t.Fatalf("stale = %v on a six-minute-old heartbeat, want true", row["stale"]) |
| + } |
| + if row["lease_live"] != true { |
| + t.Fatalf("lease_live = %v six minutes in, want true (the lease runs 15)", row["lease_live"]) |
| + } |
| + if age, _ := row["heartbeat_age"].(string); !strings.HasPrefix(age, "6m") { |
| + t.Fatalf("heartbeat_age = %v, want about six minutes", row["heartbeat_age"]) |
| + } |
| + if kids := strList(t, row, "kids"); len(kids) != 1 || kids[0] != "k1" { |
| + t.Fatalf("kids = %v, want the loaded master", kids) |
| + } |
| + |
| + status, out = postJSON(t, relayURL(fx, "titogo")+"/disable", map[string]any{}, token) |
| + if status != http.StatusOK { |
| + t.Fatalf("disable = %d %v, want 200", status, out) |
| + } |
| + if out["enabled"] != false { |
| + t.Fatalf("enabled = %v after a disable, want false", out["enabled"]) |
| + } |
| + // The disable is not COMPLETE until the region's own relay heartbeats an |
| + // authority written at or after it (spec §5). The seeded heartbeat is six |
| + // minutes older than the publish, so the region is still pending. |
| + if pending := strList(t, out, "disable_pending"); len(pending) != 1 || pending[0] != "eu-west-1" { |
| + t.Fatalf("disable_pending = %v, want the region whose relay has not caught up", pending) |
| + } |
| + if rec := relayRecord(t, fx, "titogo"); rec == nil || rec.Enabled { |
| + t.Fatalf("record = %+v after the disable door, want it off", rec) |
| + } |
| +} |
| + |
| +// TestAPIRelayStatusBeforeEnableIs404: "never enabled" is a state, not a |
| +// failure, and it is a sentence rather than an empty record — handleAPIStoreGet's |
| +// same choice. It speaks only because apiAccountApp has already proved |
| +// membership above it. |
| +func TestAPIRelayStatusBeforeEnableIs404(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-unenabled") |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-unenabled-2") |
| + |
| + status, out := getJSON(t, relayURL(fx, "titogo"), token) |
| + if status != http.StatusNotFound { |
| + t.Fatalf("status with no record = %d %v, want 404", status, out) |
| + } |
| + if !strings.Contains(str(t, out, "error"), "carlos relay enable") { |
| + t.Fatalf("error = %q, want the sentence naming the verb that fixes it", out["error"]) |
| + } |
| +} |
| + |
| +// TestAPIRelayEnableWithoutRegionsIs503: a deployment that arms no region |
| +// refuses, and the refusal names CARLOS_RELAY_REGIONS. A 500 or a silent |
| +// success would both be worse than this: the first tells a member nothing, |
| +// the second would mint an identity no relay would ever accept. |
| +func TestAPIRelayEnableWithoutRegionsIs503(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-noregions") |
| + fx.srv.cfg.RelayRegions = nil |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-noregions-2") |
| + |
| + status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token) |
| + if status != http.StatusServiceUnavailable { |
| + t.Fatalf("enable with no armed region = %d %v, want 503", status, out) |
| + } |
| + if str(t, out, "error") != errRelayNoRegions.Error() { |
| + t.Fatalf("error = %q, want the sentinel verbatim", out["error"]) |
| + } |
| + if rec := relayRecord(t, fx, "titogo"); rec != nil { |
| + t.Fatalf("a refused enable wrote a record: %+v", rec) |
| + } |
| +} |
| + |
| +// TestAPIRelayEnableNamingAnUnarmedRegionIs400: a region the caller typed |
| +// that this deployment does not run is the CALLER's mistake, so it is a 400 |
| +// naming the armed ones — not the 502 an unmatched error would read as. |
| +func TestAPIRelayEnableNamingAnUnarmedRegionIs400(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-badregion") |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-badregion-2") |
| + |
| + status, out := postJSON(t, relayURL(fx, "titogo"), |
| + map[string]any{"regions": []string{"us-east-1"}}, token) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("enable in an unarmed region = %d %v, want 400", status, out) |
| + } |
| + if msg := str(t, out, "error"); !strings.Contains(msg, "us-east-1") || !strings.Contains(msg, "eu-west-1") { |
| + t.Fatalf("error = %q, want the region asked for and the ones armed", msg) |
| + } |
| +} |
| + |
| +// TestAPIRelayAdoptedUnknownHostIs404: the ack must name a route of THIS app. |
| +// Without that the record would grow a cutover for anything a caller typed, |
| +// and status would report identities against routes that do not exist. |
| +func TestAPIRelayAdoptedUnknownHostIs404(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-adopted") |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-adopted-2") |
| + if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { |
| + t.Fatalf("enable = %d %v, want 200", status, out) |
| + } |
| + |
| + status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted", |
| + map[string]any{"host": "nowhere.example"}, token) |
| + if status != http.StatusNotFound { |
| + t.Fatalf("adopted for an unknown host = %d %v, want 404", status, out) |
| + } |
| + if msg := str(t, out, "error"); !strings.Contains(msg, "nowhere.example") { |
| + t.Fatalf("error = %q, want the host named", msg) |
| + } |
| + // An empty host is the caller's mistake before any lookup happens. |
| + if status, _ := postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": " "}, token); status != http.StatusBadRequest { |
| + t.Fatalf("adopted with no host = %d, want 400", status) |
| + } |
| + if rec := relayRecord(t, fx, "titogo"); rec != nil && len(rec.Cutovers) != 0 { |
| + t.Fatalf("a refused ack wrote a cutover: %+v", rec.Cutovers) |
| + } |
| +} |
| + |
| +// ---- the gate ---------------------------------------------------------------- |
| + |
| +// TestAPIRelayGateIsByteIdenticalWithTheStore: every app-scoped relay door |
| +// refuses a stranger, an account that does not exist and an app that was |
| +// never claimed with ONE answer — and that answer is byte-for-byte what the |
| +// STORE door gives the same caller. The cross-door comparison is the point: |
| +// if the relay's 404 ever drifted from its neighbour's, the difference itself |
| +// would tell a token holder which of a deployment's apps have a relay. |
| +func TestAPIRelayGateIsByteIdenticalWithTheStore(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-gate") |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-gate-member") |
| + eve := cliToken(t, fx, "eve@keymail.dev", "code-relay-gate-eve") |
| + |
| + // The store door's refusal for the same caller, as the yardstick. |
| + _, wantBody := getRaw(t, storeURL(fx, "titogo"), eve) |
| + |
| + doors := []struct { |
| + name string |
| + call func(u, bearer string) (int, string) |
| + }{ |
| + {"GET", func(u, bearer string) (int, string) { return getRaw(t, u, bearer) }}, |
| + {"POST", func(u, bearer string) (int, string) { return postRaw(t, u, map[string]any{}, bearer) }}, |
| + {"POST disable", func(u, bearer string) (int, string) { |
| + return postRaw(t, u+"/disable", map[string]any{}, bearer) |
| + }}, |
| + {"POST adopted", func(u, bearer string) (int, string) { |
| + return postRaw(t, u+"/adopted", map[string]any{"host": "x"}, bearer) |
| + }}, |
| + } |
| + for _, d := range doors { |
| + t.Run(d.name, func(t *testing.T) { |
| + // (a) a member of nothing, against a real account and a real app. |
| + statusA, bodyA := d.call(relayURL(fx, "titogo"), eve) |
| + // (b) a member, against an account that does not exist. |
| + statusB, bodyB := d.call(fx.ts.URL+"/api/cli/apps/zzzz/titogo/relay", token) |
| + // (c) a member of the account, against an app it never claimed. |
| + statusC, bodyC := d.call(relayURL(fx, "ghost-app"), token) |
| + if statusA != http.StatusNotFound || statusB != http.StatusNotFound || statusC != http.StatusNotFound { |
| + t.Fatalf("%s gate = %d / %d / %d, want 404 for all three", d.name, statusA, statusB, statusC) |
| + } |
| + if bodyA != bodyB || bodyB != bodyC { |
| + t.Fatalf("%s refusals are not byte-identical:\n(a) %q\n(b) %q\n(c) %q", d.name, bodyA, bodyB, bodyC) |
| + } |
| + if bodyA != wantBody { |
| + t.Fatalf("%s refusal body = %q, want the store door's own refusal %q", d.name, bodyA, wantBody) |
| + } |
| + }) |
| + } |
| + if rec := relayRecord(t, fx, "titogo"); rec != nil { |
| + t.Fatalf("a refused request wrote a record: %+v", rec) |
| + } |
| + // The token gate runs first, before any name is resolved. |
| + if status, _ := getJSON(t, relayURL(fx, "titogo"), ""); status != http.StatusUnauthorized { |
| + t.Fatalf("status with no token = %d, want 401", status) |
| + } |
| + if status, _ := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, ""); status != http.StatusUnauthorized { |
| + t.Fatalf("enable with no token = %d, want 401", status) |
| + } |
| +} |
| + |
| +// ---- rotate ------------------------------------------------------------------ |
| + |
| +// TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise: the one door with |
| +// no account segment. A member in good standing who is not on |
| +// CARLOS_RELAY_OPERATORS gets http.NotFound's own bytes — not a 403 naming |
| +// the variable, which would tell any valid token that this deployment runs a |
| +// relay worth rotating. Listed, the same caller rotates and is told which kid |
| +// the region issues from now. |
| +func TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-rotate-door") |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-rotate-door-2") |
| + rotate := fx.ts.URL + "/api/cli/relay/rotate" |
| + |
| + status, body := postRaw(t, rotate, map[string]any{"region": "eu-west-1"}, token) |
| + if status != http.StatusNotFound { |
| + t.Fatalf("rotate by a non-operator = %d %q, want 404", status, body) |
| + } |
| + if body != "404 page not found\n" { |
| + t.Fatalf("rotate refusal body = %q, want the stdlib http.NotFound output", body) |
| + } |
| + if status, _ := postRaw(t, rotate, map[string]any{"region": "eu-west-1"}, ""); status != http.StatusUnauthorized { |
| + t.Fatalf("rotate with no token = %d, want 401", status) |
| + } |
| + |
| + if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { |
| + t.Fatalf("enable = %d %v, want 200", status, out) |
| + } |
| + fx.srv.cfg.RelayOperators = []string{"alice@keymail.dev"} |
| + // A rotation refuses until every live relay in the region has loaded the |
| + // staged kid, and that refusal is the operator's to fix — a 400 naming the |
| + // box, not a 502. |
| + t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) |
| + seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1") |
| + status, out := postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("rotate before the box loaded the kid = %d %v, want 400", status, out) |
| + } |
| + if msg := str(t, out, "error"); !strings.Contains(msg, "box1") || !strings.Contains(msg, "k2") { |
| + t.Fatalf("error = %q, want the box and the kid named", msg) |
| + } |
| + |
| + seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1", "k2") |
| + status, out = postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) |
| + if status != http.StatusOK { |
| + t.Fatalf("rotate = %d %v, want 200", status, out) |
| + } |
| + if out["kid"] != "k2" || out["region"] != "eu-west-1" { |
| + t.Fatalf("rotate reply = %v, want the region on its new kid", out) |
| + } |
| + if apps, _ := out["apps"].(float64); apps != 1 { |
| + t.Fatalf("apps = %v, want the one enabled app re-issued", out["apps"]) |
| + } |
| + retire, err := time.Parse(time.RFC3339, str(t, out, "retire_no_earlier_than")) |
| + if err != nil { |
| + t.Fatalf("retire_no_earlier_than = %q: %v", out["retire_no_earlier_than"], err) |
| + } |
| + if d := time.Until(retire); d < 27*time.Hour || d > 29*time.Hour { |
| + t.Fatalf("retire_no_earlier_than is %s away, want about 28h", d) |
| + } |
| + // An empty region is the caller's own mistake, before anything moves. |
| + if status, _ := postJSON(t, rotate, map[string]any{}, token); status != http.StatusBadRequest { |
| + t.Fatalf("rotate with no region = %d, want 400", status) |
| + } |
| +} |
diff --git a/internal/console/authcli.go b/internal/console/authcli.go| index a64d3d01f..3ea381ebf 100644 |
| --- a/internal/console/authcli.go |
| +++ b/internal/console/authcli.go |
| @@ -756,6 +756,23 @@ func (s *Server) registerCLIRoutes() { |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/revoke", svcrole.Allow(svcrole.Admin), s.handleAPIScanRevoke) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/sweep", svcrole.Allow(svcrole.Admin), s.handleAPIScanSweep) |
| s.cliRoute("POST /api/cli/apps/{account}/{app}/store/scan/enforce", svcrole.Allow(svcrole.Admin), s.handleAPIScanEnforce) |
| + // The relay family (internal/console/apirelay.go, relay spec §5): the |
| + // platform STUN/TURN relay's app-facing doors. enable/status/disable are |
| + // the member's own verbs — declaring, inspecting and ending the app's own |
| + // relay estate is not an authority over anything but that app, unlike the |
| + // object store's grant. adopted is the member's cutover ack for a |
| + // TCP-address route this platform cannot restart. Literal segments under |
| + // .../relay, same as store's, so no app or host name can reach them. |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/relay", svcrole.Allow(svcrole.Admin), s.handleAPIRelayEnable) |
| + s.cliRoute("GET /api/cli/apps/{account}/{app}/relay", svcrole.Allow(svcrole.Admin), s.handleAPIRelayGet) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/relay/disable", svcrole.Allow(svcrole.Admin), s.handleAPIRelayDisable) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/relay/adopted", svcrole.Allow(svcrole.Admin), s.handleAPIRelayAdopted) |
| + // rotate is the ONE relay door with no {account} segment: it moves a |
| + // REGION's issuer pointer and re-issues every app enabled there, an |
| + // authority over the deployment's key material (RelayOperators, server.go) |
| + // rather than over any one tenant's app. apiRelayOperator is its whole |
| + // gate. |
| + s.cliRoute("POST /api/cli/relay/rotate", svcrole.Allow(svcrole.Admin), s.handleAPIRelayRotate) |
| // `carlos system` (apisystemupdate.go and apisystem.go, spec §5) — the |
| // deployment-wide doors. No {account} segment, so no service credential |
| // can be scoped to them at all. Two families sit here: the operator's |
diff --git a/relay.go b/relay.go| index 2042250ea..d674440b3 100644 |
| --- a/relay.go |
| +++ b/relay.go |
| @@ -1,13 +1,25 @@ |
| // relay.go is `carlos relay`: the platform STUN/TURN relay's own verbs |
| -// (relay spec §3-§5). Two of them exist so far and they sit at opposite |
| -// ends of the product: |
| +// (relay spec §3-§5). Three audiences, one command: |
| // |
| +// enable a MEMBER turning the relay on for their app, and having |
| +// adopted CARLOS_RELAY delivered to its config. status is the only |
| +// status sight they have of delivery per route and health per region |
| +// disable without box access; adopted is their cutover ack for a route |
| +// this platform cannot restart for them. |
| +// rotate an OPERATOR moving a region onto a newly staged master. |
| +// keygen an OPERATOR minting one. Its output is a secret. |
| // serve what carlos-relay.service runs. Not typed by hand. |
| -// keygen mint a region master. Typed by an operator, once per region |
| -// per rotation, and its output is a secret. |
| // |
| -// The app-facing family (enable, status, disable, rotate, adopted) lands |
| -// beside them later; the usage line below grows with it. |
| +// The five member/operator verbs are API mode only, on `carlos store`'s |
| +// model (store.go): the record lives in the console's deployment bucket, and |
| +// the key is derived from a master only the console holds and sealed with a |
| +// key only the app's boxes can open — there is no bucket-direct equivalent of |
| +// any of that. See requireRelayAPISession. |
| +// |
| +// The output-honesty rule is restart.go's, and it bites hardest here: enable |
| +// prints what was ISSUED and written, never that a route has picked it up. |
| +// Only status can say that, and even status says only what the console can |
| +// observe — a route's own acknowledgement, not a guess from elapsed time. |
| package main |
| import ( |
| @@ -15,22 +27,56 @@ import ( |
| "crypto/rand" |
| "encoding/base64" |
| "encoding/hex" |
| + "flag" |
| "fmt" |
| + "io" |
| "log" |
| "os" |
| "os/signal" |
| + "strings" |
| "syscall" |
| "github.com/carlosframework/platform/internal/host" |
| "github.com/carlosframework/platform/internal/relay" |
| ) |
| -const relayUsage = `usage: carlos relay <serve|keygen> [flags] |
| +const relayUsage = `usage: carlos relay <enable|status|disable|adopted|rotate|keygen|serve> [flags] |
| - serve run the relay (what carlos-relay.service execs; not typed by hand) |
| + enable --app <a> [--regions <r1,r2>] |
| + turn the relay on for one app and deliver CARLOS_RELAY to its |
| + config. Regions default to every one this deployment arms, in the |
| + console's order — the order every browser tries them in. |
| + status --app <a> |
| + what was issued per region, what each route has done about it, |
| + and each region relay's heartbeat, lease and loaded masters. The |
| + only sight of any of that without box access. |
| + disable --app <a> |
| + remove the key and end every allocation the app holds. |
| + adopted --app <a> --host <route> |
| + tell the platform that a TCP-address route you restarted by hand |
| + now runs the issued key. The one route shape this platform cannot |
| + restart for you, so the acknowledgement has to come from you. |
| + rotate --region <r> |
| + operator: move the region's issuer pointer onto the newly staged |
| + master and re-issue every app enabled there. Needs an address on |
| + CARLOS_RELAY_OPERATORS; anyone else meets the same 404 as an |
| + unknown console door, never a 403. |
| keygen mint a region master: one "<kid> <base64 key> -" line for |
| - /etc/carlos/relay-masters and for the console's |
| - CARLOS_RELAY_MASTERS_<REGION> secret` |
| + /etc/carlos/relay-masters and the same pair for the console's |
| + CARLOS_RELAY_MASTERS_<REGION> secret, which holds every master the |
| + console may need as <kid>:<key>[,<kid>:<key>]. |
| + serve run the relay (what carlos-relay.service execs; not typed by hand) |
| + |
| + --account <sqid|name> account the app belongs to (default: your only |
| + account, or the configured/chosen one) |
| + --account-id <sqid> account sqid, matched as a sqid ONLY. |
| + --console <origin> which logged-in console to use (only needed with |
| + several) |
| + |
| +enable, status, disable, adopted and rotate are signed-in only |
| +(carlos auth login): the relay's keys are derived from a master the console |
| +holds and sealed with the deployment's secrets key, so there is no |
| +direct-bucket equivalent.` |
| func runRelay(args []string) { |
| if len(args) == 0 { |
| @@ -38,6 +84,16 @@ func runRelay(args []string) { |
| os.Exit(2) |
| } |
| switch args[0] { |
| + case "enable": |
| + runRelayEnable(args[1:]) |
| + case "status": |
| + runRelayStatus(args[1:]) |
| + case "disable": |
| + runRelayDisable(args[1:]) |
| + case "adopted": |
| + runRelayAdopted(args[1:]) |
| + case "rotate": |
| + runRelayRotate(args[1:]) |
| case "serve": |
| runRelayServe(args[1:]) |
| case "keygen": |
| @@ -99,3 +155,259 @@ func runRelayKeygen(_ []string) { |
| } |
| fmt.Printf("%s %s -\n", hex.EncodeToString(kid), base64.StdEncoding.EncodeToString(key)) |
| } |
| + |
| +// ---- the app-facing family (spec §5) ---------------------------------------- |
| + |
| +// requireRelayAPISession resolves signed-in mode for every relay verb below, |
| +// refusing direct-bucket mode with relay's OWN message — requireStoreAPISession's |
| +// pattern (store.go), and for the same reason: a refusal that names another |
| +// verb's advice sends someone to a fix that cannot help them. |
| +// |
| +// There is no headless fallback, and the reason is stronger than the store's. |
| +// Enabling a relay does not merely write an object: it derives an app key from |
| +// a region master held in the CONSOLE's own environment and seals it with the |
| +// deployment's secrets key. A shell in bucket mode has neither, so there is |
| +// nothing here it could drive on its own — and a key minted anywhere else |
| +// would be one no relay in the region would accept. |
| +func requireRelayAPISession(consoleFlag string) *apiSession { |
| + sess, err := resolveMode(consoleFlag) |
| + if err != nil { |
| + log.Fatal(err) |
| + } |
| + if sess == nil { |
| + log.Fatal("carlos relay works through a console, and this shell is in direct bucket mode\n" + |
| + "(CARLOS_DEPLOYMENT_BUCKET/CARLOS_DEPLOYMENT_DIR is set). The relay's keys are derived from a\n" + |
| + "region master the CONSOLE holds and sealed with the deployment's secrets key, neither of which\n" + |
| + "exists in this shell — there is no direct-bucket equivalent. Unset the store env and run:\n" + |
| + "carlos auth login --console <url>.") |
| + } |
| + return sess |
| +} |
| + |
| +func runRelayEnable(args []string) { |
| + fs := flag.NewFlagSet("relay enable", flag.ExitOnError) |
| + app := appFlag(fs, "app to enable the relay for") |
| + regions := fs.String("regions", "", "comma-separated regions to enable (default: every region this deployment arms)") |
| + acctRaw, acctIDRaw, _ := accountFlags(fs) |
| + consoleFlag := fs.String("console", "", "console origin to enable through (API mode only; default: the only logged-in one)") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + name := strings.TrimSpace(*app) |
| + usage := "usage: carlos relay enable --app <a> [--regions <r1,r2>] [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]" |
| + if name != "" { |
| + if err := validAppName(name); err != nil { |
| + log.Fatalf("relay enable: %v", err) |
| + } |
| + } |
| + |
| + sess := requireRelayAPISession(*consoleFlag) |
| + printUsing(stdout, sess) |
| + acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| + name = resolvedApp(sess, acct, name, usage) |
| + res, err := sess.relayEnable(context.Background(), acct, name, splitList(*regions)) |
| + if err != nil { |
| + log.Fatalf("relay enable: %v", err) |
| + } |
| + printRelayStatus(stdout, res) |
| + // The output-honesty rule this whole CLI follows (restart.go's): the |
| + // console's own walk finished before this process got its 200, so the |
| + // record and the aggregate ARE written — but a route picking the key up |
| + // is a converge a box does on its own schedule, and only `status` can |
| + // say whether it has. |
| + mutedf(stdout, "delivery per route is reported by: carlos relay status --app %s", name) |
| +} |
| + |
| +func runRelayStatus(args []string) { |
| + fs := flag.NewFlagSet("relay status", flag.ExitOnError) |
| + app := appFlag(fs, "app whose relay to show") |
| + acctRaw, acctIDRaw, _ := accountFlags(fs) |
| + consoleFlag := fs.String("console", "", "console origin to read through (API mode only; default: the only logged-in one)") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + name := strings.TrimSpace(*app) |
| + usage := "usage: carlos relay status --app <a> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]" |
| + if name != "" { |
| + if err := validAppName(name); err != nil { |
| + log.Fatalf("relay status: %v", err) |
| + } |
| + } |
| + |
| + sess := requireRelayAPISession(*consoleFlag) |
| + printUsing(stdout, sess) |
| + acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| + name = resolvedApp(sess, acct, name, usage) |
| + res, err := sess.relayGet(context.Background(), acct, name) |
| + if err != nil { |
| + log.Fatalf("relay status: %v", err) |
| + } |
| + printRelayStatus(stdout, res) |
| +} |
| + |
| +func runRelayDisable(args []string) { |
| + fs := flag.NewFlagSet("relay disable", flag.ExitOnError) |
| + app := appFlag(fs, "app to disable the relay for") |
| + acctRaw, acctIDRaw, _ := accountFlags(fs) |
| + consoleFlag := fs.String("console", "", "console origin to disable through (API mode only; default: the only logged-in one)") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + name := strings.TrimSpace(*app) |
| + usage := "usage: carlos relay disable --app <a> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]" |
| + if name != "" { |
| + if err := validAppName(name); err != nil { |
| + log.Fatalf("relay disable: %v", err) |
| + } |
| + } |
| + |
| + sess := requireRelayAPISession(*consoleFlag) |
| + printUsing(stdout, sess) |
| + acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| + name = resolvedApp(sess, acct, name, usage) |
| + res, err := sess.relayDisable(context.Background(), acct, name) |
| + if err != nil { |
| + log.Fatalf("relay disable: %v", err) |
| + } |
| + printRelayStatus(stdout, res) |
| +} |
| + |
| +func runRelayAdopted(args []string) { |
| + fs := flag.NewFlagSet("relay adopted", flag.ExitOnError) |
| + app := appFlag(fs, "app the route belongs to") |
| + host := fs.String("host", "", "the route that now runs the issued relay key") |
| + acctRaw, acctIDRaw, _ := accountFlags(fs) |
| + consoleFlag := fs.String("console", "", "console origin to acknowledge through (API mode only; default: the only logged-in one)") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + name := strings.TrimSpace(*app) |
| + usage := "usage: carlos relay adopted --app <a> --host <route> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]" |
| + if name != "" { |
| + if err := validAppName(name); err != nil { |
| + log.Fatalf("relay adopted: %v", err) |
| + } |
| + } |
| + if strings.TrimSpace(*host) == "" { |
| + log.Fatalf("relay adopted: --host is required\n%s", usage) |
| + } |
| + |
| + sess := requireRelayAPISession(*consoleFlag) |
| + printUsing(stdout, sess) |
| + acct := resolvedAccount(sess, acctRaw, acctIDRaw) |
| + name = resolvedApp(sess, acct, name, usage) |
| + res, err := sess.relayAdopted(context.Background(), acct, name, strings.TrimSpace(*host)) |
| + if err != nil { |
| + log.Fatalf("relay adopted: %v", err) |
| + } |
| + printRelayStatus(stdout, res) |
| +} |
| + |
| +// runRelayRotate is the operator's verb and the one here with no --app: it |
| +// moves a REGION's issuer pointer and re-issues every app enabled there. A |
| +// caller who is not on CARLOS_RELAY_OPERATORS meets the console's byte- |
| +// identical 404, so the error printed here reads as "no such thing" rather |
| +// than "not allowed" — deliberately, since a 403 would confirm to any token |
| +// holder that this deployment runs a relay at all. |
| +func runRelayRotate(args []string) { |
| + fs := flag.NewFlagSet("relay rotate", flag.ExitOnError) |
| + region := fs.String("region", "", "region whose issuer pointer to move onto the newly staged master") |
| + consoleFlag := fs.String("console", "", "console origin to rotate through (API mode only; default: the only logged-in one)") |
| + fs.Usage = flagUsage(fs, "") |
| + fs.Parse(args) |
| + if strings.TrimSpace(*region) == "" { |
| + log.Fatal("relay rotate: --region is required\nusage: carlos relay rotate --region <r> [--console <origin>]") |
| + } |
| + |
| + sess := requireRelayAPISession(*consoleFlag) |
| + printUsing(stdout, sess) |
| + res, err := sess.relayRotate(context.Background(), strings.TrimSpace(*region)) |
| + if err != nil { |
| + log.Fatalf("relay rotate: %v", err) |
| + } |
| + printRelayRotate(stdout, res) |
| +} |
| + |
| +// splitList turns a comma-separated flag into a list, dropping empties so a |
| +// trailing comma is not a region named "". |
| +func splitList(s string) []string { |
| + var out []string |
| + for _, p := range strings.Split(s, ",") { |
| + if p = strings.TrimSpace(p); p != "" { |
| + out = append(out, p) |
| + } |
| + } |
| + return out |
| +} |
| + |
| +// printRelayStatus prints one relay record: what the member ASKED for, what |
| +// the console ISSUED, what each route has done about it, and how each region's |
| +// own relay looks. The four are kept apart on purpose — a member debugging a |
| +// failing call needs to know which of them is the one that has not happened, |
| +// and a single "relay: ok" line would hide exactly that. |
| +func printRelayStatus(w io.Writer, res apiRelayReply) { |
| + c := colorFor(w) |
| + fmt.Fprintf(w, "%s %s\n", c.head("relay for"), res.App) |
| + if !res.Enabled { |
| + mutedf(w, "not enabled — carlos relay enable --app %s", res.App) |
| + } else { |
| + fmt.Fprintf(w, " enabled in %s, incarnation %s (op %d, %s)\n", |
| + strings.Join(res.Regions, ", "), res.Incarnation, res.Op, res.Phase) |
| + } |
| + // The issued identity per region: account, app, incarnation and kid, the |
| + // public half of the credential the app mints per call. Printed so a |
| + // member can match a rejected call against the kid a region issues from. |
| + for _, region := range res.Regions { |
| + if suffix := res.Issued[region]; suffix != "" { |
| + fmt.Fprintf(w, " issued %s: %s\n", region, suffix) |
| + } |
| + } |
| + for _, rt := range res.Routes { |
| + fmt.Fprintf(w, " route %s (%s): %s\n", rt.Host, rt.Shape, rt.Delivery) |
| + } |
| + if len(res.Routes) == 0 && res.Enabled { |
| + mutedf(w, " no routes yet — the key is written and waits for the first one") |
| + } |
| + for _, rg := range res.RegionStatus { |
| + line := fmt.Sprintf(" relay %s in %s: heartbeat %s", rg.Box, rg.Region, rg.HeartbeatAge) |
| + if !rg.Current { |
| + line += ", holding an older record" |
| + } |
| + if len(rg.Listeners) > 0 { |
| + line += ", listening on " + strings.Join(rg.Listeners, " ") |
| + } |
| + if len(rg.Kids) > 0 { |
| + line += ", masters " + strings.Join(rg.Kids, ",") |
| + } |
| + switch { |
| + case rg.Stale: |
| + // STALE is the important word and it gets the warning colour: a |
| + // relay that stopped reading the authority keeps serving until its |
| + // lease runs out, so "up" and "current" are different facts. |
| + warnf(w, "%s — STALE (no successful read in over five minutes)", line) |
| + case !rg.LeaseLive: |
| + warnf(w, "%s — lease expired, this relay is no longer serving", line) |
| + default: |
| + fmt.Fprintln(w, line+", lease live") |
| + } |
| + } |
| + if res.Enabled && len(res.RegionStatus) == 0 { |
| + warnf(w, "no relay in %s has heartbeated at all — calls will not connect through TURN", |
| + strings.Join(res.Regions, ", ")) |
| + } |
| + if len(res.DisablePending) > 0 { |
| + warnf(w, "disable pending on %s — those relays have not read the record without this app yet; "+ |
| + "allocations there end on their next read", strings.Join(res.DisablePending, ", ")) |
| + } else if !res.Enabled && res.Incarnation != "" { |
| + fmt.Fprintln(w, " revoked: every relay has read a record without this app") |
| + } |
| +} |
| + |
| +// printRelayRotate is the operator's answer: which master the region issues |
| +// from now, how many apps were re-issued under it, and — the line the runbook |
| +// turns on — the earliest the OLD master may be taken out of the masters file. |
| +// Retiring it before then kills credentials that are still inside their own |
| +// four-hour lifetime (spec §5, step 4). |
| +func printRelayRotate(w io.Writer, res apiRelayRotateReply) { |
| + c := colorFor(w) |
| + fmt.Fprintf(w, "%s %s\n", c.head("rotated"), res.Region) |
| + fmt.Fprintf(w, " issuing from kid %s; %d app(s) re-issued\n", res.Kid, res.Apps) |
| + fmt.Fprintf(w, " retire the previous master no earlier than %s\n", res.RetireNoEarlierThan) |
| + mutedf(w, "leave the old line in /etc/carlos/relay-masters and the old pair in the console secret until then") |
| +} |