amadan
relay: status tells a member when to act, no-delivery routes named, CLI printers tested (review fix)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
7 files changed,
+776
−87
apimode.go+28 −1docs/site/cli.md+8 −6internal/console/apirelay.go+132 −50internal/console/apirelay_test.go+179 −0internal/console/relayops.go+41 −17relay.go+53 −13relay_test.go+335 −0
diff --git a/apimode.go b/apimode.go| index 868db69c5..8e4c850df 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -710,6 +710,18 @@ func notFoundErr() error { |
| return errors.New("not found: check --account and --app, and that your token's address is a member") |
| } |
| +// hasJSONError reports whether a body carries the {"error": "..."} shape — |
| +// the difference between a door's own diagnosis and the mux's byte-identical |
| +// text/plain refusal. apiErrorFromBody asks it to decide which message to |
| +// give; a caller with a better sentence for the second case (relayRotate) |
| +// asks the same question rather than string-matching the answer. |
| +func hasJSONError(body []byte) bool { |
| + var e struct { |
| + Error string `json:"error"` |
| + } |
| + return json.Unmarshal(body, &e) == nil && e.Error != "" |
| +} |
| + |
| // apiErrorFromBody turns a non-2xx status and its raw body into an error. |
| // The JSON {"error": "..."} shape is tried FIRST, whatever the status code — |
| // several doors (promote's unshipped-version 404, rollback's no-history 409) |
| @@ -1928,7 +1940,7 @@ func (s *apiSession) storeRotate(ctx context.Context, a apiAcct, app string, fin |
| type apiRelayRoute struct { |
| Host string `json:"host"` |
| - Shape string `json:"shape"` // unit | socket-exec | tcp-exec-file |
| + Shape string `json:"shape"` // unit | socket-exec | tcp-exec-file | no-delivery | unknown |
| Delivery string `json:"delivery"` |
| AckedAt string `json:"acked_at,omitempty"` |
| Cutover bool `json:"cutover"` |
| @@ -1943,6 +1955,10 @@ type apiRelayRegion struct { |
| Listeners []string `json:"listeners"` |
| Kids []string `json:"kids"` |
| Stale bool `json:"stale"` |
| + // The relay's own last two errors: why it stopped reading. Printed on a |
| + // stale row, because the state without the reason is not actionable. |
| + MastersError string `json:"masters_error,omitempty"` |
| + AuthorityError string `json:"authority_error,omitempty"` |
| } |
| type apiRelayReply struct { |
| @@ -2041,6 +2057,17 @@ func (s *apiSession) relayRotate(ctx context.Context, region string) (apiRelayRo |
| return apiRelayRotateReply{}, err |
| } |
| if status != http.StatusOK { |
| + // A 404 with no JSON body is either gate: a console too old to have |
| + // this door, or apiRelayOperator refusing an address that is not on |
| + // CARLOS_RELAY_OPERATORS — deliberately indistinguishable on the |
| + // wire, so the message states both possibilities rather than |
| + // notFoundErr()'s advice to check --account and --app, two flags this |
| + // verb does not have. Saying both leaks nothing: a caller who is on |
| + // the list finds out by the call succeeding. |
| + if status == http.StatusNotFound && !hasJSONError(body) { |
| + return apiRelayRotateReply{}, errors.New( |
| + "no such door on this console, or your address is not on CARLOS_RELAY_OPERATORS") |
| + } |
| return apiRelayRotateReply{}, apiErrorFromBody(status, body) |
| } |
| var out apiRelayRotateReply |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index a549e4e00..f82a76c93 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -680,12 +680,14 @@ config: a base64-wrapped JSON list of regions, each with its ICE server URLs, |
| a key and a username suffix. The app mints a TURN credential per call from |
| the key (see the relay design spec, §2). `status` shows what was issued, |
| which routes have picked it up, and the health of each region's relay, |
| -none of which needs box access. `disable` removes it and ends every |
| -allocation the app holds. `adopted --host <route>` is how a member tells the |
| -platform that a TCP-address route it restarted by hand now runs the new |
| -key. `rotate --region <r>` is the operator's verb after staging a new |
| -master; `keygen` prints one. `serve` is the unit's entry point and not a |
| -thing to run by hand. |
| +none of which needs box access. `disable` removes it; every allocation the |
| +app holds ends on the relays' next authority read, within a minute. |
| +`adopted --host <route>` is how a member tells the platform that a |
| +TCP-address route it restarted by hand now runs the new key — `status` names |
| +the routes waiting on that, since they are the one shape this platform |
| +cannot restart for you. `rotate --region <r>` is the operator's verb after |
| +staging a new master; `keygen` prints one. `serve` is the unit's entry point |
| +and not a thing to run by hand. |
| `keygen` prints one line — a key id, a key, and `-` for "no retirement date" |
| — for a region's master key. Stage that line in `/etc/carlos/relay-masters` |
diff --git a/internal/console/apirelay.go b/internal/console/apirelay.go| index 13d6e960d..c3a2854a3 100644 |
| --- a/internal/console/apirelay.go |
| +++ b/internal/console/apirelay.go |
| @@ -25,11 +25,13 @@ |
| // DELIVERY IS DELIBERATELY THIN FOR NOW. Spec §5 describes five delivery |
| // states a route can be in, most of them read from a per-route config-ack |
| // (`control/config-ack/<box>/<route host>.json`) that Task 12 has not written |
| -// yet. This file reports only what a route's shape and the record's own |
| -// Cutovers can say today — "adopted by member at <time>" once a member has |
| -// acknowledged the identities that are issued NOW, "written, awaiting |
| -// delivery" otherwise — and marks the place Task 12 fills in with a TODO. It |
| -// does not invent the ack's shape. |
| +// yet. This file reports only what a route's SHAPE, its config bundle and the |
| +// record's own Cutovers can say today: that the key was written, that a route |
| +// this platform cannot restart is waiting on the member, that a route with no |
| +// config delivery will never receive one at all, and that a member's |
| +// acknowledgement either does or no longer does name the identities issued |
| +// now. It marks the place Task 12 fills in with a TODO rather than inventing |
| +// the ack's shape. |
| // |
| // Spec: docs/superpowers/specs/2026-09-07-relay-stun-turn-design.md §2, §5. |
| package console |
| @@ -102,10 +104,10 @@ func relayDoorError(w http.ResponseWriter, verb string, err error) { |
| func relayLooksLikeClientError(err error) bool { |
| msg := err.Error() |
| for _, s := range []string{ |
| - "runs no relay in", // relayResolveRegions, relayRotate: unarmed region named |
| - "no new master is staged", // relayRotate: nothing new in CARLOS_RELAY_MASTERS_<REGION> |
| - "has not loaded kid", // relayRotate: a box has not picked up the staged master yet |
| - "has not heartbeated in", // relayRotate: no live relay in the region at all |
| + "runs no relay in", // relayResolveRegions, relayRotate: unarmed region named |
| + "no new master is staged", // relayRotate: nothing new in CARLOS_RELAY_MASTERS_<REGION> |
| + "has not loaded kid", // relayRotate: a box has not picked up the staged master yet |
| + "has heartbeated in the last", // relayRotate: no live relay in the region at all |
| } { |
| if strings.Contains(msg, s) { |
| return true |
| @@ -120,8 +122,12 @@ func relayLooksLikeClientError(err error) bool { |
| // it: which of spec §5's delivery shapes it is, and what the console can say |
| // about whether its key has arrived. |
| type apiRelayRoute struct { |
| - Host string `json:"host"` |
| - Shape string `json:"shape"` // "unit" | "socket-exec" | "tcp-exec-file" |
| + Host string `json:"host"` |
| + // One of the relayShape* constants below: the first three are spec §5's |
| + // delivery shapes, "no-delivery" is a route a key can never reach, and |
| + // "unknown" is the fail-closed answer for a route this console cannot |
| + // describe. |
| + Shape string `json:"shape"` |
| Delivery string `json:"delivery"` |
| // AckedAt and Cutover are both empty/false until a member has run |
| // `carlos relay adopted --host` for this route (relay.RouteCutover). |
| @@ -145,6 +151,12 @@ type apiRelayRegion struct { |
| // authority read is older than relayHeartbeatFresh (five minutes) cannot |
| // be read as current, so a dead relay cannot look alive by silence. |
| Stale bool `json:"stale"` |
| + // The relay's own last two errors, verbatim (relay.Heartbeat): the |
| + // masters file it could not reload, and the authority record it could not |
| + // read. Either one is the REASON a row is stale, and printing the state |
| + // without the reason leaves a member with nothing to act on. |
| + MastersError string `json:"masters_error,omitempty"` |
| + AuthorityError string `json:"authority_error,omitempty"` |
| } |
| // apiRelayReply is the one answer shape enable, status, disable and adopted |
| @@ -190,29 +202,53 @@ type apiRelayRotateReply struct { |
| RetireNoEarlierThan string `json:"retire_no_earlier_than"` // RFC3339 |
| } |
| -// relayRouteShape names which of spec §5's three delivery shapes a route is, |
| -// in relayDeliverable's OWN precedence (relayops.go), branch for branch: a |
| -// unit wins over a socket, an `instance` route is the platform's own — the |
| -// config sync restarts it whatever else the record says — and only then does |
| -// a bare TCP address read as the shape a member has to adopt by hand. |
| +// The five shapes `carlos relay status` reports a route as. The first three |
| +// are spec §5's delivery shapes; relayShapeNone is a route the config sync |
| +// can never reach at all, and relayShapeUnknown is the fail-closed answer for |
| +// a route that matches none of them. |
| +const ( |
| + relayShapeUnit = "unit" |
| + relayShapeSocket = "socket-exec" |
| + relayShapeFile = "tcp-exec-file" |
| + relayShapeNone = "no-delivery" |
| + relayShapeUnknown = "unknown" |
| +) |
| + |
| +// relayRouteShape names which shape a route is, in relayDeliverable's OWN |
| +// branch order (relayops.go), so that "what status calls this route" and |
| +// "what enable refused, or did not refuse, about it" can never disagree: |
| +// |
| +// 1. a named UNIT is a unit whatever else the row carries — the config sync |
| +// restarts it, so the key arrives without anyone being asked; |
| +// 2. a unix SOCKET is a socket-exec, restarted the same way; |
| +// 3. an `instance` route (Kind "" reads as instance) is the platform's OWN, |
| +// restarted by the config sync even though its record also carries the |
| +// TCP address the router dials. This branch sits ABOVE the address one |
| +// deliberately: reading that address as a foreign process would tell a |
| +// member to run `carlos relay adopted` for a route they must not adopt; |
| +// 4. only what is left — a TCP address belonging to a process this platform |
| +// never spawned — is the shape a member has to restart and acknowledge. |
| // |
| -// The instance branch sits ABOVE the address one for the same reason |
| -// relayDeliverable's does: an instance record carries the address the router |
| -// dials, and reading that address as "a TCP process this platform does not |
| -// own" would tell a member to run `carlos relay adopted` for a route the |
| -// platform restarts itself. |
| +// Whether that last shape can receive the file at all is a SEPARATE question |
| +// this function cannot answer (it needs the route's config bundle): |
| +// relayReply asks relayNoDelivery and downgrades the row to relayShapeNone. |
| +// |
| +// The default is relayShapeUnknown, not "unit". A route with no unit, no |
| +// socket, no address and a kind this code does not recognise is something |
| +// this console cannot describe, and calling it a unit would promise a |
| +// restart-on-write that nothing is going to perform. |
| func relayRouteShape(rt relayRoute) string { |
| switch { |
| case rt.Unit != "": |
| - return "unit" |
| + return relayShapeUnit |
| case rt.Socket != "": |
| - return "socket-exec" |
| + return relayShapeSocket |
| case rt.Kind == "" || rt.Kind == "instance": |
| - return "unit" |
| + return relayShapeUnit |
| case rt.Addr != "": |
| - return "tcp-exec-file" |
| + return relayShapeFile |
| default: |
| - return "unit" |
| + return relayShapeUnknown |
| } |
| } |
| @@ -247,35 +283,67 @@ func (s *Server) relayReply(ctx context.Context, acct *Account, app string, rec |
| // here for a relay key to ever reach. |
| continue |
| } |
| - row := apiRelayRoute{Host: rt.Host, Shape: relayRouteShape(rt), Delivery: "written, awaiting delivery"} |
| - if !rec.Enabled { |
| + shape := relayRouteShape(rt) |
| + // A route can be ADDED after an enable, and nothing re-runs enable's |
| + // refusal when it is. Asked here, every status answer names such a |
| + // route instead of reporting it as fine — which is the whole point of |
| + // spec §5's "a route added without delivery after enable shows 'no |
| + // config delivery'": it blocks the next retirement, and the member is |
| + // the only one who can fix it. |
| + if shape == relayShapeFile { |
| + none, err := s.relayNoDelivery(ctx, acct.Sqid, app, rt) |
| + if err != nil { |
| + return apiRelayReply{}, err |
| + } |
| + if none { |
| + shape = relayShapeNone |
| + } |
| + } |
| + row := apiRelayRoute{Host: rt.Host, Shape: shape} |
| + switch { |
| + case !rec.Enabled: |
| // A disabled record has no key to deliver: relayDisable removed |
| // the variable from the bundle. "Written, awaiting delivery" |
| // would name a write that has been UNDONE, and a member reading |
| // it would go looking for a key that is on its way out. |
| row.Delivery = "the relay key was removed from this app's config" |
| - } |
| - if cut := relayCutoverFor(rec, rt.Host); cut != nil && rec.Enabled { |
| - row.AckedAt = cut.At.UTC().Format(time.RFC3339) |
| - // An ack counts only for the identities it NAMED. relayAdopted |
| - // stamps the record's issued suffixes into the cutover for |
| - // exactly this comparison: a rotation since then re-issued the |
| - // app under a new kid, and reporting the older acknowledgement as |
| - // current would say a route runs a key it has never seen. |
| - if maps.Equal(cut.Suffixes, rec.IssuedSuffixes) { |
| - row.Cutover = true |
| - row.Delivery = "adopted by member at " + row.AckedAt |
| - } else { |
| - row.Delivery = "acknowledged at " + row.AckedAt + |
| - ", but the identities have changed since — re-run `carlos relay adopted --host " + rt.Host + "`" |
| + case shape == relayShapeNone: |
| + row.Delivery = "no config delivery: relay will not reach this route; " + |
| + "opt it into exec_delivery file or move it to a unit" |
| + default: |
| + row.Delivery = "written, awaiting delivery" |
| + if shape == relayShapeFile { |
| + // The ONE shape whose process this platform cannot bounce. |
| + // The console cannot know the file has landed until Task 12's |
| + // ack exists, but it can say whose move it is — and leaving |
| + // that out is how a route sits un-adopted for a month and |
| + // blocks a retirement nobody can explain. |
| + row.Delivery = "file delivered, awaiting `carlos relay adopted --host " + rt.Host + "` after you restart it" |
| + } |
| + if cut := relayCutoverFor(rec, rt.Host); cut != nil { |
| + row.AckedAt = cut.At.UTC().Format(time.RFC3339) |
| + // An ack counts only for the identities it NAMED. relayAdopted |
| + // stamps the record's issued suffixes into the cutover for |
| + // exactly this comparison: a rotation since then re-issued the |
| + // app under a new kid, and reporting the older acknowledgement |
| + // as current would say a route runs a key it has never seen. |
| + if maps.Equal(cut.Suffixes, rec.IssuedSuffixes) { |
| + row.Cutover = true |
| + row.Delivery = "adopted by member at " + row.AckedAt |
| + } else { |
| + row.Delivery = "acknowledged at " + row.AckedAt + |
| + ", but the identities have changed since — re-run `carlos relay adopted --host " + rt.Host + "`" |
| + } |
| } |
| } |
| // TODO(task 12): once internal/appconfig/sync.go writes |
| // control/config-ack/<box>/<route host>.json, read it here and |
| // report the fuller set of states spec §5 describes for a unit or |
| // socket-exec route ("adopted at <time>, restarted", "restart |
| - // failed", "hibernating, delivered on wake") instead of the two |
| - // states above, which is all the console can see without it. |
| + // failed", "hibernating, delivered on wake") instead of the states |
| + // above, which is all the console can see without it — including |
| + // whether the FILE half of a tcp-exec-file route's sentence has |
| + // actually happened yet. |
| out.Routes = append(out.Routes, row) |
| } |
| @@ -298,6 +366,13 @@ func (s *Server) relayReply(ctx context.Context, acct *Account, app string, rec |
| LeaseLive: !hb.ReadAt.IsZero() && now.Sub(hb.ReadAt) <= relay.LeaseMaxAge, |
| Stale: hb.ReadAt.IsZero() || now.Sub(hb.ReadAt) > relayHeartbeatFresh, |
| Current: auth != nil && !hb.WrittenAt.IsZero() && !hb.WrittenAt.Before(auth.WrittenAt), |
| + // The two "why has this relay stopped moving" fields, carried |
| + // straight through. A stale row without them tells a member |
| + // their calls will fail and nothing about who fixes it: a |
| + // masters error is the operator's staging, an authority error |
| + // is this console's bucket. |
| + MastersError: hb.MastersError, |
| + AuthorityError: hb.AuthorityError, |
| } |
| if hb.ReadAt.IsZero() { |
| row.HeartbeatAge = "never" |
| @@ -309,16 +384,23 @@ func (s *Server) relayReply(ctx context.Context, acct *Account, app string, rec |
| } |
| // "When a disable is complete" (spec §5): the app is out of a region's |
| - // authority once that region's own armed relay has heartbeated an |
| - // authority record written at or after the disable. rec.DisabledAt is |
| - // the record's own timestamp for that publish (relayDisable, relayops.go) |
| - // — close enough to the aggregate's own written_at for this purpose, |
| - // since the publish is the very next step in the same walk. |
| + // authority once the aggregate has been published without it AND that |
| + // region's own armed relay has heartbeated an authority record written at |
| + // or after that publish. The published record's OWN written_at is the |
| + // timestamp to compare against — the same one the Current field above |
| + // uses — because it is the object the relays actually read. rec.DisabledAt |
| + // is only this console's note of when it started the walk, and it stands |
| + // in when the aggregate has never been published at all (in which case no |
| + // relay can possibly have caught up, so every region is pending). |
| if !rec.Enabled && !rec.DisabledAt.IsZero() { |
| + publishedAt := rec.DisabledAt |
| + if auth != nil && !auth.WrittenAt.IsZero() { |
| + publishedAt = auth.WrittenAt |
| + } |
| for _, region := range rec.Regions { |
| revoked := false |
| for _, hb := range hbs { |
| - if hb.Region == region && !hb.WrittenAt.IsZero() && !hb.WrittenAt.Before(rec.DisabledAt) { |
| + if hb.Region == region && !hb.WrittenAt.IsZero() && !hb.WrittenAt.Before(publishedAt) { |
| revoked = true |
| break |
| } |
diff --git a/internal/console/apirelay_test.go b/internal/console/apirelay_test.go| index 8ad8777aa..105a816dc 100644 |
| --- a/internal/console/apirelay_test.go |
| +++ b/internal/console/apirelay_test.go |
| @@ -19,6 +19,8 @@ import ( |
| "testing" |
| "time" |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/relay" |
| ) |
| @@ -347,3 +349,180 @@ func TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise(t *testing.T) { |
| t.Fatalf("rotate with no region = %d, want 400", status) |
| } |
| } |
| + |
| +// ---- routes: what each shape is, and whose move it is ------------------------ |
| + |
| +// relayRouteRows indexes a status reply's routes by host, so an assertion |
| +// names the route it is about rather than a position in a list. |
| +func relayRouteRows(t *testing.T, out map[string]any) map[string]map[string]any { |
| + t.Helper() |
| + raw, _ := out["routes"].([]any) |
| + rows := map[string]map[string]any{} |
| + for _, r := range raw { |
| + row, ok := r.(map[string]any) |
| + if !ok { |
| + t.Fatalf("route row = %#v, want an object", r) |
| + } |
| + rows[str(t, row, "host")] = row |
| + } |
| + return rows |
| +} |
| + |
| +// wantRoute asserts one route row's shape and that its delivery sentence |
| +// contains each fragment. |
| +func wantRoute(t *testing.T, rows map[string]map[string]any, host, shape string, deliveryHas ...string) { |
| + t.Helper() |
| + row, ok := rows[host] |
| + if !ok { |
| + t.Fatalf("no route row for %s (have %v)", host, rows) |
| + } |
| + if got := str(t, row, "shape"); got != shape { |
| + t.Fatalf("%s shape = %q, want %q", host, got, shape) |
| + } |
| + got := str(t, row, "delivery") |
| + for _, want := range deliveryHas { |
| + if !strings.Contains(got, want) { |
| + t.Fatalf("%s delivery = %q, want it to contain %q", host, got, want) |
| + } |
| + } |
| +} |
| + |
| +// TestAPIRelayStatusNamesEachRouteShapeAndWhoseMoveItIs is the delivery half |
| +// of `carlos relay status`, which is the only sight a member with no box |
| +// access has of it. Four routes, four different answers: |
| +// |
| +// - a unit and a socket-exec are restarted by the config sync, so the key |
| +// arrives without anyone being asked and the row says only that it was |
| +// written; |
| +// - a TCP-address route with file delivery is the one shape this platform |
| +// cannot restart, so its row says whose move it is — spec §5's |
| +// "awaiting `carlos relay adopted`" — rather than leaving a member to |
| +// wonder why a rotation never completes; |
| +// - a route with NO config delivery, added after the enable that would have |
| +// refused it, is named as the fault it is. `adopted` refuses it too: a |
| +// cutover there would claim a process holds a key nothing ever wrote for |
| +// it, and a retirement would later be checked against that claim. |
| +// |
| +// Then the ack itself: recorded, it reads as adopted; after a rotation |
| +// re-issues the app under a new kid, the SAME ack must stop reading as |
| +// adopted, because it named identities that no longer exist. |
| +func TestAPIRelayStatusNamesEachRouteShapeAndWhoseMoveItIs(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-shapes") |
| + ctx := context.Background() |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-shapes-2") |
| + |
| + // One named bundle opts into file delivery; the app's default does not. |
| + // That is what lets two otherwise identical TCP routes differ, and it is |
| + // the per-route precedence relayExecDelivery applies (#195). |
| + if err := appconfig.SetExecDeliveryAt(ctx, fx.rel, fx.sqid, "titogo", "prod", appconfig.ExecDeliveryFile); err != nil { |
| + t.Fatal(err) |
| + } |
| + fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo", |
| + Kind: "service", Unit: "titogo-a.service", Status: "ready"}) |
| + fx.addRoute(t, registry.Route{Host: "b.titogo.oncarlos.com", App: "titogo", |
| + Kind: "service", Socket: "/run/titogo-b.sock", Status: "ready"}) |
| + fx.addRoute(t, registry.Route{Host: "c.titogo.oncarlos.com", App: "titogo", |
| + Kind: "service", Addr: "10.0.0.5:8080", Environment: "prod", Status: "ready"}) |
| + |
| + if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK { |
| + t.Fatalf("enable = %d %v, want 200", status, out) |
| + } |
| + // Added AFTER the enable, which is the only way it can exist: enable |
| + // itself refuses an app that already has one. |
| + fx.addRoute(t, registry.Route{Host: "d.titogo.oncarlos.com", App: "titogo", |
| + Kind: "service", Addr: "10.0.0.6:8080", Status: "ready"}) |
| + |
| + status, out := getJSON(t, relayURL(fx, "titogo"), token) |
| + if status != http.StatusOK { |
| + t.Fatalf("status = %d %v, want 200", status, out) |
| + } |
| + rows := relayRouteRows(t, out) |
| + if len(rows) != 4 { |
| + t.Fatalf("routes = %v, want four rows", rows) |
| + } |
| + wantRoute(t, rows, "a.titogo.oncarlos.com", "unit", "written, awaiting delivery") |
| + wantRoute(t, rows, "b.titogo.oncarlos.com", "socket-exec", "written, awaiting delivery") |
| + wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", |
| + "carlos relay adopted --host c.titogo.oncarlos.com") |
| + wantRoute(t, rows, "d.titogo.oncarlos.com", "no-delivery", |
| + "no config delivery", "exec_delivery file") |
| + |
| + // The undeliverable route cannot be acknowledged away. |
| + status, out = postJSON(t, relayURL(fx, "titogo")+"/adopted", |
| + map[string]any{"host": "d.titogo.oncarlos.com"}, token) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("adopted for a no-delivery route = %d %v, want 400", status, out) |
| + } |
| + if msg := str(t, out, "error"); !strings.Contains(msg, "no config delivery") { |
| + t.Fatalf("error = %q, want the reason named", msg) |
| + } |
| + |
| + // The deliverable one can, and then reads as adopted. |
| + if status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted", |
| + map[string]any{"host": "c.titogo.oncarlos.com"}, token); status != http.StatusOK { |
| + t.Fatalf("adopted = %d %v, want 200", status, out) |
| + } |
| + _, out = getJSON(t, relayURL(fx, "titogo"), token) |
| + rows = relayRouteRows(t, out) |
| + wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "adopted by member at") |
| + if rows["c.titogo.oncarlos.com"]["cutover"] != true { |
| + t.Fatalf("cutover = %v after an ack, want true", rows["c.titogo.oncarlos.com"]["cutover"]) |
| + } |
| + |
| + // A rotation re-issues the app under a new kid. The ack named the OLD |
| + // identities, so it must stop reading as adopted — otherwise a member |
| + // would believe a route runs a key it has never seen, and the old master |
| + // would be retired out from under it. |
| + t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) |
| + seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1", "k2") |
| + if _, err := fx.srv.relayRotate(ctx, "eu-west-1"); err != nil { |
| + t.Fatalf("rotate: %v", err) |
| + } |
| + _, out = getJSON(t, relayURL(fx, "titogo"), token) |
| + rows = relayRouteRows(t, out) |
| + wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", |
| + "the identities have changed since", "carlos relay adopted --host c.titogo.oncarlos.com") |
| + if rows["c.titogo.oncarlos.com"]["cutover"] != false { |
| + t.Fatalf("cutover = %v after a rotation, want false", rows["c.titogo.oncarlos.com"]["cutover"]) |
| + } |
| + if rows["c.titogo.oncarlos.com"]["acked_at"] == nil { |
| + t.Fatal("acked_at was dropped — the ack still happened, it just no longer counts") |
| + } |
| +} |
| + |
| +// TestAPIRelayRotateStagingRefusalsAreClientErrors pins the two refusal |
| +// sentences relayLooksLikeClientError matches by wording rather than by |
| +// sentinel: nothing new staged, and no live relay in the region at all. |
| +// Both are the OPERATOR's to fix by finishing the runbook, so both are a 400 |
| +// naming what is wrong — and if either sentence is ever reworded without |
| +// updating the matcher, they become a 502 that reads as "the console broke" |
| +// and this test is what says so. |
| +func TestAPIRelayRotateStagingRefusalsAreClientErrors(t *testing.T) { |
| + fx, _ := relayFixture(t, "code-relay-staging") |
| + token := cliToken(t, fx, "alice@keymail.dev", "code-relay-staging-2") |
| + fx.srv.cfg.RelayOperators = []string{"alice@keymail.dev"} |
| + rotate := fx.ts.URL + "/api/cli/relay/rotate" |
| + |
| + // (a) nothing new staged: the only entry is already the issuer. |
| + seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1") |
| + status, out := postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("rotate with nothing new staged = %d %v, want 400", status, out) |
| + } |
| + if msg := str(t, out, "error"); !strings.Contains(msg, "no new master is staged") { |
| + t.Fatalf("error = %q, want the nothing-staged sentence", msg) |
| + } |
| + |
| + // (b) a new master staged, but every heartbeat in the region is stale. |
| + // Issuing now would mint credentials nothing can verify, and a dead relay |
| + // must not be able to consent by silence. |
| + t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2)) |
| + seedHeartbeat(t, fx, "box1", "eu-west-1", 6*time.Minute, "k1", "k2") |
| + status, out = postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token) |
| + if status != http.StatusBadRequest { |
| + t.Fatalf("rotate with no live relay = %d %v, want 400", status, out) |
| + } |
| + if msg := str(t, out, "error"); !strings.Contains(msg, "has heartbeated in the last") { |
| + t.Fatalf("error = %q, want the no-live-relay sentence", msg) |
| + } |
| +} |
diff --git a/internal/console/relayops.go b/internal/console/relayops.go| index 924219ef8..ad0916ad3 100644 |
| --- a/internal/console/relayops.go |
| +++ b/internal/console/relayops.go |
| @@ -858,10 +858,8 @@ func (s *Server) relayExecDelivery(ctx context.Context, acct, app, envName strin |
| return m.ExecDelivery == appconfig.ExecDeliveryFile, nil |
| } |
| -// relayDeliverable is enable's second refusal (spec §5, step 2): a route that |
| -// could never receive a rotated key must not be enabled, because it would |
| -// block every future retirement forever and nobody would find out until the |
| -// rotation stalled. |
| +// relayNoDelivery answers the ONE question three callers need about a route: |
| +// can the config sync put a relay key in front of its process at all? |
| // |
| // The three shapes: an `instance` route is the platform's own — it has a |
| // socket or a unit and the config sync restarts it, so it is deliverable. A |
| @@ -869,27 +867,39 @@ func (s *Server) relayExecDelivery(ctx context.Context, acct, app, envName strin |
| // deliverable only if it has a socket or a unit to bounce, or its bundle opts |
| // into file delivery — a bare TCP address whose process this platform never |
| // spawned is exactly the route the spec refuses. |
| +// |
| +// It lives in one function because the three callers must agree: enable |
| +// REFUSES such a route (below), `carlos relay status` NAMES it (relayReply, |
| +// apirelay.go), and `carlos relay adopted` refuses to record a cutover for |
| +// one (relayAdopted). Three copies of this switch would eventually let a |
| +// member acknowledge a key that had never been written for them. |
| +func (s *Server) relayNoDelivery(ctx context.Context, acct, app string, rt relayRoute) (bool, error) { |
| + switch { |
| + case rt.Kind == "static", |
| + rt.Kind == "" || rt.Kind == "instance", |
| + rt.Socket != "" || rt.Unit != "", |
| + rt.Addr == "": |
| + return false, nil |
| + } |
| + on, err := s.relayExecDelivery(ctx, acct, app, rt.Environment) |
| + return !on, err |
| +} |
| + |
| +// relayDeliverable is enable's second refusal (spec §5, step 2): a route that |
| +// could never receive a rotated key must not be enabled, because it would |
| +// block every future retirement forever and nobody would find out until the |
| +// rotation stalled. |
| func (s *Server) relayDeliverable(ctx context.Context, acct, app string) error { |
| routes, err := s.relayAppRoutes(ctx, acct, app) |
| if err != nil { |
| return err |
| } |
| for _, rt := range routes { |
| - switch { |
| - case rt.Kind == "static": |
| - continue |
| - case rt.Kind == "" || rt.Kind == "instance": |
| - continue |
| - case rt.Socket != "" || rt.Unit != "": |
| - continue |
| - case rt.Addr == "": |
| - continue |
| - } |
| - on, err := s.relayExecDelivery(ctx, acct, app, rt.Environment) |
| + none, err := s.relayNoDelivery(ctx, acct, app, rt) |
| if err != nil { |
| return err |
| } |
| - if !on { |
| + if none { |
| return fmt.Errorf("route %s is a TCP address this platform does not own and its config bundle does not opt into file delivery, "+ |
| "so it could never pick up a rotated relay key — set it with `carlos env exec-delivery file` first: %w", rt.Host, errRelayUndelivered) |
| } |
| @@ -1068,9 +1078,23 @@ func (s *Server) relayAdopted(ctx context.Context, acct *Account, app, host, by |
| if err != nil { |
| return nil, err |
| } |
| - if !slices.ContainsFunc(routes, func(rt relayRoute) bool { return rt.Host == host }) { |
| + i := slices.IndexFunc(routes, func(rt relayRoute) bool { return rt.Host == host }) |
| + if i < 0 { |
| return nil, fmt.Errorf("%s is not a route of this app: %w", host, errRelayUnknownHost) |
| } |
| + // A route the config sync cannot reach has been sent NOTHING to adopt, so |
| + // an acknowledgement of it would be false — and worse than merely false: |
| + // a cutover is what the retirement of an old master is eventually checked |
| + // against, and one recorded here would say a process has a key it has |
| + // never seen. Such a route can only appear after an enable (enable itself |
| + // refuses it), which is exactly when a member might reach for this verb |
| + // to make the warning go away. |
| + if none, err := s.relayNoDelivery(ctx, acct.Sqid, app, routes[i]); err != nil { |
| + return nil, err |
| + } else if none { |
| + return nil, fmt.Errorf("%s has no config delivery, so no relay key has been written for it to adopt — "+ |
| + "opt it into file delivery with `carlos env exec-delivery file` or move it to a unit: %w", host, errRelayUndelivered) |
| + } |
| cut := relay.RouteCutover{ |
| Host: host, |
| Suffixes: maps.Clone(rec.IssuedSuffixes), |
diff --git a/relay.go b/relay.go| index d674440b3..a9b69efe8 100644 |
| --- a/relay.go |
| +++ b/relay.go |
| @@ -51,7 +51,8 @@ const relayUsage = `usage: carlos relay <enable|status|disable|adopted|rotate|ke |
| and each region relay's heartbeat, lease and loaded masters. The |
| only sight of any of that without box access. |
| disable --app <a> |
| - remove the key and end every allocation the app holds. |
| + remove the key. Every allocation the app holds ends on the |
| + relays' next authority read, within a minute. |
| adopted --app <a> --host <route> |
| tell the platform that a TCP-address route you restarted by hand |
| now runs the issued key. The one route shape this platform cannot |
| @@ -344,11 +345,19 @@ func splitList(s string) []string { |
| func printRelayStatus(w io.Writer, res apiRelayReply) { |
| c := colorFor(w) |
| fmt.Fprintf(w, "%s %s\n", c.head("relay for"), res.App) |
| - if !res.Enabled { |
| - mutedf(w, "not enabled — carlos relay enable --app %s", res.App) |
| - } else { |
| + switch { |
| + case res.Enabled: |
| fmt.Fprintf(w, " enabled in %s, incarnation %s (op %d, %s)\n", |
| strings.Join(res.Regions, ", "), res.Incarnation, res.Op, res.Phase) |
| + case res.Incarnation != "": |
| + // A record that HAS been enabled and is now off still carries an |
| + // operation and a phase, and both matter: a disable that stopped at |
| + // phase "config" is one the reconcile has yet to finish, and hiding |
| + // that behind "not enabled" would make a half-done disable look |
| + // complete. |
| + fmt.Fprintf(w, " disabled (op %d, %s)\n", res.Op, res.Phase) |
| + default: |
| + mutedf(w, "not enabled — carlos relay enable --app %s", res.App) |
| } |
| // The issued identity per region: account, app, incarnation and kid, the |
| // public half of the credential the app mints per call. Printed so a |
| @@ -359,11 +368,29 @@ func printRelayStatus(w io.Writer, res apiRelayReply) { |
| } |
| } |
| for _, rt := range res.Routes { |
| - fmt.Fprintf(w, " route %s (%s): %s\n", rt.Host, rt.Shape, rt.Delivery) |
| + line := fmt.Sprintf(" route %s (%s): %s", rt.Host, rt.Shape, rt.Delivery) |
| + // A route the relay key will never reach is the one row here that is |
| + // a fault rather than a state: it blocks the next master retirement |
| + // until someone fixes it, and only the member can. |
| + if rt.Shape == "no-delivery" { |
| + warnf(w, "%s", line) |
| + continue |
| + } |
| + fmt.Fprintln(w, line) |
| } |
| if len(res.Routes) == 0 && res.Enabled { |
| mutedf(w, " no routes yet — the key is written and waits for the first one") |
| } |
| + // Disable progress before region health, spec §5's own order: when a |
| + // member has just disabled, "is it off everywhere yet" is the question |
| + // they typed the command to answer. |
| + if len(res.DisablePending) > 0 { |
| + warnf(w, " disable pending on %s — those relays have not read the record without this app yet; "+ |
| + "allocations there end on their next authority read, within a minute", |
| + strings.Join(res.DisablePending, ", ")) |
| + } else if !res.Enabled && res.Incarnation != "" { |
| + fmt.Fprintln(w, " revoked: every relay has read a record without this app") |
| + } |
| for _, rg := range res.RegionStatus { |
| line := fmt.Sprintf(" relay %s in %s: heartbeat %s", rg.Box, rg.Region, rg.HeartbeatAge) |
| if !rg.Current { |
| @@ -379,8 +406,11 @@ func printRelayStatus(w io.Writer, res apiRelayReply) { |
| case rg.Stale: |
| // STALE is the important word and it gets the warning colour: a |
| // relay that stopped reading the authority keeps serving until its |
| - // lease runs out, so "up" and "current" are different facts. |
| - warnf(w, "%s — STALE (no successful read in over five minutes)", line) |
| + // lease runs out, so "up" and "current" are different facts. The |
| + // relay's own last error, when it reported one, is the whole |
| + // difference between a member filing a bug and an operator |
| + // fixing a masters file. |
| + warnf(w, "%s — STALE (no successful read in over five minutes)%s", line, relayWhyStale(rg)) |
| case !rg.LeaseLive: |
| warnf(w, "%s — lease expired, this relay is no longer serving", line) |
| default: |
| @@ -388,15 +418,25 @@ func printRelayStatus(w io.Writer, res apiRelayReply) { |
| } |
| } |
| if res.Enabled && len(res.RegionStatus) == 0 { |
| - warnf(w, "no relay in %s has heartbeated at all — calls will not connect through TURN", |
| + warnf(w, " no relay in %s has heartbeated at all — calls will not connect through TURN", |
| strings.Join(res.Regions, ", ")) |
| } |
| - if len(res.DisablePending) > 0 { |
| - warnf(w, "disable pending on %s — those relays have not read the record without this app yet; "+ |
| - "allocations there end on their next read", strings.Join(res.DisablePending, ", ")) |
| - } else if !res.Enabled && res.Incarnation != "" { |
| - fmt.Fprintln(w, " revoked: every relay has read a record without this app") |
| +} |
| + |
| +// relayWhyStale is the reason half of a stale row: the relay's own last |
| +// masters-file error, its last authority-read error, or nothing when it |
| +// reported neither (a relay that simply stopped writing heartbeats has no |
| +// error to tell us about, and inventing one would be worse than the silence). |
| +func relayWhyStale(rg apiRelayRegion) string { |
| + switch { |
| + case rg.MastersError != "" && rg.AuthorityError != "": |
| + return ": masters " + rg.MastersError + "; authority " + rg.AuthorityError |
| + case rg.MastersError != "": |
| + return ": masters " + rg.MastersError |
| + case rg.AuthorityError != "": |
| + return ": authority " + rg.AuthorityError |
| } |
| + return "" |
| } |
| // printRelayRotate is the operator's answer: which master the region issues |
diff --git a/relay_test.go b/relay_test.go| new file mode 100644 |
| index 000000000..58cf8343d |
| --- /dev/null |
| +++ b/relay_test.go |
| @@ -0,0 +1,335 @@ |
| +// Tests for `carlos relay`'s client half: the wire shape the five apiSession |
| +// methods send (matching internal/console/apirelay_test.go's server side), |
| +// and the honesty rules printRelayStatus and printRelayRotate must follow. |
| +// |
| +// The printers get the most attention here, because `carlos relay status` is |
| +// the ONLY sight a member with no box access has of whether their calls can |
| +// connect. Every rule pinned below is one where a wrong line would leave them |
| +// believing something the console did not say: that a route has the key when |
| +// it is waiting on them, that a relay is healthy when it stopped reading five |
| +// minutes ago, that a disable is finished when allocations are still live. |
| +package main |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "encoding/json" |
| + "net/http" |
| + "net/http/httptest" |
| + "strings" |
| + "testing" |
| +) |
| + |
| +// ---- apiSession.relay* wire shape ------------------------------------------- |
| + |
| +// TestAPISessionRelayWireShapes pins the path and body of each of the four |
| +// app-scoped calls plus rotate. rotate is the one with no account segment at |
| +// all — a literal path, because there is nothing per-app about moving a |
| +// region's issuer pointer. |
| +func TestAPISessionRelayWireShapes(t *testing.T) { |
| + var gotPath, gotMethod, gotAuth, gotBody string |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + gotPath, gotMethod, gotAuth = r.URL.RequestURI(), r.Method, r.Header.Get("Authorization") |
| + body := new(bytes.Buffer) |
| + body.ReadFrom(r.Body) |
| + gotBody = strings.TrimSpace(body.String()) |
| + json.NewEncoder(w).Encode(map[string]any{"app": "meet", "enabled": true}) |
| + })) |
| + defer ts.Close() |
| + sess := newTestSession(ts.URL) |
| + a := apiAcct{Seg: "meet-team"} |
| + |
| + if _, err := sess.relayEnable(context.Background(), a, "meet", nil); err != nil { |
| + t.Fatalf("relayEnable: %v", err) |
| + } |
| + if gotMethod != http.MethodPost || gotPath != "/api/cli/apps/meet-team/meet/relay" { |
| + t.Fatalf("enable = %s %s", gotMethod, gotPath) |
| + } |
| + // No regions named sends the literal {} the door documents as its |
| + // default, never {"regions":null}. |
| + if gotBody != "{}" { |
| + t.Fatalf("enable body = %q, want {}", gotBody) |
| + } |
| + if gotAuth != "Bearer cli-token" { |
| + t.Fatalf("auth = %q", gotAuth) |
| + } |
| + |
| + if _, err := sess.relayEnable(context.Background(), a, "meet", []string{"eu-west-1"}); err != nil { |
| + t.Fatalf("relayEnable(regions): %v", err) |
| + } |
| + if gotBody != `{"regions":["eu-west-1"]}` { |
| + t.Fatalf("enable body with regions = %q", gotBody) |
| + } |
| + |
| + if _, err := sess.relayGet(context.Background(), a, "meet"); err != nil { |
| + t.Fatalf("relayGet: %v", err) |
| + } |
| + if gotMethod != http.MethodGet || gotPath != "/api/cli/apps/meet-team/meet/relay" { |
| + t.Fatalf("status = %s %s", gotMethod, gotPath) |
| + } |
| + |
| + if _, err := sess.relayDisable(context.Background(), a, "meet"); err != nil { |
| + t.Fatalf("relayDisable: %v", err) |
| + } |
| + if gotMethod != http.MethodPost || gotPath != "/api/cli/apps/meet-team/meet/relay/disable" { |
| + t.Fatalf("disable = %s %s", gotMethod, gotPath) |
| + } |
| + |
| + if _, err := sess.relayAdopted(context.Background(), a, "meet", "x.meet.example"); err != nil { |
| + t.Fatalf("relayAdopted: %v", err) |
| + } |
| + if gotPath != "/api/cli/apps/meet-team/meet/relay/adopted" { |
| + t.Fatalf("adopted path = %s", gotPath) |
| + } |
| + if gotBody != `{"host":"x.meet.example"}` { |
| + t.Fatalf("adopted body = %q", gotBody) |
| + } |
| + |
| + if _, err := sess.relayRotate(context.Background(), "eu-west-1"); err != nil { |
| + t.Fatalf("relayRotate: %v", err) |
| + } |
| + if gotPath != "/api/cli/relay/rotate" { |
| + t.Fatalf("rotate path = %s, want the literal deployment-wide door", gotPath) |
| + } |
| + if gotBody != `{"region":"eu-west-1"}` { |
| + t.Fatalf("rotate body = %q", gotBody) |
| + } |
| +} |
| + |
| +// TestAPISessionRelayRotateNotFoundNamesTheOperatorList: the rotate door |
| +// answers a non-operator with the mux's own byte-identical 404, which is |
| +// deliberate — a 403 would confirm to any token holder that this deployment |
| +// runs a relay. So the CLI cannot tell "not allowed" from "no such door", and |
| +// the message must say BOTH rather than fall back to the generic membership |
| +// advice, which names --account and --app: two flags this verb does not have, |
| +// sending an operator to check something they cannot even type. |
| +func TestAPISessionRelayRotateNotFoundNamesTheOperatorList(t *testing.T) { |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + http.NotFound(w, r) |
| + })) |
| + defer ts.Close() |
| + |
| + _, err := newTestSession(ts.URL).relayRotate(context.Background(), "eu-west-1") |
| + if err == nil { |
| + t.Fatal("rotate against a 404 door returned no error") |
| + } |
| + if !strings.Contains(err.Error(), "CARLOS_RELAY_OPERATORS") { |
| + t.Fatalf("error = %q, want the operator list named", err) |
| + } |
| + if strings.Contains(err.Error(), "--account") || strings.Contains(err.Error(), "--app") { |
| + t.Fatalf("error = %q, want no advice about flags this verb does not have", err) |
| + } |
| +} |
| + |
| +// TestAPISessionRelayServerDiagnosisSurvives: a 404 that DOES carry a JSON |
| +// error is a real server sentence ("the relay has never been enabled...") and |
| +// must reach the member verbatim, not be rewritten into the generic one. |
| +func TestAPISessionRelayServerDiagnosisSurvives(t *testing.T) { |
| + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + w.WriteHeader(http.StatusNotFound) |
| + json.NewEncoder(w).Encode(map[string]any{"error": "the relay has never been enabled for this app"}) |
| + })) |
| + defer ts.Close() |
| + |
| + _, err := newTestSession(ts.URL).relayGet(context.Background(), apiAcct{Seg: "t"}, "meet") |
| + if err == nil || !strings.Contains(err.Error(), "never been enabled") { |
| + t.Fatalf("relayGet 404 = %v, want the console's own text", err) |
| + } |
| +} |
| + |
| +// ---- printRelayStatus -------------------------------------------------------- |
| +// |
| +// Driven against literal replies, no stub server: the wording rules are the |
| +// thing under test, and they must not depend on the wire plumbing above. |
| + |
| +func TestPrintRelayStatus(t *testing.T) { |
| + healthy := apiRelayReply{ |
| + App: "meet", Enabled: true, Incarnation: "0123456789abcdef", |
| + Regions: []string{"eu-west-1"}, Op: 3, Phase: "done", |
| + Issued: map[string]string{"eu-west-1": "abc:meet:0123456789abcdef:k1"}, |
| + RegionStatus: []apiRelayRegion{{ |
| + Region: "eu-west-1", Box: "relay-1", HeartbeatAge: "12s", Current: true, |
| + LeaseLive: true, Listeners: []string{"udp:3478"}, Kids: []string{"k1"}, |
| + }}, |
| + } |
| + |
| + cases := []struct { |
| + name string |
| + res apiRelayReply |
| + want []string |
| + wantNot []string |
| + modifier func(*apiRelayReply) |
| + }{ |
| + { |
| + name: "healthy", |
| + res: healthy, |
| + want: []string{"relay for meet", "enabled in eu-west-1", "0123456789abcdef", |
| + "op 3", "done", "issued eu-west-1: abc:meet:0123456789abcdef:k1", |
| + "relay relay-1 in eu-west-1: heartbeat 12s", "udp:3478", "masters k1", "lease live"}, |
| + // A healthy relay must never be described as stale or as holding |
| + // an old record — the two words a member acts on. |
| + wantNot: []string{"STALE", "older record", "disable pending"}, |
| + }, |
| + { |
| + name: "stale names the relay's own reason", |
| + res: healthy, |
| + modifier: func(r *apiRelayReply) { |
| + r.RegionStatus = []apiRelayRegion{{ |
| + Region: "eu-west-1", Box: "relay-1", HeartbeatAge: "6m0s", Current: false, |
| + LeaseLive: true, Stale: true, Kids: []string{"k1"}, |
| + AuthorityError: "get control/relay.json: 503", |
| + }} |
| + }, |
| + want: []string{"STALE", "holding an older record", "6m0s", "authority", "503"}, |
| + // Still inside its fifteen-minute lease, so it IS still serving: |
| + // saying otherwise would send someone hunting a dead box. |
| + wantNot: []string{"no longer serving"}, |
| + }, |
| + { |
| + name: "lease expired", |
| + res: healthy, |
| + modifier: func(r *apiRelayReply) { |
| + r.RegionStatus = []apiRelayRegion{{ |
| + Region: "eu-west-1", Box: "relay-1", HeartbeatAge: "40m0s", |
| + Current: true, LeaseLive: false, Stale: false, |
| + }} |
| + }, |
| + want: []string{"lease expired", "no longer serving"}, |
| + }, |
| + { |
| + name: "no heartbeat at all", |
| + res: healthy, |
| + modifier: func(r *apiRelayReply) { |
| + r.RegionStatus = nil |
| + }, |
| + want: []string{"no relay in eu-west-1 has heartbeated at all", "will not connect"}, |
| + wantNot: []string{"lease live"}, |
| + }, |
| + { |
| + name: "disable pending", |
| + res: healthy, |
| + modifier: func(r *apiRelayReply) { |
| + r.Enabled, r.Op, r.Phase = false, 4, "done" |
| + r.Issued = nil |
| + r.DisablePending = []string{"eu-west-1"} |
| + }, |
| + // A disabled record still shows its operation and phase: a |
| + // disable stuck at phase "config" is not the same state as one |
| + // that finished, and "not enabled" alone would hide the |
| + // difference. |
| + want: []string{"disabled (op 4, done)", "disable pending on eu-west-1", |
| + "next authority read, within a minute"}, |
| + wantNot: []string{"revoked"}, |
| + }, |
| + { |
| + name: "revoked", |
| + res: healthy, |
| + modifier: func(r *apiRelayReply) { |
| + r.Enabled, r.Op = false, 4 |
| + r.Issued = nil |
| + }, |
| + want: []string{"revoked: every relay has read a record without this app"}, |
| + wantNot: []string{"disable pending"}, |
| + }, |
| + { |
| + name: "never enabled", |
| + res: apiRelayReply{App: "meet"}, |
| + want: []string{"not enabled", "carlos relay enable --app meet"}, |
| + // Nothing was ever issued, so there is no operation to report. |
| + wantNot: []string{"op 0", "revoked"}, |
| + }, |
| + { |
| + name: "route deliveries", |
| + res: healthy, |
| + modifier: func(r *apiRelayReply) { |
| + r.Routes = []apiRelayRoute{ |
| + {Host: "a.meet", Shape: "unit", Delivery: "written, awaiting delivery"}, |
| + {Host: "b.meet", Shape: "socket-exec", Delivery: "written, awaiting delivery"}, |
| + {Host: "c.meet", Shape: "tcp-exec-file", |
| + Delivery: "file delivered, awaiting `carlos relay adopted --host c.meet` after you restart it"}, |
| + {Host: "d.meet", Shape: "no-delivery", |
| + Delivery: "no config delivery: relay will not reach this route; opt it into exec_delivery file or move it to a unit"}, |
| + {Host: "e.meet", Shape: "tcp-exec-file", Cutover: true, |
| + AckedAt: "2026-09-07T10:00:00Z", Delivery: "adopted by member at 2026-09-07T10:00:00Z"}, |
| + } |
| + }, |
| + want: []string{ |
| + "route a.meet (unit): written, awaiting delivery", |
| + "route b.meet (socket-exec): written", |
| + "route c.meet (tcp-exec-file): file delivered, awaiting `carlos relay adopted --host c.meet`", |
| + "route d.meet (no-delivery): no config delivery", |
| + "route e.meet (tcp-exec-file): adopted by member at 2026-09-07T10:00:00Z", |
| + }, |
| + // The lesson restart.go paid for: never say a box has acted. |
| + wantNot: []string{"no routes yet"}, |
| + }, |
| + { |
| + name: "enabled with no routes", |
| + res: healthy, |
| + want: []string{"no routes yet"}, |
| + }, |
| + } |
| + |
| + for _, tc := range cases { |
| + t.Run(tc.name, func(t *testing.T) { |
| + res := tc.res |
| + if tc.modifier != nil { |
| + tc.modifier(&res) |
| + } |
| + var buf bytes.Buffer |
| + printRelayStatus(&buf, res) |
| + out := buf.String() |
| + for _, want := range tc.want { |
| + if !strings.Contains(out, want) { |
| + t.Fatalf("missing %q:\n%s", want, out) |
| + } |
| + } |
| + for _, no := range tc.wantNot { |
| + if strings.Contains(out, no) { |
| + t.Fatalf("must not contain %q:\n%s", no, out) |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestPrintRelayStatusNeverClaimsDelivery is the output-honesty rule of this |
| +// whole file in one assertion: the console can say a key was WRITTEN and it |
| +// can repeat a member's own acknowledgement, but nothing here may claim a |
| +// running process has picked anything up. Only Task 12's per-route ack can |
| +// say that, and until it exists no sentence may imply it. |
| +func TestPrintRelayStatusNeverClaimsDelivery(t *testing.T) { |
| + var buf bytes.Buffer |
| + printRelayStatus(&buf, apiRelayReply{ |
| + App: "meet", Enabled: true, Incarnation: "0123456789abcdef", |
| + Regions: []string{"eu-west-1"}, Phase: "done", |
| + Routes: []apiRelayRoute{{Host: "a.meet", Shape: "unit", Delivery: "written, awaiting delivery"}}, |
| + }) |
| + out := strings.ToLower(buf.String()) |
| + for _, mustNot := range []string{"is using", "now running the", "has the key", "picked it up"} { |
| + if strings.Contains(out, mustNot) { |
| + t.Fatalf("printRelayStatus over-claims delivery (%q):\n%s", mustNot, buf.String()) |
| + } |
| + } |
| +} |
| + |
| +// ---- printRelayRotate -------------------------------------------------------- |
| + |
| +// TestPrintRelayRotate: the operator's answer has to carry the retirement |
| +// deadline, because retiring the old master early kills credentials still |
| +// inside their own four-hour lifetime — and nothing else in this system will |
| +// remind them. |
| +func TestPrintRelayRotate(t *testing.T) { |
| + var buf bytes.Buffer |
| + printRelayRotate(&buf, apiRelayRotateReply{ |
| + Region: "eu-west-1", Kid: "k2", Apps: 7, |
| + RetireNoEarlierThan: "2026-09-08T16:00:00Z", |
| + }) |
| + out := buf.String() |
| + for _, want := range []string{"rotated eu-west-1", "kid k2", "7 app(s) re-issued", |
| + "no earlier than 2026-09-08T16:00:00Z", "/etc/carlos/relay-masters"} { |
| + if !strings.Contains(out, want) { |
| + t.Fatalf("printRelayRotate missing %q:\n%s", want, out) |
| + } |
| + } |
| +} |