amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package console
2
3// The relay's doors (relay spec §5), through the real mux and a real bearer
4// token. What is pinned here, in the order it would hurt if lost: a stranger
5// probing the relay learns nothing — its refusal is byte-for-byte the STORE
6// door's refusal, so the two cannot be told apart by anything but the path;
7// a deployment with no armed region refuses to enable and NAMES the variable
8// an operator would set, rather than minting a key nothing can verify; status
9// tells the truth about a region whose relay has gone quiet, because "stale"
10// is the only way a member with no box access finds out; and rotate — the one
11// door with no account in its path — answers a caller who is not on
12// CARLOS_RELAY_OPERATORS with the same 404 an unknown account gets, never a
13// 403 that would confirm this deployment runs a relay at all.
14
15import (
16 "context"
17 "maps"
18 "net/http"
19 "strings"
20 "testing"
21 "time"
22
23 "github.com/carlosframework/platform/internal/appconfig"
24 "github.com/carlosframework/platform/internal/provision"
25 "github.com/carlosframework/platform/internal/registry"
26 "github.com/carlosframework/platform/internal/relay"
27)
28
29// relayURL is the app-scoped family's root: enable POSTs here, status GETs
30// here, and disable/adopted hang off it.
31func relayURL(fx *consoleFixture, app string) string {
32 return fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/" + app + "/relay"
33}
34
35// seedHeartbeat writes one armed relay's heartbeat as the box itself would,
36// with its last SUCCESSFUL authority read `age` in the past — the clock
37// status reads, and the one a dead relay cannot move.
38func seedHeartbeat(t *testing.T, fx *consoleFixture, box, region string, age time.Duration, kids ...string) {
39 t.Helper()
40 seedHeartbeatRetiring(t, fx, box, region, age, nil, kids...)
41}
42
43// seedHeartbeatRetiring is seedHeartbeat with the retiring map set, for the
44// one test that covers the hop from a relay's heartbeat to the status reply.
45func seedHeartbeatRetiring(t *testing.T, fx *consoleFixture, box, region string, age time.Duration, retiring map[string]time.Time, kids ...string) {
46 t.Helper()
47 now := time.Now()
48 hb := relay.Heartbeat{
49 Box: box, Region: region, ReadAt: now.Add(-age), WrittenAt: now.Add(-age),
50 Listeners: []string{"udp:3478", "tcp:3478"}, Kids: kids, Retiring: retiring,
51 }
52 if err := relay.WriteHeartbeat(context.Background(), fx.rel, hb); err != nil {
53 t.Fatalf("seed heartbeat: %v", err)
54 }
55}
56
57// region pulls one row out of a status reply's region_status list.
58func regionRow(t *testing.T, out map[string]any, i int) map[string]any {
59 t.Helper()
60 rows, ok := out["region_status"].([]any)
61 if !ok || len(rows) <= i {
62 t.Fatalf("region_status = %#v, want at least %d row(s)", out["region_status"], i+1)
63 }
64 row, ok := rows[i].(map[string]any)
65 if !ok {
66 t.Fatalf("region_status[%d] = %#v, want an object", i, rows[i])
67 }
68 return row
69}
70
71// ---- enable, status, disable -------------------------------------------------
72
73// TestAPIRelayEnableThenStatusThenDisable is the member's whole path through
74// the doors: turn it on, read what was issued and how the region looks, turn
75// it off. Each assertion is on something only the door can get wrong — the
76// operations themselves are pinned in relayops_test.go.
77func TestAPIRelayEnableThenStatusThenDisable(t *testing.T) {
78 fx, _ := relayFixture(t, "code-relay-door")
79
80 status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, cliToken(t, fx, "alice@keymail.dev", "code-relay-door-2"))
81 if status != http.StatusOK {
82 t.Fatalf("enable = %d %v, want 200", status, out)
83 }
84 if out["enabled"] != true {
85 t.Fatalf("enabled = %v, want true", out["enabled"])
86 }
87 inc := str(t, out, "incarnation")
88 if len(inc) != 16 {
89 t.Fatalf("incarnation = %q, want 16 hex characters", inc)
90 }
91 if got := strList(t, out, "regions"); len(got) != 1 || got[0] != "eu-west-1" {
92 t.Fatalf("regions = %v, want the one armed region", got)
93 }
94 // The username suffix is echoed, the key never is: the suffix is the same
95 // public identity the app's own CARLOS_RELAY carries, and a member needs
96 // it to match a call's credential against a rotation.
97 issued, ok := out["issued"].(map[string]any)
98 if !ok || !strings.Contains(issued["eu-west-1"].(string), inc) {
99 t.Fatalf("issued = %#v, want a suffix naming the incarnation", out["issued"])
100 }
101 // Nothing else: no key, no envelope, no variable. The record holds the
102 // digest of what was sealed and the console cannot open it anyway, but a
103 // door that grew a "value" field would be the one place a key could leak
104 // back out of this system.
105 for _, k := range []string{"value", "key", "envelope", "secret", "master"} {
106 if _, ok := out[k]; ok {
107 t.Fatalf("the enable reply carries %q — nothing key-shaped may cross this door", k)
108 }
109 }
110
111 // A relay that read the authority six minutes ago is STALE, and the lease
112 // it serves on is still live (fifteen minutes) — two different clocks, and
113 // a member debugging a failing call needs both.
114 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-door-3")
115 seedHeartbeat(t, fx, "box1", "eu-west-1", 6*time.Minute, "k1")
116 status, out = getJSON(t, relayURL(fx, "titogo"), token)
117 if status != http.StatusOK {
118 t.Fatalf("status = %d %v, want 200", status, out)
119 }
120 row := regionRow(t, out, 0)
121 if row["region"] != "eu-west-1" || row["box"] != "box1" {
122 t.Fatalf("region row = %#v, want the seeded box", row)
123 }
124 if row["stale"] != true {
125 t.Fatalf("stale = %v on a six-minute-old heartbeat, want true", row["stale"])
126 }
127 if row["lease_live"] != true {
128 t.Fatalf("lease_live = %v six minutes in, want true (the lease runs 15)", row["lease_live"])
129 }
130 if age, _ := row["heartbeat_age"].(string); !strings.HasPrefix(age, "6m") {
131 t.Fatalf("heartbeat_age = %v, want about six minutes", row["heartbeat_age"])
132 }
133 if kids := strList(t, row, "kids"); len(kids) != 1 || kids[0] != "k1" {
134 t.Fatalf("kids = %v, want the loaded master", kids)
135 }
136
137 status, out = postJSON(t, relayURL(fx, "titogo")+"/disable", map[string]any{}, token)
138 if status != http.StatusOK {
139 t.Fatalf("disable = %d %v, want 200", status, out)
140 }
141 if out["enabled"] != false {
142 t.Fatalf("enabled = %v after a disable, want false", out["enabled"])
143 }
144 // The disable is not COMPLETE until the region's own relay heartbeats an
145 // authority written at or after it (spec §5). The seeded heartbeat is six
146 // minutes older than the publish, so the region is still pending.
147 if pending := strList(t, out, "disable_pending"); len(pending) != 1 || pending[0] != "eu-west-1" {
148 t.Fatalf("disable_pending = %v, want the region whose relay has not caught up", pending)
149 }
150 if rec := relayRecord(t, fx, "titogo"); rec == nil || rec.Enabled {
151 t.Fatalf("record = %+v after the disable door, want it off", rec)
152 }
153}
154
155// TestAPIRelayStatusBeforeEnableIs404: "never enabled" is a state, not a
156// failure, and it is a sentence rather than an empty record — handleAPIStoreGet's
157// same choice. It speaks only because apiAccountApp has already proved
158// membership above it.
159func TestAPIRelayStatusBeforeEnableIs404(t *testing.T) {
160 fx, _ := relayFixture(t, "code-relay-unenabled")
161 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-unenabled-2")
162
163 status, out := getJSON(t, relayURL(fx, "titogo"), token)
164 if status != http.StatusNotFound {
165 t.Fatalf("status with no record = %d %v, want 404", status, out)
166 }
167 if !strings.Contains(str(t, out, "error"), "carlos relay enable") {
168 t.Fatalf("error = %q, want the sentence naming the verb that fixes it", out["error"])
169 }
170}
171
172// TestAPIRelayEnableWithoutRegionsIs503: a deployment that arms no region
173// refuses, and the refusal names CARLOS_RELAY_REGIONS. A 500 or a silent
174// success would both be worse than this: the first tells a member nothing,
175// the second would mint an identity no relay would ever accept.
176func TestAPIRelayEnableWithoutRegionsIs503(t *testing.T) {
177 fx, _ := relayFixture(t, "code-relay-noregions")
178 fx.srv.cfg.RelayRegions = nil
179 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-noregions-2")
180
181 status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token)
182 if status != http.StatusServiceUnavailable {
183 t.Fatalf("enable with no armed region = %d %v, want 503", status, out)
184 }
185 if str(t, out, "error") != errRelayNoRegions.Error() {
186 t.Fatalf("error = %q, want the sentinel verbatim", out["error"])
187 }
188 if rec := relayRecord(t, fx, "titogo"); rec != nil {
189 t.Fatalf("a refused enable wrote a record: %+v", rec)
190 }
191}
192
193// TestAPIRelayEnableNamingAnUnarmedRegionIs400: a region the caller typed
194// that this deployment does not run is the CALLER's mistake, so it is a 400
195// naming the armed ones — not the 502 an unmatched error would read as.
196func TestAPIRelayEnableNamingAnUnarmedRegionIs400(t *testing.T) {
197 fx, _ := relayFixture(t, "code-relay-badregion")
198 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-badregion-2")
199
200 status, out := postJSON(t, relayURL(fx, "titogo"),
201 map[string]any{"regions": []string{"us-east-1"}}, token)
202 if status != http.StatusBadRequest {
203 t.Fatalf("enable in an unarmed region = %d %v, want 400", status, out)
204 }
205 if msg := str(t, out, "error"); !strings.Contains(msg, "us-east-1") || !strings.Contains(msg, "eu-west-1") {
206 t.Fatalf("error = %q, want the region asked for and the ones armed", msg)
207 }
208}
209
210// TestAPIRelayAdoptedUnknownHostIs404: the ack must name a route of THIS app.
211// Without that the record would grow a cutover for anything a caller typed,
212// and status would report identities against routes that do not exist.
213func TestAPIRelayAdoptedUnknownHostIs404(t *testing.T) {
214 fx, _ := relayFixture(t, "code-relay-adopted")
215 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-adopted-2")
216 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
217 t.Fatalf("enable = %d %v, want 200", status, out)
218 }
219
220 status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted",
221 map[string]any{"host": "nowhere.example"}, token)
222 if status != http.StatusNotFound {
223 t.Fatalf("adopted for an unknown host = %d %v, want 404", status, out)
224 }
225 if msg := str(t, out, "error"); !strings.Contains(msg, "nowhere.example") {
226 t.Fatalf("error = %q, want the host named", msg)
227 }
228 // An empty host is the caller's mistake before any lookup happens.
229 if status, _ := postJSON(t, relayURL(fx, "titogo")+"/adopted", map[string]any{"host": " "}, token); status != http.StatusBadRequest {
230 t.Fatalf("adopted with no host = %d, want 400", status)
231 }
232 if rec := relayRecord(t, fx, "titogo"); rec != nil && len(rec.Cutovers) != 0 {
233 t.Fatalf("a refused ack wrote a cutover: %+v", rec.Cutovers)
234 }
235}
236
237// TestAPIRelayAdoptedRefusesARouteThePlatformRestarts: the member verb
238// exists for the ONE route shape this platform cannot restart. A unit or
239// socket-exec route is bounced by the config sync itself, so its cutover is
240// an observation — the ack that sync writes, naming what the host actually
241// rendered. A member's word here would stamp the identities the console
242// ISSUED over that observation, and a route whose restart quietly failed
243// would then read as cut over and let the old master be retired under it.
244func TestAPIRelayAdoptedRefusesARouteThePlatformRestarts(t *testing.T) {
245 fx, _ := relayFixture(t, "code-relay-adopted-shape")
246 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-adopted-shape-2")
247 fx.addRoute(t, registry.Route{Host: "s.titogo.oncarlos.com", App: "titogo",
248 Kind: "service", Socket: "/run/titogo-s.sock", Status: "ready"})
249 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
250 t.Fatalf("enable = %d %v, want 200", status, out)
251 }
252
253 status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted",
254 map[string]any{"host": "s.titogo.oncarlos.com"}, token)
255 if status != http.StatusBadRequest {
256 t.Fatalf("adopted for a socket-exec route = %d %v, want 400", status, out)
257 }
258 if msg := str(t, out, "error"); !strings.Contains(msg,
259 "the platform restarts this route itself; its adoption is observed, not declared") {
260 t.Fatalf("error = %q, want the reason in those words", msg)
261 }
262 if rec := relayRecord(t, fx, "titogo"); rec != nil && len(rec.Cutovers) != 0 {
263 t.Fatalf("a refused ack wrote a cutover: %+v", rec.Cutovers)
264 }
265}
266
267// TestAPIRelayMultiUpstreamRouteIsRefusedThenReadsAsTCPExecFile: a record
268// names its upstream EITHER as the legacy Addr or as the Addrs set, never
269// both (provision.ValidateShape), so a multi-upstream external route has an
270// empty Addr. Read unfolded, it looked like a route with no address at all —
271// which the delivery check reads as deliverable — so the enable spec §5 step
272// 2 exists to refuse went through, status called the route "unknown", and
273// `carlos relay adopted` accepted it. Both halves are pinned here: the
274// refusal names the host, and once the bundle opts in, the same route reads
275// as the tcp-exec-file it is.
276func TestAPIRelayMultiUpstreamRouteIsRefusedThenReadsAsTCPExecFile(t *testing.T) {
277 fx, _ := relayFixture(t, "code-relay-addrs")
278 ctx := context.Background()
279 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-addrs-2")
280 if err := provision.CreateRecord(ctx, fx.rel, provision.Record{
281 Schema: 1, Host: "m.titogo.oncarlos.com", Slug: "m", App: "titogo",
282 Account: fx.sqid, Kind: "service", Channel: "stable",
283 Addrs: []string{"10.0.0.5:8080", "10.0.0.6:8080"},
284 CreatedAt: time.Now().UTC(),
285 }); err != nil {
286 t.Fatalf("seed a multi-upstream record: %v", err)
287 }
288
289 status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token)
290 if status != http.StatusBadRequest {
291 t.Fatalf("enable over a multi-upstream route with no file delivery = %d %v, want 400", status, out)
292 }
293 if msg := str(t, out, "error"); !strings.Contains(msg, "m.titogo.oncarlos.com") {
294 t.Fatalf("error = %q, want the host named", msg)
295 }
296
297 if err := appconfig.SetExecDelivery(ctx, fx.rel, fx.sqid, "titogo", appconfig.ExecDeliveryFile); err != nil {
298 t.Fatal(err)
299 }
300 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
301 t.Fatalf("enable after opting into file delivery = %d %v, want 200", status, out)
302 }
303 _, out = getJSON(t, relayURL(fx, "titogo"), token)
304 wantRoute(t, relayRouteRows(t, out), "m.titogo.oncarlos.com", "tcp-exec-file",
305 "written, awaiting delivery")
306}
307
308// ---- the gate ----------------------------------------------------------------
309
310// TestAPIRelayGateIsByteIdenticalWithTheStore: every app-scoped relay door
311// refuses a stranger, an account that does not exist and an app that was
312// never claimed with ONE answer — and that answer is byte-for-byte what the
313// STORE door gives the same caller. The cross-door comparison is the point:
314// if the relay's 404 ever drifted from its neighbour's, the difference itself
315// would tell a token holder which of a deployment's apps have a relay.
316func TestAPIRelayGateIsByteIdenticalWithTheStore(t *testing.T) {
317 fx, _ := relayFixture(t, "code-relay-gate")
318 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-gate-member")
319 eve := cliToken(t, fx, "eve@keymail.dev", "code-relay-gate-eve")
320
321 // The store door's refusal for the same caller, as the yardstick.
322 _, wantBody := getRaw(t, storeURL(fx, "titogo"), eve)
323
324 doors := []struct {
325 name string
326 call func(u, bearer string) (int, string)
327 }{
328 {"GET", func(u, bearer string) (int, string) { return getRaw(t, u, bearer) }},
329 {"POST", func(u, bearer string) (int, string) { return postRaw(t, u, map[string]any{}, bearer) }},
330 {"POST disable", func(u, bearer string) (int, string) {
331 return postRaw(t, u+"/disable", map[string]any{}, bearer)
332 }},
333 {"POST adopted", func(u, bearer string) (int, string) {
334 return postRaw(t, u+"/adopted", map[string]any{"host": "x"}, bearer)
335 }},
336 }
337 for _, d := range doors {
338 t.Run(d.name, func(t *testing.T) {
339 // (a) a member of nothing, against a real account and a real app.
340 statusA, bodyA := d.call(relayURL(fx, "titogo"), eve)
341 // (b) a member, against an account that does not exist.
342 statusB, bodyB := d.call(fx.ts.URL+"/api/cli/apps/zzzz/titogo/relay", token)
343 // (c) a member of the account, against an app it never claimed.
344 statusC, bodyC := d.call(relayURL(fx, "ghost-app"), token)
345 if statusA != http.StatusNotFound || statusB != http.StatusNotFound || statusC != http.StatusNotFound {
346 t.Fatalf("%s gate = %d / %d / %d, want 404 for all three", d.name, statusA, statusB, statusC)
347 }
348 if bodyA != bodyB || bodyB != bodyC {
349 t.Fatalf("%s refusals are not byte-identical:\n(a) %q\n(b) %q\n(c) %q", d.name, bodyA, bodyB, bodyC)
350 }
351 if bodyA != wantBody {
352 t.Fatalf("%s refusal body = %q, want the store door's own refusal %q", d.name, bodyA, wantBody)
353 }
354 })
355 }
356 if rec := relayRecord(t, fx, "titogo"); rec != nil {
357 t.Fatalf("a refused request wrote a record: %+v", rec)
358 }
359 // The token gate runs first, before any name is resolved.
360 if status, _ := getJSON(t, relayURL(fx, "titogo"), ""); status != http.StatusUnauthorized {
361 t.Fatalf("status with no token = %d, want 401", status)
362 }
363 if status, _ := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, ""); status != http.StatusUnauthorized {
364 t.Fatalf("enable with no token = %d, want 401", status)
365 }
366}
367
368// ---- rotate ------------------------------------------------------------------
369
370// TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise: the one door with
371// no account segment. A member in good standing who is not on
372// CARLOS_RELAY_OPERATORS gets http.NotFound's own bytes — not a 403 naming
373// the variable, which would tell any valid token that this deployment runs a
374// relay worth rotating. Listed, the same caller rotates and is told which kid
375// the region issues from now.
376func TestAPIRelayRotateNeedsAnOperatorAndSaysNothingOtherwise(t *testing.T) {
377 fx, _ := relayFixture(t, "code-relay-rotate-door")
378 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-rotate-door-2")
379 rotate := fx.ts.URL + "/api/cli/relay/rotate"
380
381 status, body := postRaw(t, rotate, map[string]any{"region": "eu-west-1"}, token)
382 if status != http.StatusNotFound {
383 t.Fatalf("rotate by a non-operator = %d %q, want 404", status, body)
384 }
385 if body != "404 page not found\n" {
386 t.Fatalf("rotate refusal body = %q, want the stdlib http.NotFound output", body)
387 }
388 if status, _ := postRaw(t, rotate, map[string]any{"region": "eu-west-1"}, ""); status != http.StatusUnauthorized {
389 t.Fatalf("rotate with no token = %d, want 401", status)
390 }
391
392 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
393 t.Fatalf("enable = %d %v, want 200", status, out)
394 }
395 fx.srv.cfg.RelayOperators = []string{"alice@keymail.dev"}
396 // A rotation refuses until every live relay in the region has loaded the
397 // staged kid, and that refusal is the operator's to fix — a 400 naming the
398 // box, not a 502.
399 t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2))
400 seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1")
401 status, out := postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token)
402 if status != http.StatusBadRequest {
403 t.Fatalf("rotate before the box loaded the kid = %d %v, want 400", status, out)
404 }
405 if msg := str(t, out, "error"); !strings.Contains(msg, "box1") || !strings.Contains(msg, "k2") {
406 t.Fatalf("error = %q, want the box and the kid named", msg)
407 }
408
409 seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1", "k2")
410 status, out = postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token)
411 if status != http.StatusOK {
412 t.Fatalf("rotate = %d %v, want 200", status, out)
413 }
414 if out["kid"] != "k2" || out["region"] != "eu-west-1" {
415 t.Fatalf("rotate reply = %v, want the region on its new kid", out)
416 }
417 if apps, _ := out["apps"].(float64); apps != 1 {
418 t.Fatalf("apps = %v, want the one enabled app re-issued", out["apps"])
419 }
420 retire, err := time.Parse(time.RFC3339, str(t, out, "retire_no_earlier_than"))
421 if err != nil {
422 t.Fatalf("retire_no_earlier_than = %q: %v", out["retire_no_earlier_than"], err)
423 }
424 if d := time.Until(retire); d < 27*time.Hour || d > 29*time.Hour {
425 t.Fatalf("retire_no_earlier_than is %s away, want about 28h", d)
426 }
427 // An empty region is the caller's own mistake, before anything moves.
428 if status, _ := postJSON(t, rotate, map[string]any{}, token); status != http.StatusBadRequest {
429 t.Fatalf("rotate with no region = %d, want 400", status)
430 }
431}
432
433// ---- routes: what each shape is, and whose move it is ------------------------
434
435// relayRouteRows indexes a status reply's routes by host, so an assertion
436// names the route it is about rather than a position in a list.
437func relayRouteRows(t *testing.T, out map[string]any) map[string]map[string]any {
438 t.Helper()
439 raw, _ := out["routes"].([]any)
440 rows := map[string]map[string]any{}
441 for _, r := range raw {
442 row, ok := r.(map[string]any)
443 if !ok {
444 t.Fatalf("route row = %#v, want an object", r)
445 }
446 rows[str(t, row, "host")] = row
447 }
448 return rows
449}
450
451// wantRoute asserts one route row's shape and that its delivery sentence
452// contains each fragment.
453func wantRoute(t *testing.T, rows map[string]map[string]any, host, shape string, deliveryHas ...string) {
454 t.Helper()
455 row, ok := rows[host]
456 if !ok {
457 t.Fatalf("no route row for %s (have %v)", host, rows)
458 }
459 if got := str(t, row, "shape"); got != shape {
460 t.Fatalf("%s shape = %q, want %q", host, got, shape)
461 }
462 got := str(t, row, "delivery")
463 for _, want := range deliveryHas {
464 if !strings.Contains(got, want) {
465 t.Fatalf("%s delivery = %q, want it to contain %q", host, got, want)
466 }
467 }
468}
469
470// TestAPIRelayStatusNamesEachRouteShapeAndWhoseMoveItIs is the delivery half
471// of `carlos relay status`, which is the only sight a member with no box
472// access has of it. Four routes, four different answers:
473//
474// - a unit and a socket-exec are restarted by the config sync, so the key
475// arrives without anyone being asked and the row says only that it was
476// written;
477// - a TCP-address route with file delivery is the one shape this platform
478// cannot restart, so its row says whose move it is — spec §5's
479// "awaiting `carlos relay adopted`" — rather than leaving a member to
480// wonder why a rotation never completes;
481// - a route with NO config delivery, added after the enable that would have
482// refused it, is named as the fault it is. `adopted` refuses it too: a
483// cutover there would claim a process holds a key nothing ever wrote for
484// it, and a retirement would later be checked against that claim.
485//
486// Then the ack itself: recorded, it reads as adopted; after a rotation
487// re-issues the app under a new kid, the SAME ack must stop reading as
488// adopted, because it named identities that no longer exist.
489func TestAPIRelayStatusNamesEachRouteShapeAndWhoseMoveItIs(t *testing.T) {
490 fx, _ := relayFixture(t, "code-relay-shapes")
491 ctx := context.Background()
492 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-shapes-2")
493
494 // One named bundle opts into file delivery; the app's default does not.
495 // That is what lets two otherwise identical TCP routes differ, and it is
496 // the per-route precedence relayExecDelivery applies (#195).
497 if err := appconfig.SetExecDeliveryAt(ctx, fx.rel, fx.sqid, "titogo", "prod", appconfig.ExecDeliveryFile); err != nil {
498 t.Fatal(err)
499 }
500 fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo",
501 Kind: "service", Unit: "titogo-a.service", Status: "ready"})
502 fx.addRoute(t, registry.Route{Host: "b.titogo.oncarlos.com", App: "titogo",
503 Kind: "service", Socket: "/run/titogo-b.sock", Status: "ready"})
504 fx.addRoute(t, registry.Route{Host: "c.titogo.oncarlos.com", App: "titogo",
505 Kind: "service", Addr: "10.0.0.5:8080", Environment: "prod", Status: "ready"})
506
507 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
508 t.Fatalf("enable = %d %v, want 200", status, out)
509 }
510 // Added AFTER the enable, which is the only way it can exist: enable
511 // itself refuses an app that already has one.
512 fx.addRoute(t, registry.Route{Host: "d.titogo.oncarlos.com", App: "titogo",
513 Kind: "service", Addr: "10.0.0.6:8080", Status: "ready"})
514
515 status, out := getJSON(t, relayURL(fx, "titogo"), token)
516 if status != http.StatusOK {
517 t.Fatalf("status = %d %v, want 200", status, out)
518 }
519 rows := relayRouteRows(t, out)
520 if len(rows) != 4 {
521 t.Fatalf("routes = %v, want four rows", rows)
522 }
523 wantRoute(t, rows, "a.titogo.oncarlos.com", "unit", "written, awaiting delivery")
524 wantRoute(t, rows, "b.titogo.oncarlos.com", "socket-exec", "written, awaiting delivery")
525 // No box has synced yet, so no config-ack exists — c reads exactly like
526 // a and b until one does (Task 12: a tcp-exec-file route's "file
527 // delivered, awaiting adopted" sentence is earned by an ack, not
528 // assumed from its shape alone).
529 wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "written, awaiting delivery")
530 wantRoute(t, rows, "d.titogo.oncarlos.com", "no-delivery",
531 "no config delivery", "exec_delivery file")
532
533 // Now a box's config sync delivers the file and writes its ack
534 // (appconfig.WriteConfigAck, seeded here exactly as syncExecHost would)
535 // — only then does c's row say whose move it is.
536 issued := relayRecord(t, fx, "titogo").IssuedSuffixes
537 if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{
538 Box: "box1", Host: "c.titogo.oncarlos.com", Shape: "tcp-exec-file", At: time.Now(),
539 RelaySuffixes: maps.Clone(issued),
540 }); err != nil {
541 t.Fatalf("seed ack c: %v", err)
542 }
543 _, out = getJSON(t, relayURL(fx, "titogo"), token)
544 rows = relayRouteRows(t, out)
545 wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file",
546 "carlos relay adopted --host c.titogo.oncarlos.com")
547
548 // The undeliverable route cannot be acknowledged away.
549 status, out = postJSON(t, relayURL(fx, "titogo")+"/adopted",
550 map[string]any{"host": "d.titogo.oncarlos.com"}, token)
551 if status != http.StatusBadRequest {
552 t.Fatalf("adopted for a no-delivery route = %d %v, want 400", status, out)
553 }
554 if msg := str(t, out, "error"); !strings.Contains(msg, "no config delivery") {
555 t.Fatalf("error = %q, want the reason named", msg)
556 }
557
558 // The deliverable one can, and then reads as adopted.
559 if status, out := postJSON(t, relayURL(fx, "titogo")+"/adopted",
560 map[string]any{"host": "c.titogo.oncarlos.com"}, token); status != http.StatusOK {
561 t.Fatalf("adopted = %d %v, want 200", status, out)
562 }
563 _, out = getJSON(t, relayURL(fx, "titogo"), token)
564 rows = relayRouteRows(t, out)
565 wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file", "adopted by member at")
566 if rows["c.titogo.oncarlos.com"]["cutover"] != true {
567 t.Fatalf("cutover = %v after an ack, want true", rows["c.titogo.oncarlos.com"]["cutover"])
568 }
569
570 // A rotation re-issues the app under a new kid. The ack named the OLD
571 // identities, so it must stop reading as adopted — otherwise a member
572 // would believe a route runs a key it has never seen, and the old master
573 // would be retired out from under it.
574 t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2))
575 seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1", "k2")
576 if _, err := fx.srv.relayRotate(ctx, "eu-west-1"); err != nil {
577 t.Fatalf("rotate: %v", err)
578 }
579 _, out = getJSON(t, relayURL(fx, "titogo"), token)
580 rows = relayRouteRows(t, out)
581 wantRoute(t, rows, "c.titogo.oncarlos.com", "tcp-exec-file",
582 "the identities have changed since", "carlos relay adopted --host c.titogo.oncarlos.com")
583 if rows["c.titogo.oncarlos.com"]["cutover"] != false {
584 t.Fatalf("cutover = %v after a rotation, want false", rows["c.titogo.oncarlos.com"]["cutover"])
585 }
586 if rows["c.titogo.oncarlos.com"]["acked_at"] == nil {
587 t.Fatal("acked_at was dropped — the ack still happened, it just no longer counts")
588 }
589}
590
591// TestAPIRelayRotateStagingRefusalsAreClientErrors pins the two refusal
592// sentences relayLooksLikeClientError matches by wording rather than by
593// sentinel: nothing new staged, and no live relay in the region at all.
594// Both are the OPERATOR's to fix by finishing the runbook, so both are a 400
595// naming what is wrong — and if either sentence is ever reworded without
596// updating the matcher, they become a 502 that reads as "the console broke"
597// and this test is what says so.
598func TestAPIRelayRotateStagingRefusalsAreClientErrors(t *testing.T) {
599 fx, _ := relayFixture(t, "code-relay-staging")
600 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-staging-2")
601 fx.srv.cfg.RelayOperators = []string{"alice@keymail.dev"}
602 rotate := fx.ts.URL + "/api/cli/relay/rotate"
603
604 // (a) nothing new staged: the only entry is already the issuer.
605 seedHeartbeat(t, fx, "box1", "eu-west-1", time.Minute, "k1")
606 status, out := postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token)
607 if status != http.StatusBadRequest {
608 t.Fatalf("rotate with nothing new staged = %d %v, want 400", status, out)
609 }
610 if msg := str(t, out, "error"); !strings.Contains(msg, "no new master is staged") {
611 t.Fatalf("error = %q, want the nothing-staged sentence", msg)
612 }
613
614 // (b) a new master staged, but every heartbeat in the region is stale.
615 // Issuing now would mint credentials nothing can verify, and a dead relay
616 // must not be able to consent by silence.
617 t.Setenv(relayMastersEnv("eu-west-1"), "k1:"+relayTestMaster(1)+",k2:"+relayTestMaster(2))
618 seedHeartbeat(t, fx, "box1", "eu-west-1", 6*time.Minute, "k1", "k2")
619 status, out = postJSON(t, rotate, map[string]any{"region": "eu-west-1"}, token)
620 if status != http.StatusBadRequest {
621 t.Fatalf("rotate with no live relay = %d %v, want 400", status, out)
622 }
623 if msg := str(t, out, "error"); !strings.Contains(msg, "has heartbeated in the last") {
624 t.Fatalf("error = %q, want the no-live-relay sentence", msg)
625 }
626}
627
628// TestAPIRelayStatusReadsConfigAcks is Task 12's console half: three
629// unit-backed routes, one config-ack each, seeded exactly as config sync
630// itself writes them (control/config-ack/<box>/<host>.json,
631// appconfig.WriteConfigAck) rather than through any sync machinery — the
632// door's whole job here is reading and matching what's already in the
633// bucket, not producing it. An ack naming the issued identities and a
634// successful restart reads as adopted; one naming them with a restart
635// error reads as failed; one naming identities from BEFORE this app's
636// enable (a stale ack, the shape a rotation or a re-enable would leave
637// behind) must not read as adopted at all — see relayMatchingAck's own
638// doc comment for why.
639func TestAPIRelayStatusReadsConfigAcks(t *testing.T) {
640 fx, _ := relayFixture(t, "code-relay-config-acks")
641 ctx := context.Background()
642 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-config-acks-2")
643
644 fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo",
645 Kind: "service", Unit: "titogo-a.service", Status: "ready"})
646 fx.addRoute(t, registry.Route{Host: "b.titogo.oncarlos.com", App: "titogo",
647 Kind: "service", Unit: "titogo-b.service", Status: "ready"})
648 fx.addRoute(t, registry.Route{Host: "c.titogo.oncarlos.com", App: "titogo",
649 Kind: "service", Unit: "titogo-c.service", Status: "ready"})
650 fx.addRoute(t, registry.Route{Host: "d.titogo.oncarlos.com", App: "titogo",
651 Kind: "service", Unit: "titogo-d.service", Status: "ready"})
652
653 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
654 t.Fatalf("enable = %d %v, want 200", status, out)
655 }
656 issued := relayRecord(t, fx, "titogo").IssuedSuffixes
657 if issued["eu-west-1"] == "" {
658 t.Fatalf("enable issued nothing for eu-west-1: %v", issued)
659 }
660
661 now := time.Now()
662 if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{
663 Box: "box1", Host: "a.titogo.oncarlos.com", Shape: "unit", At: now,
664 Restarted: true, RelaySuffixes: maps.Clone(issued),
665 }); err != nil {
666 t.Fatalf("seed ack a: %v", err)
667 }
668 if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{
669 Box: "box1", Host: "b.titogo.oncarlos.com", Shape: "unit", At: now,
670 RestartError: "systemctl try-restart carlos-app@b.titogo.oncarlos.com: exit status 1",
671 RelaySuffixes: maps.Clone(issued),
672 }); err != nil {
673 t.Fatalf("seed ack b: %v", err)
674 }
675 if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{
676 Box: "box1", Host: "c.titogo.oncarlos.com", Shape: "unit", At: now,
677 Restarted: true,
678 RelaySuffixes: map[string]string{"eu-west-1": "abc:titogo:stale0000000:k0"},
679 }); err != nil {
680 t.Fatalf("seed ack c: %v", err)
681 }
682 if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{
683 Box: "box1", Host: "d.titogo.oncarlos.com", Shape: "unit", At: now,
684 Hibernating: true, RelaySuffixes: maps.Clone(issued),
685 }); err != nil {
686 t.Fatalf("seed ack d: %v", err)
687 }
688
689 _, out := getJSON(t, relayURL(fx, "titogo"), token)
690 rows := relayRouteRows(t, out)
691 wantRoute(t, rows, "a.titogo.oncarlos.com", "unit", "adopted at", "restarted")
692 wantRoute(t, rows, "b.titogo.oncarlos.com", "unit", "restart failed",
693 "systemctl try-restart carlos-app@b.titogo.oncarlos.com: exit status 1")
694 if got := str(t, rows["c.titogo.oncarlos.com"], "delivery"); strings.Contains(got, "adopted") {
695 t.Fatalf("c delivery = %q, a stale-suffix ack must not read as adopted", got)
696 }
697 if got := str(t, rows["c.titogo.oncarlos.com"], "delivery"); got != "written, awaiting delivery" {
698 t.Fatalf("c delivery = %q, want the base written-awaiting-delivery sentence", got)
699 }
700 wantRoute(t, rows, "d.titogo.oncarlos.com", "unit", "hibernating, delivered on wake")
701}
702
703// TestAPIRelayStatusNeverAdoptsOnEmptyIssuedSuffixes is the review's finding
704// 2 (2026-09-07): maps.Equal(nil, nil) is true, so without an explicit
705// guard an enabled record still at phase "config" — IssuedSuffixes not
706// written yet, the state an enable interrupted between spec §5 steps 4 and
707// 5 persists in until the reconcile finishes it — would match ANY route's
708// ordinary pre-relay ack (no CARLOS_RELAY entry at all, RelaySuffixes nil)
709// and print an adoption for identities the console has never actually
710// issued.
711func TestAPIRelayStatusNeverAdoptsOnEmptyIssuedSuffixes(t *testing.T) {
712 fx, _ := relayFixture(t, "code-relay-empty-issued")
713 ctx := context.Background()
714 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-empty-issued-2")
715
716 fx.addRoute(t, registry.Route{Host: "a.titogo.oncarlos.com", App: "titogo",
717 Kind: "service", Unit: "titogo-a.service", Status: "ready"})
718
719 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
720 t.Fatalf("enable = %d %v, want 200", status, out)
721 }
722
723 // Roll the finished enable back to the interrupted-mid-walk shape by
724 // hand: still enabled, phase "config", nothing issued yet.
725 rec, etag, err := relay.GetRecord(ctx, fx.rel, fx.sqid, "titogo")
726 if err != nil {
727 t.Fatalf("GetRecord: %v", err)
728 }
729 rec.Phase = "config"
730 rec.IssuedSuffixes = nil
731 if _, err := relay.PutRecord(ctx, fx.rel, fx.sqid, "titogo", rec, etag); err != nil {
732 t.Fatalf("PutRecord: %v", err)
733 }
734
735 // An ordinary ack — the shape a route synced before this app ever
736 // enabled the relay carries: RelaySuffixes nil, no error naming
737 // anything wrong with it.
738 if err := appconfig.WriteConfigAck(ctx, fx.rel, appconfig.ConfigAck{
739 Box: "box1", Host: "a.titogo.oncarlos.com", Shape: "unit", At: time.Now(),
740 Restarted: true,
741 }); err != nil {
742 t.Fatalf("seed ack: %v", err)
743 }
744
745 _, out := getJSON(t, relayURL(fx, "titogo"), token)
746 rows := relayRouteRows(t, out)
747 if got := str(t, rows["a.titogo.oncarlos.com"], "delivery"); strings.Contains(got, "adopted") {
748 t.Fatalf("delivery = %q, an empty-IssuedSuffixes record must never read as adopted", got)
749 }
750}
751
752// TestAPIRelayStatusCarriesRetiringMasters covers the hop from a relay's
753// heartbeat to the status reply (discussion #120). Without it the console
754// half of the retiring display can be deleted with the whole suite green,
755// and an operator mid-rotation silently reads "nothing is retiring" — which
756// is indistinguishable from the true answer and is the wrong one to act on.
757func TestAPIRelayStatusCarriesRetiringMasters(t *testing.T) {
758 fx, _ := relayFixture(t, "code-relay-retiring")
759 token := cliToken(t, fx, "alice@keymail.dev", "code-relay-retiring-2")
760 if status, out := postJSON(t, relayURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
761 t.Fatalf("enable = %d %v, want 200", status, out)
762 }
763
764 retire := time.Date(2026, 9, 9, 12, 0, 0, 0, time.UTC)
765 seedHeartbeatRetiring(t, fx, "box1", "eu-west-1", time.Minute,
766 map[string]time.Time{"k1": retire}, "k1", "k2")
767
768 status, out := getJSON(t, relayURL(fx, "titogo"), token)
769 if status != http.StatusOK {
770 t.Fatalf("status = %d %v, want 200", status, out)
771 }
772 row := regionRow(t, out, 0)
773 got, ok := row["retiring"].(map[string]any)
774 if !ok {
775 t.Fatalf("region row carries no retiring map: %#v", row)
776 }
777 if got["k1"] != retire.Format(time.RFC3339) {
778 t.Errorf("retiring[k1] = %v, want %s", got["k1"], retire.Format(time.RFC3339))
779 }
780 // k2 is loaded but carries no retirement date, so it must be absent
781 // rather than present with a zero time.
782 if _, ok := got["k2"]; ok {
783 t.Errorf("k2 has no retirement date and must not appear: %#v", got)
784 }
785}
786