carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1// relay.go is `carlos relay`: the platform STUN/TURN relay's own verbs
2// (relay spec §3-§5). Three audiences, one command:
3//
4// enable a MEMBER turning the relay on for their app, and having
5// adopted CARLOS_RELAY delivered to its config. status is the only
6// status sight they have of delivery per route and health per region
7// disable without box access; adopted is their cutover ack for a route
8// this platform cannot restart for them.
9// rotate an OPERATOR moving a region onto a newly staged master.
10// keygen an OPERATOR minting one. Its output is a secret.
11// serve what carlos-relay.service runs. Not typed by hand.
12//
13// The five member/operator verbs are API mode only, on `carlos store`'s
14// model (store.go): the record lives in the console's deployment bucket, and
15// the key is derived from a master only the console holds and sealed with a
16// key only the app's boxes can open — there is no bucket-direct equivalent of
17// any of that. See requireRelayAPISession.
18//
19// The output-honesty rule is restart.go's, and it bites hardest here: enable
20// prints what was ISSUED and written, never that a route has picked it up.
21// Only status can say that, and even status says only what the console can
22// observe — a route's own acknowledgement, not a guess from elapsed time.
23package main
24
25import (
26 "context"
27 "crypto/rand"
28 "encoding/base64"
29 "encoding/hex"
30 "flag"
31 "fmt"
32 "io"
33 "log"
34 "os"
35 "os/signal"
36 "sort"
37 "strings"
38 "syscall"
39
40 "github.com/carlosframework/platform/internal/host"
41 "github.com/carlosframework/platform/internal/relay"
42)
43
44const relayUsage = `usage: carlos relay <enable|status|disable|adopted|rotate|keygen|serve> [flags]
45
46 enable --app <a> [--regions <r1,r2>]
47 turn the relay on for one app and deliver CARLOS_RELAY to its
48 config. Regions default to every one this deployment arms, in the
49 console's order — the order every browser tries them in.
50 status --app <a>
51 what was issued per region, what each route has done about it,
52 and each region relay's heartbeat, lease and loaded masters. The
53 only sight of any of that without box access.
54 disable --app <a>
55 remove the key. Every allocation the app holds ends on the
56 relays' next authority read, within a minute.
57 adopted --app <a> --host <route>
58 tell the platform that a TCP-address route you restarted by hand
59 now runs the issued key. The one route shape this platform cannot
60 restart for you, so the acknowledgement has to come from you.
61 rotate --region <r>
62 operator: move the region's issuer pointer onto the newly staged
63 master and re-issue every app enabled there. Needs an address on
64 CARLOS_RELAY_OPERATORS; anyone else meets the same 404 as an
65 unknown console door, never a 403.
66 keygen mint a region master: one "<kid> <base64 key> -" line for
67 /etc/carlos/relay-masters and the same pair for the console's
68 CARLOS_RELAY_MASTERS_<REGION> secret, which holds every master the
69 console may need as <kid>:<key>[,<kid>:<key>].
70 serve run the relay (what carlos-relay.service execs; not typed by hand)
71
72 --account <sqid|name> account the app belongs to (default: your only
73 account, or the configured/chosen one)
74 --account-id <sqid> account sqid, matched as a sqid ONLY.
75 --console <origin> which logged-in console to use (only needed with
76 several)
77
78enable, status, disable, adopted and rotate are signed-in only
79(carlos auth login): the relay's keys are derived from a master the console
80holds and sealed with the deployment's secrets key, so there is no
81direct-bucket equivalent.`
82
83func runRelay(args []string) {
84 if len(args) == 0 {
85 fmt.Fprintln(os.Stderr, relayUsage)
86 os.Exit(2)
87 }
88 switch args[0] {
89 case "enable":
90 runRelayEnable(args[1:])
91 case "status":
92 runRelayStatus(args[1:])
93 case "disable":
94 runRelayDisable(args[1:])
95 case "adopted":
96 runRelayAdopted(args[1:])
97 case "rotate":
98 runRelayRotate(args[1:])
99 case "serve":
100 runRelayServe(args[1:])
101 case "keygen":
102 runRelayKeygen(args[1:])
103 case "-h", "--help", "help":
104 fmt.Println(relayUsage)
105 default:
106 fmt.Fprintln(os.Stderr, relayUsage)
107 os.Exit(2)
108 }
109}
110
111// runRelayServe is the unit's entry point. SIGTERM is systemd's stop, and
112// it must reach relay.Serve rather than killing the process, so the relay
113// closes its allocations and flushes its last minute of metering — the
114// bytes it forwarded in that minute are billable, and a hard kill loses
115// them.
116func runRelayServe(_ []string) {
117 ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
118 defer stop()
119
120 // A store that will not open is waited out, not died on: the unit is
121 // Restart=always and systemd's start limit would park it in `failed`
122 // within ten seconds, taking STUN — which needs no store — down with
123 // it. Configuration errors below are fatal, because waiting fixes
124 // nothing about a missing public IP or a malformed CIDR.
125 store, err := relay.OpenStoreWithRetry(ctx, host.OpenDeployStore, log.Printf)
126 if err != nil {
127 return // ctx cancelled: systemd asked us to stop before we ever started
128 }
129 cfg, err := relay.ConfigFromEnv(ctx, os.Getenv, host.ReadPublicIP, host.ReadPlacementRegion, store)
130 if err != nil {
131 log.Fatalf("relay serve: %v", err)
132 }
133 // The box label is the metrics loop's, so the relay's heartbeat and its
134 // hourly metering objects land under the same name every other row this
135 // box publishes uses. It lives in internal/host, which internal/relay
136 // does not import, so it is filled in here.
137 if cfg.Box == "" {
138 cfg.Box = host.BoxLabel()
139 }
140 if err := relay.Serve(ctx, cfg, log.Printf); err != nil {
141 log.Fatalf("relay serve: %v", err)
142 }
143}
144
145// runRelayKeygen prints one masters-file line: a short key id, the key, and
146// "-" for "no retirement date". The key is printed once and never stored by
147// this process — the operator stages it on the region's boxes and puts the
148// same pair in the console's secret (spec §4).
149func runRelayKeygen(_ []string) {
150 key := make([]byte, relay.MasterKeyLen)
151 if _, err := rand.Read(key); err != nil {
152 log.Fatalf("relay keygen: %v", err)
153 }
154 kid := make([]byte, 4)
155 if _, err := rand.Read(kid); err != nil {
156 log.Fatalf("relay keygen: %v", err)
157 }
158 fmt.Printf("%s %s -\n", hex.EncodeToString(kid), base64.StdEncoding.EncodeToString(key))
159}
160
161// ---- the app-facing family (spec §5) ----------------------------------------
162
163// requireRelayAPISession resolves signed-in mode for every relay verb below,
164// refusing direct-bucket mode with relay's OWN message — requireStoreAPISession's
165// pattern (store.go), and for the same reason: a refusal that names another
166// verb's advice sends someone to a fix that cannot help them.
167//
168// There is no headless fallback, and the reason is stronger than the store's.
169// Enabling a relay does not merely write an object: it derives an app key from
170// a region master held in the CONSOLE's own environment and seals it with the
171// deployment's secrets key. A shell in bucket mode has neither, so there is
172// nothing here it could drive on its own — and a key minted anywhere else
173// would be one no relay in the region would accept.
174func requireRelayAPISession(consoleFlag string) *apiSession {
175 sess, err := resolveMode(consoleFlag)
176 if err != nil {
177 log.Fatal(err)
178 }
179 if sess == nil {
180 log.Fatal("carlos relay works through a console, and this shell is in direct bucket mode\n" +
181 "(CARLOS_DEPLOYMENT_BUCKET/CARLOS_DEPLOYMENT_DIR is set). The relay's keys are derived from a\n" +
182 "region master the CONSOLE holds and sealed with the deployment's secrets key, neither of which\n" +
183 "exists in this shell — there is no direct-bucket equivalent. Unset the store env and run:\n" +
184 "carlos auth login --console <url>.")
185 }
186 return sess
187}
188
189func runRelayEnable(args []string) {
190 fs := flag.NewFlagSet("relay enable", flag.ExitOnError)
191 app := appFlag(fs, "app to enable the relay for")
192 regions := fs.String("regions", "", "comma-separated regions to enable (default: every region this deployment arms)")
193 acctRaw, acctIDRaw, _ := accountFlags(fs)
194 consoleFlag := consoleOption(fs, "console origin to enable through (API mode only; default: the only logged-in one)")
195 fs.Usage = flagUsage(fs, "")
196 fs.Parse(args)
197 name := strings.TrimSpace(*app)
198 usage := "usage: carlos relay enable --app <a> [--regions <r1,r2>] [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]"
199 if name != "" {
200 if err := validAppName(name); err != nil {
201 log.Fatalf("relay enable: %v", err)
202 }
203 }
204
205 sess := requireRelayAPISession(*consoleFlag)
206 printUsing(stdout, sess)
207 acct := resolvedAccount(sess, acctRaw, acctIDRaw)
208 name = resolvedApp(sess, acct, name, usage)
209 res, err := sess.relayEnable(context.Background(), acct, name, splitList(*regions))
210 if err != nil {
211 log.Fatalf("relay enable: %v", err)
212 }
213 printRelayStatus(stdout, res)
214 // The output-honesty rule this whole CLI follows (restart.go's): the
215 // console's own walk finished before this process got its 200, so the
216 // record and the aggregate ARE written — but a route picking the key up
217 // is a converge a box does on its own schedule, and only `status` can
218 // say whether it has.
219 mutedf(stdout, "delivery per route is reported by: carlos relay status --app %s", name)
220}
221
222func runRelayStatus(args []string) {
223 fs := flag.NewFlagSet("relay status", flag.ExitOnError)
224 app := appFlag(fs, "app whose relay to show")
225 acctRaw, acctIDRaw, _ := accountFlags(fs)
226 consoleFlag := consoleOption(fs, "console origin to read through (API mode only; default: the only logged-in one)")
227 fs.Usage = flagUsage(fs, "")
228 fs.Parse(args)
229 name := strings.TrimSpace(*app)
230 usage := "usage: carlos relay status --app <a> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]"
231 if name != "" {
232 if err := validAppName(name); err != nil {
233 log.Fatalf("relay status: %v", err)
234 }
235 }
236
237 sess := requireRelayAPISession(*consoleFlag)
238 printUsing(stdout, sess)
239 acct := resolvedAccount(sess, acctRaw, acctIDRaw)
240 name = resolvedApp(sess, acct, name, usage)
241 res, err := sess.relayGet(context.Background(), acct, name)
242 if err != nil {
243 log.Fatalf("relay status: %v", err)
244 }
245 printRelayStatus(stdout, res)
246}
247
248func runRelayDisable(args []string) {
249 fs := flag.NewFlagSet("relay disable", flag.ExitOnError)
250 app := appFlag(fs, "app to disable the relay for")
251 acctRaw, acctIDRaw, _ := accountFlags(fs)
252 consoleFlag := consoleOption(fs, "console origin to disable through (API mode only; default: the only logged-in one)")
253 fs.Usage = flagUsage(fs, "")
254 fs.Parse(args)
255 name := strings.TrimSpace(*app)
256 usage := "usage: carlos relay disable --app <a> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]"
257 if name != "" {
258 if err := validAppName(name); err != nil {
259 log.Fatalf("relay disable: %v", err)
260 }
261 }
262
263 sess := requireRelayAPISession(*consoleFlag)
264 printUsing(stdout, sess)
265 acct := resolvedAccount(sess, acctRaw, acctIDRaw)
266 name = resolvedApp(sess, acct, name, usage)
267 res, err := sess.relayDisable(context.Background(), acct, name)
268 if err != nil {
269 log.Fatalf("relay disable: %v", err)
270 }
271 printRelayStatus(stdout, res)
272}
273
274func runRelayAdopted(args []string) {
275 fs := flag.NewFlagSet("relay adopted", flag.ExitOnError)
276 app := appFlag(fs, "app the route belongs to")
277 host := fs.String("host", "", "the route that now runs the issued relay key")
278 acctRaw, acctIDRaw, _ := accountFlags(fs)
279 consoleFlag := consoleOption(fs, "console origin to acknowledge through (API mode only; default: the only logged-in one)")
280 fs.Usage = flagUsage(fs, "")
281 fs.Parse(args)
282 name := strings.TrimSpace(*app)
283 usage := "usage: carlos relay adopted --app <a> --host <route> [--account <sqid-or-name>] [--account-id <sqid>] [--console <origin>]"
284 if name != "" {
285 if err := validAppName(name); err != nil {
286 log.Fatalf("relay adopted: %v", err)
287 }
288 }
289 if strings.TrimSpace(*host) == "" {
290 log.Fatalf("relay adopted: --host is required\n%s", usage)
291 }
292
293 sess := requireRelayAPISession(*consoleFlag)
294 printUsing(stdout, sess)
295 acct := resolvedAccount(sess, acctRaw, acctIDRaw)
296 name = resolvedApp(sess, acct, name, usage)
297 res, err := sess.relayAdopted(context.Background(), acct, name, strings.TrimSpace(*host))
298 if err != nil {
299 log.Fatalf("relay adopted: %v", err)
300 }
301 printRelayStatus(stdout, res)
302}
303
304// runRelayRotate is the operator's verb and the one here with no --app: it
305// moves a REGION's issuer pointer and re-issues every app enabled there. A
306// caller who is not on CARLOS_RELAY_OPERATORS meets the console's byte-
307// identical 404, so the error printed here reads as "no such thing" rather
308// than "not allowed" — deliberately, since a 403 would confirm to any token
309// holder that this deployment runs a relay at all.
310func runRelayRotate(args []string) {
311 fs := flag.NewFlagSet("relay rotate", flag.ExitOnError)
312 region := fs.String("region", "", "region whose issuer pointer to move onto the newly staged master")
313 consoleFlag := consoleOption(fs, "console origin to rotate through (API mode only; default: the only logged-in one)")
314 fs.Usage = flagUsage(fs, "")
315 fs.Parse(args)
316 if strings.TrimSpace(*region) == "" {
317 log.Fatal("relay rotate: --region is required\nusage: carlos relay rotate --region <r> [--console <origin>]")
318 }
319
320 sess := requireRelayAPISession(*consoleFlag)
321 printUsing(stdout, sess)
322 res, err := sess.relayRotate(context.Background(), strings.TrimSpace(*region))
323 if err != nil {
324 log.Fatalf("relay rotate: %v", err)
325 }
326 printRelayRotate(stdout, res)
327}
328
329// splitList turns a comma-separated flag into a list, dropping empties so a
330// trailing comma is not a region named "".
331func splitList(s string) []string {
332 var out []string
333 for _, p := range strings.Split(s, ",") {
334 if p = strings.TrimSpace(p); p != "" {
335 out = append(out, p)
336 }
337 }
338 return out
339}
340
341// printRelayStatus prints one relay record: what the member ASKED for, what
342// the console ISSUED, what each route has done about it, and how each region's
343// own relay looks. The four are kept apart on purpose — a member debugging a
344// failing call needs to know which of them is the one that has not happened,
345// and a single "relay: ok" line would hide exactly that.
346func printRelayStatus(w io.Writer, res apiRelayReply) {
347 c := colorFor(w)
348 fmt.Fprintf(w, "%s %s\n", c.head("relay for"), res.App)
349 switch {
350 case res.Enabled:
351 fmt.Fprintf(w, " enabled in %s, incarnation %s (op %d, %s)\n",
352 strings.Join(res.Regions, ", "), res.Incarnation, res.Op, res.Phase)
353 case res.Incarnation != "":
354 // A record that HAS been enabled and is now off still carries an
355 // operation and a phase, and both matter: a disable that stopped at
356 // phase "config" is one the reconcile has yet to finish, and hiding
357 // that behind "not enabled" would make a half-done disable look
358 // complete.
359 fmt.Fprintf(w, " disabled (op %d, %s)\n", res.Op, res.Phase)
360 default:
361 mutedf(w, "not enabled — carlos relay enable --app %s", res.App)
362 }
363 // The issued identity per region: account, app, incarnation and kid, the
364 // public half of the credential the app mints per call. Printed so a
365 // member can match a rejected call against the kid a region issues from.
366 for _, region := range res.Regions {
367 if suffix := res.Issued[region]; suffix != "" {
368 fmt.Fprintf(w, " issued %s: %s\n", region, suffix)
369 }
370 }
371 for _, rt := range res.Routes {
372 line := fmt.Sprintf(" route %s (%s): %s", rt.Host, rt.Shape, rt.Delivery)
373 // A route the relay key will never reach is the one row here that is
374 // a fault rather than a state: it blocks the next master retirement
375 // until someone fixes it, and only the member can.
376 if rt.Shape == "no-delivery" {
377 warnf(w, "%s", line)
378 continue
379 }
380 fmt.Fprintln(w, line)
381 }
382 if len(res.Routes) == 0 && res.Enabled {
383 mutedf(w, " no routes yet — the key is written and waits for the first one")
384 }
385 // Disable progress before region health, spec §5's own order: when a
386 // member has just disabled, "is it off everywhere yet" is the question
387 // they typed the command to answer.
388 if len(res.DisablePending) > 0 {
389 warnf(w, " disable pending on %s — those relays have not read the record without this app yet; "+
390 "allocations there end on their next authority read, within a minute",
391 strings.Join(res.DisablePending, ", "))
392 } else if !res.Enabled && res.Incarnation != "" {
393 fmt.Fprintln(w, " revoked: every relay has read a record without this app")
394 }
395 for _, rg := range res.RegionStatus {
396 line := fmt.Sprintf(" relay %s in %s: heartbeat %s", rg.Box, rg.Region, rg.HeartbeatAge)
397 if !rg.Current {
398 line += ", holding an older record"
399 }
400 if len(rg.Listeners) > 0 {
401 line += ", listening on " + strings.Join(rg.Listeners, " ")
402 }
403 if len(rg.Kids) > 0 {
404 line += ", masters " + strings.Join(rg.Kids, ",")
405 }
406 // A retiring master is the operator's cue, not the member's: it is
407 // the only thing that says when the old key may leave this box's
408 // /etc/carlos/relay-masters. Printed per box, because the region is
409 // done with a key only once every relay there has stopped loading
410 // it (discussion #120).
411 if len(rg.Retiring) > 0 {
412 kids := make([]string, 0, len(rg.Retiring))
413 for kid := range rg.Retiring {
414 kids = append(kids, kid)
415 }
416 sort.Strings(kids)
417 parts := make([]string, 0, len(kids))
418 for _, kid := range kids {
419 parts = append(parts, kid+" after "+rg.Retiring[kid])
420 }
421 line += ", retiring " + strings.Join(parts, ",")
422 }
423 switch {
424 case rg.Stale:
425 // STALE is the important word and it gets the warning colour: a
426 // relay that stopped reading the authority keeps serving until its
427 // lease runs out, so "up" and "current" are different facts. The
428 // relay's own last error, when it reported one, is the whole
429 // difference between a member filing a bug and an operator
430 // fixing a masters file.
431 warnf(w, "%s — STALE (no successful read in over five minutes)%s", line, relayWhyStale(rg))
432 case !rg.LeaseLive:
433 warnf(w, "%s — lease expired, this relay is no longer serving", line)
434 default:
435 fmt.Fprintln(w, line+", lease live")
436 }
437 }
438 if res.Enabled && len(res.RegionStatus) == 0 {
439 warnf(w, " no relay in %s has heartbeated at all — calls will not connect through TURN",
440 strings.Join(res.Regions, ", "))
441 }
442}
443
444// relayWhyStale is the reason half of a stale row: the relay's own last
445// masters-file error, its last authority-read error, or nothing when it
446// reported neither (a relay that simply stopped writing heartbeats has no
447// error to tell us about, and inventing one would be worse than the silence).
448func relayWhyStale(rg apiRelayRegion) string {
449 switch {
450 case rg.MastersError != "" && rg.AuthorityError != "":
451 return ": masters " + rg.MastersError + "; authority " + rg.AuthorityError
452 case rg.MastersError != "":
453 return ": masters " + rg.MastersError
454 case rg.AuthorityError != "":
455 return ": authority " + rg.AuthorityError
456 }
457 return ""
458}
459
460// printRelayRotate is the operator's answer: which master the region issues
461// from now, how many apps were re-issued under it, and — the line the runbook
462// turns on — the earliest the OLD master may be taken out of the masters file.
463// Retiring it before then kills credentials that are still inside their own
464// four-hour lifetime (spec §5, step 4).
465func printRelayRotate(w io.Writer, res apiRelayRotateReply) {
466 c := colorFor(w)
467 fmt.Fprintf(w, "%s %s\n", c.head("rotated"), res.Region)
468 fmt.Fprintf(w, " issuing from kid %s; %d app(s) re-issued\n", res.Kid, res.Apps)
469 fmt.Fprintf(w, " retire the previous master no earlier than %s\n", res.RetireNoEarlierThan)
470 mutedf(w, "leave the old line in /etc/carlos/relay-masters and the old pair in the console secret until then")
471}
472