carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package main
2
3import (
4 "context"
5 "crypto/sha256"
6 "encoding/hex"
7 "errors"
8 "flag"
9 "fmt"
10 "io"
11 "log"
12 "net/http"
13 "net/url"
14 "os"
15 "os/signal"
16 "slices"
17 "strconv"
18 "strings"
19 "syscall"
20 "time"
21)
22
23const (
24 // importCheckWait bounds the wait for "uploaded" after the 202: the
25 // console's integrity check (30 minutes) and its put (20), and a CAS.
26 importCheckWait = 55 * time.Minute
27 // importSwapWait and importPlaceWait are spec §5's waits for the box.
28 importSwapWait = 10 * time.Minute
29 importPlaceWait = 5 * time.Minute
30 // importVerifyTries is how many times a verify answered 502, 503 or 504
31 // (the gate while the instance wakes, or a backhaul) is asked again,
32 // waiting Retry-After, or importVerifyRetry without one.
33 importVerifyTries = 20
34 importVerifyRetry = 3 * time.Second
35)
36
37// The usage block of carlos instances import, one line to a literal. The
38// continuation lines are indented when the block is put together.
39const (
40 importUsageSynopsis = "usage: carlos instances import --app <a> --host <h> --file <snapshot> --replaces <host:port>"
41 importUsageOptions = "[--verify <path>] [--environment <e>]"
42 importUsageCommit = "carlos instances import commit --app <a> --host <h>"
43 importUsageAbort = "carlos instances import abort --app <a> --host <h>"
44 importUsageDescription = "carlos instances import moves an instance the platform never ran onto the platform."
45)
46
47// importUsage is the block flagUsage prints above the flags of import,
48// import commit and import abort.
49func importUsage() string {
50 optionsIndent := strings.Repeat(" ", 30)
51 verbIndent := strings.Repeat(" ", 7)
52 return strings.Join([]string{
53 importUsageSynopsis,
54 optionsIndent + importUsageOptions,
55 verbIndent + importUsageCommit,
56 verbIndent + importUsageAbort,
57 "",
58 importUsageDescription,
59 }, "\n")
60}
61
62// importCheckWaitOverLine is what the CLI says when the console has not
63// kept the upload by the end of importCheckWait.
64func importCheckWaitOverLine(app, host string) string {
65 return fmt.Sprintf("the console has not kept the upload of %s after %d minutes. Check carlos instances list. To start over, run carlos instances import abort --app %s --host %s, then run the import again.",
66 host, int(importCheckWait/time.Minute), app, host)
67}
68
69// importEndedLine is what the CLI says when the record it was following
70// has gone, or now carries another import.
71func importEndedLine(host string) string {
72 return fmt.Sprintf("the import of %s has ended. It was deleted, or another import of %s began. Check carlos instances list.", host, host)
73}
74
75// importNothingToAbortError is import abort's error for a host with no
76// record. The CLI cannot tell a typo from an import that was aborted and
77// cleaned up, so the sentence says both.
78func importNothingToAbortError(host, app string) error {
79 return fmt.Errorf("%s is not an instance of %s, so there is no import to abort. An import that was aborted and cleaned up leaves no instance behind.", host, app)
80}
81
82// errImportFailed is an import that ended badly once its own line was
83// printed: the command exits 1 and adds nothing.
84var errImportFailed = errors.New("instances import failed")
85
86// importOver is an import that ended on the console or the box, not by
87// this command: refused, aborted, rolled back, or no longer on the record.
88// line is what to print. The token is not shown: there is nothing left to
89// verify.
90type importOver struct{ line string }
91
92func (e importOver) Error() string { return e.line }
93
94// importLive is an exit that leaves the import live and uncommitted
95// (spec §5, ruling M17): the token is shown first, then lines.
96type importLive struct{ lines []string }
97
98func (e importLive) Error() string { return strings.Join(e.lines, "\n") }
99
100// importStopSignals end an import early, through its live exit: Ctrl-C, a
101// supervisor's SIGTERM, and a closed terminal (SIGHUP). Not SIGPIPE, unlike
102// carlos db's set: a write to a reset connection (the upload, a poll, the
103// verify) raises it, and that must stay the request's own error rather than
104// cancel the whole import. syscall.SIGHUP and SIGTERM exist on every
105// platform the CLI builds for, as carlos db's set relies on.
106var importStopSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGHUP}
107
108// importJob is one `carlos instances import`.
109type importJob struct {
110 acct apiAcct
111 app, host, file, replaces, environment, verify, message string
112 verifyClient *http.Client // nil: a client with a 2-minute timeout
113}
114
115// runInstancesImport is `carlos instances import` (seeded-instances spec §5).
116func runInstancesImport(args []string) {
117 if len(args) > 0 && (args[0] == "commit" || args[0] == "abort") {
118 runInstancesImportVerb(args[0], args[1:])
119 return
120 }
121 fs := flag.NewFlagSet("instances import", flag.ExitOnError)
122 messageFlags := addMutationMessageFlags(fs)
123 app := appFlag(fs, "app the instance belongs to")
124 host := fs.String("host", "", "instance host — one DNS label + \".\" + the app's instance domain")
125 file := fs.String("file", "", "the snapshot to import: a consistent SQLite file taken after the old process stopped")
126 replaces := fs.String("replaces", "", "the address of the route this import takes over, as host:port")
127 environment := fs.String("environment", "", "named config bundle this route binds to (default: the app's default bundle) — carlos env environments lists them")
128 verify := fs.String("verify", "", "a path to request with the verify token once swapped. A 2xx commits the import, anything else aborts it")
129 consoleFlag := consoleOption(fs, "console to create through (default: the only logged-in one)")
130 acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)")
131 acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account")
132 fs.Usage = flagUsage(fs, importUsage())
133 fs.Parse(args)
134 if *host == "" || *file == "" || *replaces == "" {
135 fs.Usage()
136 os.Exit(2)
137 }
138 // A --verify that cannot make a request would fail only after the swap,
139 // and abort a good import over a typo.
140 if *verify != "" {
141 if _, err := url.Parse(importVerifyTarget(strings.TrimSpace(*host), strings.TrimSpace(*verify))); err != nil {
142 fs.Usage()
143 os.Exit(2)
144 }
145 }
146 sess := requireAPISession("import", *consoleFlag)
147 printUsing(stdout, sess)
148 acct := resolvedAccount(sess, acctRaw, acctIDRaw)
149 app2 := resolvedApp(sess, acct, *app, instancesRecordsUsage)
150 message, merr := resolveMutationMessage(fs, messageFlags)
151 if merr != nil {
152 log.Fatalf("instances import: %v", merr)
153 }
154 // Ctrl-C, SIGTERM or a closed terminal cancels the import's context
155 // rather than killing the process, so an import left live still shows
156 // its token and the commands to finish it (M17).
157 ctx, stop := signal.NotifyContext(context.Background(), importStopSignals...)
158 defer stop()
159 err := sess.runImport(ctx, stdout, importJob{acct: acct, app: app2, host: strings.TrimSpace(*host), file: *file,
160 replaces: strings.TrimSpace(*replaces), environment: strings.TrimSpace(*environment), verify: strings.TrimSpace(*verify), message: message})
161 switch {
162 case errors.Is(err, errImportFailed):
163 stop()
164 os.Exit(1)
165 case err != nil:
166 log.Fatalf("instances import: %v", err)
167 }
168}
169
170// runImport hashes the file and begins, then drives the import (driveImport)
171// and settles what its ending prints. The verify token stays in memory. It
172// is printed (SD-N72) right after begin without --verify, and with --verify
173// only before an exit that leaves the import live and uncommitted. A
174// cancelled ctx after begin is such an exit: the token and the commands to
175// finish by hand, and no commit or abort is sent. Cancelled before begin
176// answers, it is begin's error.
177func (s *apiSession) runImport(ctx context.Context, w io.Writer, j importJob) error {
178 f, err := os.Open(j.file)
179 if err != nil {
180 return err
181 }
182 defer f.Close()
183 h := sha256.New()
184 size, err := io.Copy(h, f)
185 if err != nil {
186 return err
187 }
188 fmt.Fprintln(w, fmt.Sprintf("uploading %s (%s)", j.file, humanBytes(&size)))
189 rec, token, err := s.importBegin(ctx, j.acct, j.app, j.host, j.environment, hex.EncodeToString(h.Sum(nil)), size, j.replaces, j.message)
190 if err != nil {
191 return err
192 }
193 id := ""
194 if rec.Seed != nil {
195 id = rec.Seed.ID
196 }
197 shown := false
198 showToken := func() {
199 if !shown {
200 fmt.Fprintln(w, fmt.Sprintf("verify token: %s. It is shown only this once.", token))
201 shown = true
202 }
203 }
204 if j.verify == "" {
205 showToken()
206 }
207 err = s.driveImport(ctx, w, j, f, size, id, token)
208 var over importOver
209 var live importLive
210 switch {
211 case err == nil, errors.Is(err, errImportFailed):
212 return err
213 case errors.As(err, &over):
214 fmt.Fprintln(w, over.line)
215 return errImportFailed
216 }
217 showToken()
218 if ctx.Err() != nil {
219 fmt.Fprintln(w, importFinishLine(j.app, j.host))
220 return errImportFailed
221 }
222 if errors.As(err, &live) {
223 for _, l := range live.lines {
224 fmt.Fprintln(w, l)
225 }
226 return errImportFailed
227 }
228 return err
229}
230
231// driveImport is everything after begin: upload, wait for the check,
232// stopped, wait for the swap, and then either the hand-finish commands or
233// the verify and its commit or abort.
234func (s *apiSession) driveImport(ctx context.Context, w io.Writer, j importJob, f *os.File, size int64, id, token string) error {
235 if _, err := f.Seek(0, io.SeekStart); err != nil {
236 return err
237 }
238 if err := s.importUpload(ctx, j.acct, j.app, j.host, id, f, size); err != nil {
239 return err
240 }
241 // The console answered 202 and checks the file on its own; the record
242 // stays awaiting-upload until it is kept.
243 if _, err := s.pollImport(ctx, j, importCheckWait, importLook(j.host, id, "uploaded", "ready", "swapped", "placed", "committed")); err != nil {
244 if errors.Is(err, errSeedWaitOver) {
245 return importLive{lines: []string{importCheckWaitOverLine(j.app, j.host)}}
246 }
247 return err
248 }
249 if err := s.importStep(ctx, j.acct, j.app, j.host, id, "stopped"); err != nil {
250 return err
251 }
252 fmt.Fprintln(w, fmt.Sprintf("uploaded. The old process of %s is marked stopped, and the box swaps it on its next pass.", j.host))
253 fmt.Fprintln(w, fmt.Sprintf("waiting for the box to swap %s", j.host))
254 swapped, err := s.pollImport(ctx, j, importSwapWait, importLook(j.host, id, "swapped", "placed", "committed"))
255 if err != nil {
256 if errors.Is(err, errSeedWaitOver) {
257 return importLive{lines: []string{fmt.Sprintf("the box has not swapped %s after %d minutes. Check carlos instances list, then commit or abort.",
258 j.host, int(importSwapWait/time.Minute))}}
259 }
260 return err
261 }
262 if swapped.Seed.State == "committed" {
263 // Committed by hand (the token was shown at begin) between two looks.
264 fmt.Fprintln(w, importCommittedLine(j.host))
265 return nil
266 }
267 fmt.Fprintln(w, fmt.Sprintf("swapped. Until it is committed, %s answers only requests that carry the verify token.", j.host))
268 if j.verify == "" {
269 fmt.Fprintln(w, importFinishLine(j.app, j.host))
270 return nil
271 }
272 client := j.verifyClient
273 if client == nil {
274 client = &http.Client{Timeout: 2 * time.Minute}
275 }
276 code, verr := verifyImport(ctx, client, j.host, j.verify, token)
277 if ctx.Err() != nil {
278 // Cancelled is not a verdict: neither commit nor abort.
279 return ctx.Err()
280 }
281 if verr == nil && code >= 200 && code < 300 {
282 fmt.Fprintln(w, fmt.Sprintf("verify passed (%d). Committing.", code))
283 fmt.Fprintln(w, "waiting for the box to finish placing the snapshot")
284 if _, err := s.pollImport(ctx, j, importPlaceWait, importLook(j.host, id, "placed", "committed")); err != nil {
285 if errors.Is(err, errSeedWaitOver) {
286 return importLive{lines: []string{importFinishLine(j.app, j.host)}}
287 }
288 return err
289 }
290 if err := s.importStep(ctx, j.acct, j.app, j.host, id, "commit"); err != nil {
291 return err
292 }
293 fmt.Fprintln(w, importCommittedLine(j.host))
294 return nil
295 }
296 status := strconv.Itoa(code)
297 if verr != nil {
298 status = verr.Error()
299 }
300 fmt.Fprintln(w, fmt.Sprintf("verify failed (%s). Aborting the import.", status))
301 if err := s.importStep(ctx, j.acct, j.app, j.host, id, "abort"); err != nil {
302 return err
303 }
304 fmt.Fprintln(w, importAbortedLine(j.host))
305 return errImportFailed
306}
307
308// pollImport is pollSeed for an import: a record that has gone ends the
309// import with the line to print, importEndedLine, rather than the
310// membership sentence.
311func (s *apiSession) pollImport(ctx context.Context, j importJob, limit time.Duration, look func(apiInstanceRecord) (bool, error)) (apiInstanceRecord, error) {
312 rec, err := s.pollSeed(ctx, j.acct, j.app, j.host, limit, look)
313 var gone seedGone
314 if errors.As(err, &gone) {
315 return rec, importOver{line: importEndedLine(j.host)}
316 }
317 return rec, err
318}
319
320// importLook is pollSeed's look for an import: done once the seed is in
321// one of states. A refused, aborted or rolled-back seed, or a record that
322// no longer carries this import (id, when known), ends the import with the
323// line to print: SD-N78, the seed's reason, seedCell's "aborting", or, for
324// a record that carries no such import, importEndedLine.
325func importLook(host, id string, states ...string) func(apiInstanceRecord) (bool, error) {
326 return func(rec apiInstanceRecord) (bool, error) {
327 sd := rec.Seed
328 if sd == nil || (id != "" && sd.ID != id) {
329 return false, importOver{line: importEndedLine(host)}
330 }
331 switch sd.State {
332 case "refused":
333 return false, importOver{line: fmt.Sprintf("the box refused the import: %s", sd.Reason)}
334 case "aborted", "rolled-back":
335 if sd.Reason != "" {
336 return false, importOver{line: sd.Reason}
337 }
338 return false, importOver{line: seedCell(rec)}
339 }
340 return slices.Contains(states, sd.State), nil
341 }
342}
343
344// importLanded is, per verb, the states that prove it landed.
345var importLanded = map[string][]string{
346 "stopped": {"ready", "swapped", "placed", "committed"},
347 "commit": {"committed"},
348 "abort": {"aborted", "rolled-back"},
349}
350
351// importStep runs one of the import's verbs on seed id, which the console
352// requires (final review F1). When the console refuses it, the record says
353// whether it landed anyway: a stopped whose first answer was lost is
354// refused naming a later state (SD-N54), and a commit or abort that
355// overlapped another is refused as changed (SD-N60). An abort whose record
356// has then gone is done, since the abort finishes a rolled-back import and
357// deletes its record.
358func (s *apiSession) importStep(ctx context.Context, a apiAcct, app, host, id, verb string) error {
359 err := s.importVerb(ctx, a, app, host, id, verb)
360 if err == nil {
361 return nil
362 }
363 rec, found, rerr := s.instanceGet(ctx, a, app, host)
364 switch {
365 case rerr != nil:
366 return err
367 case !found:
368 if verb == "abort" && id != "" {
369 return nil
370 }
371 return err
372 case rec.Seed == nil || (id != "" && rec.Seed.ID != id):
373 return err
374 case slices.Contains(importLanded[verb], rec.Seed.State):
375 return nil
376 }
377 return err
378}
379
380func importFinishLine(app, host string) string {
381 return fmt.Sprintf("commit with: carlos instances import commit --app %s --host %s. Abort with: carlos instances import abort --app %s --host %s.", app, host, app, host)
382}
383
384func importCommittedLine(host string) string {
385 return fmt.Sprintf("committed. %s is served by the platform now.", host)
386}
387
388func importAbortedLine(host string) string {
389 return fmt.Sprintf("aborted. The box puts the old route of %s back on its next pass.", host)
390}
391
392// runInstancesImportVerb is `carlos instances import commit|abort`: the
393// verb alone, for an import left to be finished by hand.
394func runInstancesImportVerb(verb string, args []string) {
395 fs := flag.NewFlagSet("instances import "+verb, flag.ExitOnError)
396 app := appFlag(fs, "app the instance belongs to")
397 host := fs.String("host", "", "instance host — one DNS label + \".\" + the app's instance domain")
398 consoleFlag := consoleOption(fs, "console to create through (default: the only logged-in one)")
399 acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)")
400 acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account")
401 fs.Usage = flagUsage(fs, importUsage())
402 fs.Parse(args)
403 if *host == "" {
404 fs.Usage()
405 os.Exit(2)
406 }
407 sess := requireAPISession("import", *consoleFlag)
408 printUsing(stdout, sess)
409 acct := resolvedAccount(sess, acctRaw, acctIDRaw)
410 app2 := resolvedApp(sess, acct, *app, instancesRecordsUsage)
411 if err := sess.runImportVerb(context.Background(), stdout, acct, app2, strings.TrimSpace(*host), verb); err != nil {
412 log.Fatalf("instances import %s: %v", verb, err)
413 }
414}
415
416// runImportVerb acts on the import the host holds now: it reads the record
417// and sends the seed id it read (final review F1), so a re-begun import is
418// only ever acted on by a command run after it began. A host with no record,
419// or no seed, sends no id, and the console's own answer is the error.
420func (s *apiSession) runImportVerb(ctx context.Context, w io.Writer, a apiAcct, app, host, verb string) error {
421 rec, found, err := s.instanceGet(ctx, a, app, host)
422 if err != nil {
423 return err
424 }
425 id := ""
426 if found && rec.Seed != nil {
427 id = rec.Seed.ID
428 }
429 if err := s.importStep(ctx, a, app, host, id, verb); err != nil {
430 if verb == "abort" && !found && err.Error() == fmt.Sprintf("%s is not an instance of %s", host, app) {
431 return importNothingToAbortError(host, app)
432 }
433 return err
434 }
435 if verb == "commit" {
436 fmt.Fprintln(w, importCommittedLine(host))
437 return nil
438 }
439 fmt.Fprintln(w, importAbortedLine(host))
440 return nil
441}
442
443// verifyImport is the verify request through the edge (spec §5.5) to
444// https://<host><path>.
445func verifyImport(ctx context.Context, client *http.Client, host, path, token string) (int, error) {
446 return verifyImportURL(ctx, client, importVerifyTarget(host, path), token)
447}
448
449// importVerifyTarget is the verify URL, https://<host><path>.
450func importVerifyTarget(host, path string) string {
451 if !strings.HasPrefix(path, "/") {
452 path = "/" + path
453 }
454 return "https://" + host + path
455}
456
457// verifyImportURL asks target with the verify token and returns the
458// status that decides the import. 502, 503 and 504 are asked again (the
459// gate answers 503 with Retry-After while the instance wakes, and a
460// backhaul can answer 502 or 504 meanwhile), up to importVerifyTries
461// times. A redirect is never followed: only a 2xx of the path itself
462// commits.
463func verifyImportURL(ctx context.Context, client *http.Client, target, token string) (int, error) {
464 c := *client
465 c.CheckRedirect = neverFollow
466 for try := 0; ; try++ {
467 req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
468 if err != nil {
469 return 0, err
470 }
471 req.Header.Set("Carlos-Import-Verify", token)
472 res, err := c.Do(req)
473 if err != nil {
474 return 0, err
475 }
476 io.Copy(io.Discard, io.LimitReader(res.Body, 1<<20))
477 res.Body.Close()
478 switch res.StatusCode {
479 case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
480 default:
481 return res.StatusCode, nil
482 }
483 if try >= importVerifyTries {
484 return res.StatusCode, nil
485 }
486 wait := importVerifyRetry
487 if n, err := strconv.Atoi(res.Header.Get("Retry-After")); err == nil && n >= 0 && n <= 30 {
488 wait = time.Duration(n) * time.Second
489 }
490 if err := seedWait(ctx, wait); err != nil {
491 return 0, err
492 }
493 }
494}
495