carlos / platform Public

branch:

Clone

git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1	package main
2
3	import (
4	"bufio"
5	"bytes"
6	"context"
7	"encoding/json"
8	"errors"
9	"flag"
10	"fmt"
11	"io"
12	"maps"
13	"math"
14	"net/http"
15	"net/url"
16	"os"
17	"slices"
18	"strings"
19	"time"
20
21	"github.com/carlosframework/platform/internal/awsinv"
22	"github.com/carlosframework/platform/internal/hostfence"
23	"github.com/carlosframework/platform/internal/hostresolve"
24	"github.com/carlosframework/platform/internal/moves"
25	"github.com/carlosframework/platform/internal/provision"
26	)
27
28	// `carlos move` (spec §12, moves Phase 1 Plan 6 Task 1a): request a move of
29	// one instance to another region, read its state, cut, abort, and retry a
30	// stuck one. Operators only, through the console, with
31	// CARLOS_MOVES_PHASE1 on there. The verb refuses to send anything to a
32	// console whose move answer lacks the Phase 1 marker (preflight L4), so it
33	// can never start a legacy move.
34	//
35	// Waiting is the default. The wait polls the flat view every two seconds and
36	// reads detail=1 only when the state or the phase changes (L19). A 429, on a
37	// read or on a mutation, backs off to ten seconds, is said once, and never
38	// ends the command: the doors' rate limit answers before any handler, so the
39	// same call is sent again. The outcome is read only from `last`, bound to the
40	// id the console gave the move (L2, Review Focus 1), and its regions are that
41	// move's own, bound from its own answers; a region the wait never learnt is
42	// never guessed. Exit codes (L24): 0 done, or a command accepted with
43	// --no-wait (which reads nothing after the acceptance); 3 rolled back or
44	// aborted; 4 stuck, rolling back past its release deadline, relocating,
45	// ready for a manual cut, or an outcome not recorded; 5 refused (by the
46	// console, or before anything was sent); 1 the console not reached, its
47	// answer unreadable, or a 5xx; 2 a usage error.
48	//
49	// What the operator reads is this file's own copy, through moveSay (the copy
50	// gate's sink, movesLocalSinks), or text the console wrote, as serverText
51	// through saySpoken. serverText is made only in roundTrip, from the body of a
52	// response the HTTP client returned (movesServerText). Go's own errors are
53	// never shown. The flag help is said by moveFlagHelp (movesSinkAllow).
54
55	const (
56	moveExitDone = 0
57	moveExitError = 1
58	moveExitUsage = 2
59	moveExitRolled = 3
60	moveExitWaiting = 4
61	moveExitRefused = 5
62	moveProtocolMark = "phase1"
63	)
64
65	// The sentences this file adds (copy record MV-N350 to MV-N379, pending).
66	const (
67	moveNotEnabled = "Moves are not enabled on this console."
68	moveNoApp = "carlos move needs --app: name the app the instance belongs to."
69	moveAsRoot = "Run carlos move as yourself, not as root. It uses your own credentials."
70	moveDirectMode = "carlos move works through a console. Unset CARLOS_DEPLOYMENT_BUCKET and CARLOS_DEPLOYMENT_DIR and run carlos auth login."
71	moveNotRecorded = "This move's outcome is not recorded."
72	moveLimited = "The console is limiting requests. Checking every 10 seconds."
73	moveNeedsHost = "carlos move needs --host <host>."
74	moveNeedsTo = "carlos move needs --to <region>."
75	moveNeedsRetry = "carlos move resolve needs --retry or --declare-lost <box>."
76	moveUnreachable = "Could not reach the console. Check the connection and try again."
77	moveUnreadable = "The console's answer could not be read."
78	moveNotFound = "Not found. Check --account and --app, and that you are a member of the instance's account."
79	moveNoSession = "carlos move could not sign in to a console. Run carlos auth login, or check --console, CARLOS_CONSOLE and CARLOS_SERVICE_TOKEN."
80	moveStopped = "Stopped waiting."
81	moveRetryOrLost = "carlos move resolve takes --retry or --declare-lost, not both."
82	moveNoAWS = "declare-lost needs your AWS credentials to confirm the box is stopped or terminated."
83	moveNoFlipTime = "The move's flip time could not be read, so its target cannot be declared lost from here."
84	moveOtherLoss = "The console asks to confirm a loss carlos move does not know. Use carlos ops host resolve."
85	moveLossChanged = "The move changed since it was read. Nothing was declared. Run the command again."
86	// The offline subcommand named no verb, or one it does not have (MV-O181,
87	// which replaced MV-O56 when stage became a verb).
88	moveOfflineUsage = "carlos move offline needs stage, start, seal, flip, open, finish, abort or status."
89	)
90
91	func moveSay(out io.Writer, line string) {
92	fmt.Fprintln(out, line)
93	}
94
95	// serverText is text the console wrote: its refusal sentence, or the stuck
96	// reason its detail answer carries from the record (a sentence a provision
97	// transition recorded). It is made only in roundTrip, from the body of a
98	// response the HTTP client returned, and said only through saySpoken. The
99	// copy gate holds both (TestMovesServerTextHasOneMaker).
100	type serverText struct{ spoken string }
101
102	// saySpoken says text the console wrote, as it wrote it.
103	func saySpoken(out io.Writer, t serverText) {
104	fmt.Fprintln(out, t.spoken)
105	}
106
107	func runMove(args []string) {
108	deps := moveDeps{session: resolveMode, euid: os.Geteuid, poll: 2 * time.Second, stdin: os.Stdin, ec2: awsinv.NewEC2Reader}
109	os.Exit(moveMain(context.Background(), args, os.Stdout, deps))
110	}
111
112	// moveDeps are what moveMain reaches outside itself, so a test can run it
113	// against an httptest console.
114	type moveDeps struct {
115	session func(consoleFlag string) (*apiSession, error)
116	euid func() int
117	poll time.Duration
118	// stdin is where a declare-lost confirmation is read; ec2 is the
119	// operator's own read-only EC2 seam (awsinv.NewEC2Reader).
120	stdin io.Reader
121	ec2 func(ctx context.Context, region string) (awsinv.ExactEC2API, error)
122	}
123
124	type moveArgs struct {
125	verb string // request \| status \| cut \| abort \| resolve \| offline
126	offline string // the offline verb: stage \| start \| seal \| flip \| open \| finish \| abort \| status
127	host, to, app string
128	account, accountID string
129	console string
130	manualCut, noWait, wait bool
131	retry bool
132	lostBox string // the box to declare lost (Task 1b)
133	}
134
135	// parseMoveArgs reads `carlos move [status\|cut\|abort\|resolve] <flags>`. Each
136	// verb takes only its own flags; a flag another verb owns is a usage error.
137	// Usage problems are said on out and returned as an error.
138	func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {
139	a := moveArgs{verb: "request"}
140	if len(args) > 0 {
141	switch args[0] {
142	case "status", "cut", "abort", "resolve":
143	a.verb, args = args[0], args[1:]
144	case "offline":
145	if len(args) < 2 \|\| !slices.Contains(offlineVerbs, args[1]) {
146	moveSay(out, moveOfflineUsage)
147	return moveArgs{}, errors.New("no offline verb")
148	}
149	a.verb, a.offline, args = "offline", args[1], args[2:]
150	}
151	}
152	name := "move " + a.verb
153	if a.offline != "" {
154	name += " " + a.offline
155	}
156	fs := flag.NewFlagSet(name, flag.ContinueOnError)
157	var parsed strings.Builder
158	fs.SetOutput(&parsed)
159	fs.Usage = func() {} // the help is said by moveFlagHelp
160	fs.StringVar(&a.host, "host", "", "the instance's host name")
161	fs.StringVar(&a.app, "app", "", "the app the instance belongs to")
162	fs.StringVar(&a.app, "a", "", "shorthand for --app")
163	fs.StringVar(&a.account, "account", "", "the account the app belongs to (default: your default account)")
164	fs.StringVar(&a.accountID, "account-id", "", "the account, by its id")
165	fs.StringVar(&a.console, "console", "", "the console to work through (default: the only logged-in one)")
166	switch a.verb {
167	case "request":
168	fs.StringVar(&a.to, "to", "", "the region to move the instance to")
169	fs.BoolVar(&a.manualCut, "manual-cut", false, "stop when the move is ready, and cut with carlos move cut")
170	fs.BoolVar(&a.noWait, "no-wait", false, "return once the console has accepted the request")
171	case "status":
172	fs.BoolVar(&a.wait, "wait", false, "wait for the move in flight, or the last one, to finish")
173	case "cut", "abort":
174	fs.BoolVar(&a.noWait, "no-wait", false, "return once the console has accepted it")
175	case "offline":
176	if a.offline == "start" \|\| a.offline == "stage" {
177	fs.StringVar(&a.to, "to", "", "the region to move the instance to")
178	}
179	case "resolve":
180	fs.BoolVar(&a.retry, "retry", false, "retry a stuck move once its cause is fixed")
181	fs.StringVar(&a.lostBox, "declare-lost", "", "declare the move's target box lost, once AWS shows it stopped or terminated")
182	fs.BoolVar(&a.noWait, "no-wait", false, "return once the console has accepted it")
183	}
184	if err := fs.Parse(args); err != nil {
185	moveFlagHelp(out, fs, &parsed)
186	return moveArgs{}, err
187	}
188	if fs.NArg() > 0 {
189	moveSay(out, moveNeedsHost)
190	return moveArgs{}, errors.New("unexpected arguments")
191	}
192	switch {
193	case a.host == "":
194	moveSay(out, moveNeedsHost)
195	return moveArgs{}, errors.New("no --host")
196	case (a.verb == "request" \|\| a.offline == "start" \|\| a.offline == "stage") && a.to == "":
197	moveSay(out, moveNeedsTo)
198	return moveArgs{}, errors.New("no --to")
199	case a.verb == "resolve" && a.retry && a.lostBox != "":
200	moveSay(out, moveRetryOrLost)
201	return moveArgs{}, errors.New("--retry and --declare-lost")
202	case a.verb == "resolve" && !a.retry && a.lostBox == "":
203	moveSay(out, moveNeedsRetry)
204	return moveArgs{}, errors.New("no --retry or --declare-lost")
205	}
206	if a.verb != "status" && a.verb != "offline" {
207	a.wait = !a.noWait
208	}
209	return a, nil
210	}
211
212	// moveFlagHelp says the flag help: what the flag package wrote while parsing
213	// (its error for a bad flag, if any) and its rendering of the verb's flags
214	// and their usage strings. It is the copy gate's one help emission
215	// (movesSinkAllow); the usage strings go to copy review with the CLI page.
216	func moveFlagHelp(out io.Writer, fs flag.FlagSet, parsed strings.Builder) {
217	fs.SetOutput(parsed)
218	fs.PrintDefaults()
219	moveSay(out, strings.TrimRight(parsed.String(), "\n"))
220	}
221
222	// moveMain runs one `carlos move` and returns its exit code.
223	func moveMain(ctx context.Context, args []string, out io.Writer, d moveDeps) int {
224	a, err := parseMoveArgs(args, out)
225	if errors.Is(err, flag.ErrHelp) {
226	return moveExitDone
227	}
228	if err != nil {
229	return moveExitUsage
230	}
231	// L25: the operator's own credentials are what count (declare-lost reads
232	// the cloud with them), and run as root rootaws replaces them. One rule
233	// for the whole verb.
234	if d.euid() == 0 {
235	moveSay(out, moveAsRoot)
236	return moveExitRefused
237	}
238	// resolveMode's own sentences are in no copy record (the general record
239	// is the nearest-region plan's), so carlos move says its own.
240	sess, err := d.session(a.console)
241	if err != nil {
242	moveSay(out, moveNoSession)
243	return moveExitError
244	}
245	if sess == nil {
246	moveSay(out, moveDirectMode)
247	return moveExitRefused
248	}
249	acct := resolvedAccount(sess, &a.account, &a.accountID)
250	app := resolvedApp(sess, acct, a.app, moveNoApp)
251	m := &mover{sess: sess, out: out, host: a.host, poll: d.poll,
252	path: apiPath(acct, app, "/instances/"+url.PathEscape(a.host)+"/move")}
253
254	// The offline move is the operator's tool for a console with Phase 1 off.
255	// It has no Phase 1 marker to check: its doors refuse for themselves.
256	if a.verb == "offline" {
257	m.path += "/offline"
258	return m.offline(ctx, a)
259	}
260
261	// L4: read the status first, and send nothing to a console that does
262	// not answer as Phase 1.
263	view, code := m.readFlat(ctx)
264	if code >= 0 {
265	return code
266	}
267	if view.Protocol != moveProtocolMark {
268	moveSay(out, moveNotEnabled)
269	return moveExitRefused
270	}
271
272	switch a.verb {
273	case "status":
274	return m.status(ctx, view, a.wait)
275	case "request":
276	cut := "auto"
277	if a.manualCut {
278	cut = "manual"
279	}
280	return m.mutate(ctx, http.MethodPost, "", map[string]string{"region": a.to, "cut": cut}, a.wait, true)
281	case "cut":
282	return m.mutate(ctx, http.MethodPost, "/cut", struct{}{}, a.wait, false)
283	case "abort":
284	return m.mutate(ctx, http.MethodDelete, "", nil, a.wait, false)
285	default: // resolve
286	if a.lostBox != "" {
287	return m.declareLost(ctx, a.lostBox, d, a.wait)
288	}
289	return m.mutate(ctx, http.MethodPost, "/resolve", map[string]bool{"retry": true}, a.wait, false)
290	}
291	}
292
293	// The console's answers, as this CLI reads them (Task 2's shapes, copied
294	// here as the CLI convention is).
295	type moveFlat struct {
296	State string `json:"state"`
297	Phase string `json:"phase"`
298	Target string `json:"target"`
299	UpdatedAt int64 `json:"updated_at"`
300	ID string `json:"id"`
301	Last *moveOutcome `json:"last"`
302	Protocol string `json:"protocol"`
303	}
304
305	// moveOutcome is the record's last close (D11). Reason is read when the
306	// console's answer carries one; today's record does not, and the CLI never
307	// makes one up.
308	type moveOutcome struct {
309	ID string `json:"id"`
310	Outcome string `json:"outcome"`
311	At time.Time `json:"at"`
312	OfflineS float64 `json:"offline_s"`
313	Reason string `json:"reason"`
314	}
315
316	type moveAnswer struct {
317	moveFlat
318	Detail *moveDetailView `json:"detail"`
319	// stuck is the journal's stuck reason as server text, set by roundTrip
320	// (hasStuck when there is one).
321	stuck serverText
322	hasStuck bool
323	// waiting is the offline status view's "waiting": the sentence the next
324	// verb would be refused with, as the console wrote it (set by roundTrip).
325	WireWaiting string `json:"waiting"` // read only by roundTrip
326	waiting serverText
327	hasWaiting bool
328	}
329
330	type moveDetailView struct {
331	From string `json:"from"`
332	Region string `json:"region"`
333	Move *moveJournal `json:"move"`
334	Claim *moveClaim `json:"claim"`
335	TargetReport *moveReport `json:"target_report"`
336	WaitingForBox string `json:"waiting_for_box"`
337	}
338
339	type moveJournal struct {
340	Incarnation string `json:"incarnation"`
341	ID string `json:"id"`
342	To string `json:"to"`
343	Cut string `json:"cut"`
344	State string `json:"state"`
345	CutPhase string `json:"cut_phase"`
346	CutBudgetS int `json:"cut_budget_s"`
347	FlippedAt *time.Time `json:"flipped_at"`
348	Reason string `json:"reason"`
349	WireStuck string `json:"stuck_reason"` // read only by roundTrip
350	Rollback *moveRollback `json:"rollback"`
351	}
352
353	type moveRollback struct {
354	Reason string `json:"reason"`
355	TargetPhase string `json:"target_phase"`
356	ReleaseDeadline *time.Time `json:"release_deadline"`
357	}
358
359	type moveClaim struct {
360	Kind string `json:"kind"`
361	MvBox string `json:"target"`
362	}
363
364	type moveReport struct {
365	Checks map[string]struct {
366	OK bool `json:"ok"`
367	Reason string `json:"reason"`
368	} `json:"checks"`
369	}
370
371	// mover is one invocation's conversation with the move door.
372	type mover struct {
373	sess *apiSession
374	out io.Writer
375	host string
376	path string
377	poll time.Duration
378	// limited: a 429 was answered, so the command waits 10 seconds between
379	// calls now and has said so (once).
380	limited bool
381	}
382
383	func (m *mover) withDetail() string {
384	if strings.Contains(m.path, "?") {
385	return m.path + "&detail=1"
386	}
387	return m.path + "?detail=1"
388	}
389
390	// moveTransient is how many failed reads in a row the wait rides out before
391	// it gives up with exit 1: a console restart is not a failed move.
392	const moveTransient = 5
393
394	// moveReply is one response from the move door, as roundTrip read it.
395	type moveReply struct {
396	status int
397	answer moveAnswer // decoded, on a 2xx
398	body []byte // the 2xx body as read (a declare-lost inspection)
399	refusal serverText // the console's own sentence, on another status
400	refused bool // refusal holds one
401	moved string // the console's new origin, on a console-shaped redirect
402	failed bool // the console was not reached, or its answer not read
403	unread bool // ... and it was the answer that could not be read
404	stopped bool // the context ended the call
405	}
406
407	// roundTrip sends one request to the move door and reads its response. It is
408	// the one maker of serverText (movesServerText): the refusal's "error"
409	// sentence, and the stuck reason of a detail answer, read from the body of
410	// the response the HTTP client returned.
411	func (m *mover) roundTrip(ctx context.Context, method, path string, in any) moveReply {
412	var body io.Reader
413	if in != nil {
414	raw, err := json.Marshal(in)
415	if err != nil {
416	return moveReply{failed: true, unread: true}
417	}
418	body = bytes.NewReader(raw)
419	}
420	req, err := http.NewRequestWithContext(ctx, method, m.sess.c.base+path, body)
421	if err != nil {
422	return moveReply{failed: true}
423	}
424	if in != nil {
425	req.Header.Set("Content-Type", "application/json")
426	}
427	if m.sess.c.token != "" {
428	req.Header.Set("Authorization", "Bearer "+m.sess.c.token)
429	}
430	res, err := m.sess.c.httpc.Do(req)
431	if err != nil {
432	return moveReply{failed: true, stopped: ctx.Err() != nil}
433	}
434	defer res.Body.Close()
435	r := moveReply{status: res.StatusCode}
436	if moved := consoleMoved(m.sess.c.base, res.StatusCode, res.Header.Get("Location")); moved != nil {
437	r.moved = moved.To
438	return r
439	}
440	raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
441	if err != nil {
442	r.failed = true
443	return r
444	}
445	if res.StatusCode/100 == 2 {
446	r.body = raw
447	if json.Unmarshal(raw, &r.answer) != nil {
448	r.failed, r.unread = true, true
449	return r
450	}
451	if d := r.answer.Detail; d != nil && d.Move != nil && d.Move.WireStuck != "" {
452	r.answer.stuck, r.answer.hasStuck = serverText{spoken: d.Move.WireStuck}, true
453	}
454	if r.answer.WireWaiting != "" {
455	r.answer.waiting, r.answer.hasWaiting = serverText{spoken: r.answer.WireWaiting}, true
456	}
457	return r
458	}
459	var e struct {
460	Error string `json:"error"`
461	}
462	if json.Unmarshal(raw, &e) == nil && e.Error != "" {
463	r.refusal, r.refused = serverText{spoken: e.Error}, true
464	}
465	return r
466	}
467
468	// flat is the reply's flat view.
469	func (r moveReply) flat() moveFlat {
470	return r.answer.moveFlat
471	}
472
473	// sayReply says why a reply ended the command: the console's own sentence
474	// when it gave one, else this file's.
475	func (m *mover) sayReply(r moveReply) {
476	switch {
477	case r.moved != "":
478	moveSay(m.out, fmt.Sprintf("This console has moved to %s. Run carlos auth login --console %s, then run the command again.", r.moved, r.moved))
479	case r.stopped:
480	moveSay(m.out, moveStopped)
481	case r.unread:
482	moveSay(m.out, moveUnreadable)
483	case r.failed:
484	moveSay(m.out, moveUnreachable)
485	case r.refused:
486	saySpoken(m.out, r.refusal)
487	case r.status == http.StatusNotFound:
488	moveSay(m.out, moveNotFound)
489	default:
490	moveSay(m.out, fmt.Sprintf("The console answered %d and gave no reason.", r.status))
491	}
492	}
493
494	// limit notes a 429: from now on the command waits 10 seconds between calls,
495	// and says so the first time.
496	func (m *mover) limit() {
497	if !m.limited {
498	m.limited = true
499	moveSay(m.out, moveLimited)
500	}
501	}
502
503	// get reads one answer, riding out 429s (backing off, said once) and a few
504	// transient failures. code is -1 on success, else the exit code to return.
505	func (m *mover) get(ctx context.Context, path string) (moveAnswer, int) {
506	r, code := m.getReply(ctx, path)
507	return r.answer, code
508	}
509
510	// getReply is get with the whole reply, for a reader of the raw body.
511	func (m *mover) getReply(ctx context.Context, path string) (moveReply, int) {
512	failures := 0
513	for {
514	r := m.roundTrip(ctx, http.MethodGet, path, nil)
515	ok := !r.failed && r.moved == ""
516	switch {
517	case ok && r.status == http.StatusOK:
518	return r, -1
519	case r.moved != "" \|\| r.stopped:
520	m.sayReply(r)
521	return moveReply{}, moveExitError
522	case ok && r.status == http.StatusTooManyRequests:
523	m.limit()
524	case ok && r.status >= 400 && r.status < 500:
525	m.sayReply(r)
526	return moveReply{}, moveExitRefused
527	default:
528	failures++
529	if failures >= moveTransient {
530	m.sayReply(r)
531	return moveReply{}, moveExitError
532	}
533	}
534	if !m.sleep(ctx) {
535	moveSay(m.out, moveStopped)
536	return moveReply{}, moveExitError
537	}
538	}
539	}
540
541	func (m *mover) interval() time.Duration {
542	if m.limited {
543	return 5 * m.poll // 10 s at the production 2 s poll
544	}
545	return m.poll
546	}
547
548	// sleep waits one interval; false when the context ended first.
549	func (m *mover) sleep(ctx context.Context) bool {
550	t := time.NewTimer(m.interval())
551	defer t.Stop()
552	select {
553	case <-ctx.Done():
554	return false
555	case <-t.C:
556	return true
557	}
558	}
559
560	func (m *mover) readFlat(ctx context.Context) (moveFlat, int) {
561	a, code := m.get(ctx, m.path)
562	return a.moveFlat, code
563	}
564
565	func (m *mover) readDetail(ctx context.Context) (moveAnswer, int) {
566	a, code := m.get(ctx, m.withDetail())
567	if code < 0 && a.Detail == nil {
568	a.Detail = &moveDetailView{}
569	}
570	return a, code
571	}
572
573	// send sends one mutation, sending the same call again after the back-off
574	// while the doors' rate limit answers 429 (fix round 1, ruling 1: the limit
575	// answers before any handler, so nothing was done).
576	func (m *mover) send(ctx context.Context, method, tail string, body any) moveReply {
577	for {
578	r := m.roundTrip(ctx, method, m.path+tail, body)
579	if r.failed \|\| r.moved != "" \|\| r.status != http.StatusTooManyRequests {
580	return r
581	}
582	m.limit()
583	if !m.sleep(ctx) {
584	return moveReply{failed: true, stopped: true}
585	}
586	}
587	}
588
589	// mutate sends one move mutation. A refusal is the console's own sentence
590	// and exit 5. On acceptance the answer names the move (Task 2). With
591	// --no-wait that answer is all: its state is said, and nothing more is read
592	// (fix round 2, ruling 2). Else the move's detail, read once now, binds its
593	// regions, a request says M-C1, and the wait follows the move's id.
594	func (m *mover) mutate(ctx context.Context, method, tail string, body any, wait, request bool) int {
595	return m.accepted(ctx, m.send(ctx, method, tail, body), wait, request)
596	}
597
598	// accepted ends a mutation from the console's reply: a refusal or a failure
599	// is said and ends it; an acceptance is said (--no-wait) or waited on.
600	func (m *mover) accepted(ctx context.Context, r moveReply, wait, request bool) int {
601	if code := m.refusal(r); code >= 0 {
602	return code
603	}
604	view := r.flat()
605	if view.ID == "" \|\| view.Protocol != moveProtocolMark {
606	moveSay(m.out, moveNotEnabled)
607	return moveExitRefused
608	}
609	if !wait {
610	moveSay(m.out, fmt.Sprintf("Move %s of %s is %s.", view.ID, m.host, view.State))
611	return moveExitDone
612	}
613	w := newMoveWatch(m, view.ID)
614	w.toRegion = view.Target
615	a, code := m.readDetail(ctx)
616	if code >= 0 {
617	return code
618	}
619	w.learn(a)
620	if request && w.first("start") {
621	moveSay(m.out, moves.MoveStartingLine(m.host, w.fromRegion, w.toRegion))
622	}
623	return w.run(ctx)
624	}
625
626	// status answers `carlos move status`: without --wait, one line for the
627	// move in flight (and what the wait would say of its state), or the last
628	// move's outcome; with --wait, the wait for the move in flight, or the
629	// outcome of the last one when nothing is in flight.
630	func (m *mover) status(ctx context.Context, view moveFlat, wait bool) int {
631	id := view.ID
632	if id == "" && view.Last != nil {
633	id = view.Last.ID
634	}
635	if id == "" {
636	moveSay(m.out, fmt.Sprintf("No move is in flight for %s.", m.host))
637	return moveExitDone
638	}
639	w := newMoveWatch(m, id)
640	if wait {
641	return w.run(ctx)
642	}
643	a, code := m.readDetail(ctx)
644	if code >= 0 {
645	return code
646	}
647	if a.ID == id {
648	moveSay(m.out, fmt.Sprintf("Move %s of %s is %s.", id, m.host, a.State))
649	}
650	// The state's own sentence, and its exit code when it is one the wait
651	// would end on (an outcome, stuck, waiting); else 0.
652	if code := w.observe(a); code >= 0 {
653	return code
654	}
655	return moveExitDone
656	}
657
658	// moveWatch is the plan's moveWait: it follows one move, by id, and prints
659	// each sentence once.
660	type moveWatch struct {
661	m *mover
662	id string
663	fromRegion, toRegion string // this move's, from its own answers only
664	said map[string]bool
665	lastKey string
666	last moveAnswer // the detail read last
667	}
668
669	func newMoveWatch(m mover, id string) moveWatch {
670	return &moveWatch{m: m, id: id, said: map[string]bool{}}
671	}
672
673	// first reports whether key's sentence is due: true the first time only.
674	func (w *moveWatch) first(key string) bool {
675	if w.said[key] {
676	return false
677	}
678	w.said[key] = true
679	return true
680	}
681
682	// learn binds the move's regions from an answer about this move, and only
683	// such an answer: another move's record never names this one's regions.
684	func (w *moveWatch) learn(a moveAnswer) {
685	if a.ID != w.id {
686	return
687	}
688	if a.Detail.From != "" {
689	w.fromRegion = a.Detail.From
690	}
691	if a.Target != "" {
692	w.toRegion = a.Target
693	}
694	}
695
696	// run polls until this move ends or needs an operator, and returns the exit
697	// code. The detail is read only when the state or the phase changes (L19);
698	// between reads, the detail already read is looked at again, so a release
699	// deadline that passes while nothing changes is still seen. The outcome is
700	// never inferred from a state seen: it is last, bound to this move's id.
701	func (w *moveWatch) run(ctx context.Context) int {
702	for first := true; ; first = false {
703	if !first && !w.m.sleep(ctx) {
704	moveSay(w.m.out, moveStopped)
705	return moveExitError
706	}
707	view, code := w.m.readFlat(ctx)
708	if code >= 0 {
709	return code
710	}
711	a := w.last
712	if key := view.State + "\|" + view.Phase + "\|" + view.ID; key != w.lastKey {
713	if a, code = w.m.readDetail(ctx); code >= 0 {
714	return code
715	}
716	w.lastKey, w.last = key, a
717	}
718	if code := w.observe(a); code >= 0 {
719	return code
720	}
721	}
722	}
723
724	// observe prints what one detail answer says of this move, and returns an
725	// exit code when the wait ends here (else -1).
726	func (w *moveWatch) observe(a moveAnswer) int {
727	w.learn(a)
728	host, from, to := w.m.host, w.fromRegion, w.toRegion
729	if a.ID != w.id {
730	// This move is not in flight: its outcome is last, or it is not
731	// recorded (Review Focus 1).
732	return w.outcome(a)
733	}
734	j := a.Detail.Move
735	if j == nil {
736	j = &moveJournal{}
737	}
738	if w.first("start") {
739	moveSay(w.m.out, moves.MoveStartingLine(host, from, to))
740	}
741	switch a.State {
742	case "preparing":
743	if w.first("preparing") {
744	moveSay(w.m.out, fmt.Sprintf("Copying data. %s keeps serving while this runs.", host))
745	}
746	if r := a.Detail.TargetReport; r != nil {
747	if w.first("checking") {
748	moveSay(w.m.out, fmt.Sprintf("Checking that %s can run %s.", to, host))
749	}
750	ids := make([]string, 0, len(r.Checks))
751	for id := range r.Checks {
752	ids = append(ids, id)
753	}
754	slices.Sort(ids)
755	for _, id := range ids {
756	if c := r.Checks[id]; !c.OK && c.Reason != "" {
757	if w.first("check:" + c.Reason) {
758	moveSay(w.m.out, fmt.Sprintf("%s cannot take this move yet: %s.", to, c.Reason))
759	}
760	}
761	}
762	}
763	case "ready":
764	// A cut the source refused (the cut guard, a replica layout it
765	// cannot cut) is back at ready with the reason recorded, unpinned
766	// until the cause is gone and a repin clears the reason. Until then
767	// it is not ready to cut, so M-C12 is not said (fix round 1).
768	if j.Reason != "" {
769	if w.first("refused:" + j.Reason) {
770	moveSay(w.m.out, fmt.Sprintf("The cut did not start: %s.", j.Reason))
771	}
772	if j.Cut == "manual" {
773	return moveExitWaiting
774	}
775	break
776	}
777	if j.Cut == "manual" {
778	moveSay(w.m.out, fmt.Sprintf("Ready to cut. Run carlos move cut --host %s when you are ready.", host))
779	return moveExitWaiting
780	}
781	case "cutting":
782	if j.CutPhase != "" && j.CutPhase != "requested" && j.CutBudgetS > 0 {
783	if w.first("cut") {
784	moveSay(w.m.out, moves.CutStartingLine(to, host, moves.CutBudgetText(j.CutBudgetS)))
785	}
786	}
787	case "landing", "committed":
788	if w.first("landing") {
789	moveSay(w.m.out, moves.FlippedLine(from, to))
790	}
791	case "rolling_back", "reverting":
792	reason := j.Reason
793	if j.Rollback != nil && j.Rollback.Reason != "" {
794	reason = j.Rollback.Reason
795	}
796	if reason != "" {
797	if w.first("rollback") {
798	moveSay(w.m.out, moves.MovingBackLine(to, host, reason, from))
799	}
800	}
801	if rb := j.Rollback; rb != nil && rb.ReleaseDeadline != nil && time.Now().After(*rb.ReleaseDeadline) && rb.TargetPhase == "releasing" {
802	box := "the target box"
803	if c := a.Detail.Claim; c != nil && c.MvBox != "" {
804	box = c.MvBox
805	}
806	moveSay(w.m.out, fmt.Sprintf("Waiting for %s in %s to confirm it has stopped %s. %s is offline.", box, to, host, host))
807	return moveExitWaiting
808	}
809	case "stuck":
810	moveSay(w.m.out, fmt.Sprintf("%s could not start in %s or in %s. It is offline. Run carlos move resolve --host %s --retry once the cause is fixed.", host, to, from, host))
811	if a.hasStuck {
812	saySpoken(w.m.out, a.stuck)
813	}
814	return moveExitWaiting
815	case "relocating":
816	if region := a.Detail.WaitingForBox; region != "" {
817	moveSay(w.m.out, fmt.Sprintf("%s is waiting for a box in %s. Its last box was terminated before the move finished. Its data up to the last completed upload is kept in %s. It will start when a box joins the region, and any writes after that upload will be reported then.", host, region, region))
818	return moveExitWaiting
819	}
820	moveSay(w.m.out, fmt.Sprintf("%s is relocating after its box was terminated. This move will not finish.", host))
821	return moveExitWaiting
822	}
823	return -1
824	}
825
826	// outcome is the end of a move that is no longer in flight: last, if it is
827	// this move's, else not recorded. Its regions are the move's own, as the
828	// wait learnt them; one it never learnt is never filled in from the record
829	// (fix round 2, ruling 3). Without its region a sentence that names it gives
830	// way: M-C9 alone for M-C8 (whose destination is unknown), and the outcome
831	// line for M-C7, M-C9 and the aborted line.
832	func (w *moveWatch) outcome(a moveAnswer) int {
833	host := w.m.host
834	l := a.Last
835	if l == nil \|\| l.ID != w.id {
836	moveSay(w.m.out, moveNotRecorded)
837	return moveExitWaiting
838	}
839	from, to := w.fromRegion, w.toRegion
840	switch l.Outcome {
841	case "done":
842	if to == "" {
843	return w.outcomeLine(l, moveExitDone)
844	}
845	moveSay(w.m.out, moves.MovedLine(host, to, int(math.Round(l.OfflineS))))
846	return moveExitDone
847	case "rolled_back":
848	if from == "" {
849	return w.outcomeLine(l, moveExitRolled)
850	}
851	// M-C8 for a rollback the wait missed only when the outcome carries
852	// its reason (ruling 4), and only with its destination known.
853	if l.Reason != "" && to != "" && w.first("rollback") {
854	moveSay(w.m.out, moves.MovingBackLine(to, host, l.Reason, from))
855	}
856	moveSay(w.m.out, moves.MovedBackLine(host, from))
857	return moveExitRolled
858	case "aborted":
859	if from == "" {
860	return w.outcomeLine(l, moveExitRolled)
861	}
862	moveSay(w.m.out, fmt.Sprintf("The move was aborted. %s stays in %s.", host, from))
863	return moveExitRolled
864	}
865	moveSay(w.m.out, moveNotRecorded)
866	return moveExitWaiting
867	}
868
869	// outcomeLine says the outcome without a region, when the wait never learnt
870	// the move's own.
871	func (w moveWatch) outcomeLine(l moveOutcome, code int) int {
872	moveSay(w.m.out, fmt.Sprintf("Move %s of %s is %s.", l.ID, w.m.host, l.Outcome))
873	return code
874	}
875
876	// declareLost is carlos move resolve --declare-lost <box> (Task 1b): claim
877	// Plan 10's move door, inspect then declare, through declareHostLoss, the
878	// half carlos ops host resolve shares. The operator types the host name to
879	// confirm (Paul's ruling of 2026-10-10); the declaration carries the
880	// challenge's own ID. M-C18 is said for a move-target-lost inspection, with
881	// the move's flip time from detail=1; a destroyed-local-only one has its own
882	// sentence. The box is read through the operator's own EC2 credentials,
883	// never the console's.
884	func (m *mover) declareLost(ctx context.Context, box string, d moveDeps, wait bool) int {
885	a, code := m.readDetail(ctx)
886	if code >= 0 {
887	return code
888	}
889	read := a.Detail.Move // the move M-C18 describes
890	var flipped *time.Time
891	if read != nil {
892	flipped = read.FlippedAt
893	}
894	refused := -1 // the exit code a refusal already said, if any
895	stdin := bufio.NewReader(d.stdin)
896	door := hostLossDoor{
897	inspect: func(ctx context.Context) (hostresolve.Inspection, error) {
898	var in hostresolve.Inspection
899	r := m.send(ctx, http.MethodPost, "/resolve", map[string]string{"declare_lost": box})
900	if code := m.refusal(r); code >= 0 {
901	refused = code
902	return in, errHostLossNotConfirmed
903	}
904	if err := json.Unmarshal(r.body, &in); err != nil {
905	moveSay(m.out, moveUnreadable)
906	refused = moveExitError
907	return in, err
908	}
909	return in, nil
910	},
911	confirm: func(in hostresolve.Inspection) bool {
912	switch in.Details.Consequences.Transition {
913	case "move-target-lost":
914	// M-C18 describes the move read above: the challenge must
915	// name that move, and its incarnation when both carry one
916	// (fix round 1, finding 1). Otherwise nothing is shown or
917	// declared, and nothing is tried again.
918	if read == nil \|\| read.ID != in.Details.Op.ID \|\|
919	(read.Incarnation != "" && in.Details.HolderIncarnation != "" && read.Incarnation != in.Details.HolderIncarnation) {
920	moveSay(m.out, moveLossChanged)
921	return false
922	}
923	if flipped == nil {
924	moveSay(m.out, moveNoFlipTime)
925	return false
926	}
927	moveSay(m.out, fmt.Sprintf("This declares %s lost. Anything %s wrote on %s after %s is lost. Type the host name to confirm.", box, m.host, box, flipped.UTC().Format("2006-01-02 15:04:05 UTC")))
928	case "destroyed-local-only":
929	moveSay(m.out, fmt.Sprintf("This records that %s is gone with its disks. %s keeps nothing there that it still needs. Type the host name to confirm.", box, m.host))
930	default:
931	moveSay(m.out, moveOtherLoss)
932	return false
933	}
934	line, _ := stdin.ReadString('\n')
935	if strings.TrimSpace(line) != m.host {
936	moveSay(m.out, fmt.Sprintf("That is not %s. Nothing was declared.", m.host))
937	return false
938	}
939	return true
940	},
941	reader: d.ec2,
942	now: time.Now,
943	}
944	var declared moveReply
945	door.declare = func(ctx context.Context, sub hostfence.Submission) error {
946	declared = m.send(ctx, http.MethodPost, "/resolve", map[string]any{"declare_lost": box, "submission": sub})
947	return nil
948	}
949	stage, _ := declareHostLoss(ctx, door)
950	switch {
951	case refused >= 0:
952	return refused
953	case stage == hostLossConfirm:
954	return moveExitRefused
955	case stage == hostLossReader:
956	moveSay(m.out, moveNoAWS)
957	return moveExitRefused
958	case stage == hostLossObserve:
959	moveSay(m.out, fmt.Sprintf("AWS does not show %s stopped or terminated, so it cannot be declared lost.", box))
960	return moveExitRefused
961	}
962	return m.accepted(ctx, declared, wait, false)
963	}
964
965	// refusal says a reply that is not an acceptance and returns its exit code,
966	// or -1 for a 2xx.
967	func (m *mover) refusal(r moveReply) int {
968	switch {
969	case r.failed \|\| r.moved != "":
970	m.sayReply(r)
971	return moveExitError
972	case r.status >= 400 && r.status < 500:
973	m.sayReply(r)
974	return moveExitRefused
975	case r.status < 200 \|\| r.status > 299:
976	m.sayReply(r)
977	return moveExitError
978	}
979	return -1
980	}
981
982	// ---- carlos move offline ----
983	//
984	// An operator moves one instance with planned downtime, one step at a time:
985	// start, seal, flip, open, finish, abort, and status (offline-move plan,
986	// Task 7). stage comes before start when the operator wants the offline
987	// window short: it copies the instance's data to the target while it keeps
988	// serving, and can be run again. Each verb posts to its door, then polls the status door every
989	// interval until the step it started is done or has failed, and says which
990	// verb comes next. It never runs the next verb by itself. The exit codes are
991	// the family's: 0 the step is done, 3 the move was aborted, 4 the step failed
992	// or the move waits for the operator, 5 the console refused, 1 an error.
993	// Refusals are the console's own sentences (serverText); the CLI invents none.
994
995	// offlineVerbs are the eight verbs after `carlos move offline`.
996	var offlineVerbs = []string{"stage", "start", "seal", "flip", "open", "finish", "abort", "status"}
997
998	// offlineView is the offline status view (GET) and the 202 every mutating
999	// door answers with.
1000	type offlineView struct {
1001	Host string `json:"host"`
1002	State string `json:"state"`
1003	ID string `json:"id"`
1004	From string `json:"from"`
1005	To string `json:"to"`
1006	Sealer string `json:"sealer"`
1007	Boxes []string `json:"boxes"`
1008	Next string `json:"next"`
1009	CanAbort bool `json:"can_abort"`
1010	// NeverOffline: staged and never started, so an abort only removes
1011	// the staged copy.
1012	NeverOffline bool `json:"never_offline"`
1013	// StagedAt is the latest stage request (Unix nanoseconds), which
1014	// Copying.Answered equals once the stager has run its round.
1015	StagedAt int64 `json:"staged_at"`
1016	// Failed: a box's report says the step cannot finish (the status view's
1017	// own flag; the CLI says MV-O6 or MV-O9 for it).
1018	Failed bool `json:"failed"`
1019	Left []offlineLeftView `json:"left"`
1020	Reports []offlineReportView `json:"reports"`
1021	Items []offlineItemView `json:"items"`
1022	Last *moveOutcome `json:"last"`
1023	// Copying is the pre-copy's progress (the console's stage), nil when no
1024	// box has staged this move.
1025	Copying *offlineCopying `json:"stage"`
1026	}
1027
1028	// offlineCopying is the status view's stage: the stager's last round. The
1029	// fields are named apart from the tree's for the copy gate; Trouble is the
1030	// box's own diagnostic, not copy.
1031	type offlineCopying struct {
1032	Stager string `json:"box"`
1033	Phase string `json:"phase"`
1034	RoundAt int64 `json:"at"`
1035	Total int `json:"replica_objects"`
1036	TotalBytes int64 `json:"replica_bytes"`
1037	Done int `json:"staged_objects"`
1038	DoneBytes int64 `json:"staged_bytes"`
1039	Trouble string `json:"error"`
1040	Answered int64 `json:"request"`
1041	}
1042
1043	// megabytes is a byte count as a progress line says it.
1044	func megabytes(n int64) string { return fmt.Sprintf("%.1f", float64(n)/1e6) }
1045
1046	// sayStaged says the stage's last round: how much of the replica the
1047	// target holds and when (MV-O183), that the round failed (MV-O185), or
1048	// that none has run yet (MV-O186).
1049	func (w *offlineRun) sayStaged(v offlineView) {
1050	host, c := w.m.host, v.Copying
1051	if c == nil {
1052	moveSay(w.m.out, fmt.Sprintf("Waiting for the first staging round of %s.", host))
1053	return
1054	}
1055	if c.Phase == moves.OfflinePhaseFailed {
1056	moveSay(w.m.out, fmt.Sprintf("The last staging round failed: %s. Run carlos move offline stage --host %s --to %s to try again.", c.Trouble, host, v.To))
1057	return
1058	}
1059	at := time.Unix(c.RoundAt, 0).UTC().Format("2006-01-02 15:04 UTC")
1060	moveSay(w.m.out, fmt.Sprintf("Staged %d of %d objects for %s in %s (%s of %s MB) at %s.", c.Done, c.Total, host, v.To, megabytes(c.DoneBytes), megabytes(c.TotalBytes), at))
1061	}
1062
1063	// sayStartWhenReady is MV-O184: what comes after a stage.
1064	func (w *offlineRun) sayStartWhenReady(v offlineView) {
1065	host := w.m.host
1066	moveSay(w.m.out, fmt.Sprintf("%s keeps serving. Run carlos move offline start --host %s --to %s when you are ready to take it offline.", host, host, v.To))
1067	}
1068
1069	type offlineReportView struct {
1070	Box string `json:"box"`
1071	Role string `json:"role"`
1072	Phase string `json:"phase"`
1073	Fresh bool `json:"fresh"`
1074	Verdicts map[string]offlineVerdict `json:"items"`
1075	}
1076
1077	// offlineLeftView is a path a source box could not park and left in place,
1078	// with the reason it gave (a box diagnostic, not copy).
1079	type offlineLeftView struct {
1080	Holder string `json:"box"`
1081	Where string `json:"path"`
1082	Because string `json:"reason"`
1083	}
1084
1085	// offlineVerdict is one check of a report; its names are apart from the
1086	// tree's for the copy gate (see offlineItemView).
1087	type offlineVerdict struct {
1088	Passed bool `json:"ok"`
1089	Why string `json:"reason"`
1090	}
1091
1092	// The fields are named apart from the rest of the tree's (Detail, ID): the
1093	// copy gate reads a field by its name across the scanned files.
1094	type offlineItemView struct {
1095	Slug string `json:"id"`
1096	Owner string `json:"owner"`
1097	Check string `json:"status"`
1098	Diagnostic string `json:"detail"`
1099	}
1100
1101	// offlineReply is one answer of the offline doors: the view, and the sentence
1102	// the console says the next verb waits for (server text).
1103	type offlineReply struct {
1104	v offlineView
1105	waiting serverText
1106	hasWaiting bool
1107	}
1108
1109	// report is the box's report of a role, if it wrote one.
1110	func (v offlineView) report(role, box string) *offlineReportView {
1111	for i := range v.Reports {
1112	if v.Reports[i].Role == role && v.Reports[i].Box == box {
1113	return &v.Reports[i]
1114	}
1115	}
1116	return nil
1117	}
1118
1119	// failed names the items an owner's report failed, by the box's own
1120	// diagnostic (or the item's name when it gave none), joined. Empty when none
1121	// failed.
1122	func (v offlineView) failed(owner string) string {
1123	var parts []string
1124	for _, it := range v.Items {
1125	if it.Owner != owner \|\| it.Check != "failed" {
1126	continue
1127	}
1128	if it.Diagnostic == "" {
1129	parts = append(parts, it.Slug)
1130	continue
1131	}
1132	parts = append(parts, it.Diagnostic)
1133	}
1134	return strings.Join(parts, "; ")
1135	}
1136
1137	// failedReport is the role's report when it says it failed.
1138	func (v offlineView) failedReport(role string) *offlineReportView {
1139	for i := range v.Reports {
1140	if v.Reports[i].Role == role && v.Reports[i].Phase == moves.OfflinePhaseFailed {
1141	return &v.Reports[i]
1142	}
1143	}
1144	return nil
1145	}
1146
1147	// reason is what a failed report says: its failing checks, else its phase.
1148	func (r offlineReportView) reason() string {
1149	var parts []string
1150	for _, id := range slices.Sorted(maps.Keys(r.Verdicts)) {
1151	c := r.Verdicts[id]
1152	switch {
1153	case c.Passed:
1154	case c.Why != "":
1155	parts = append(parts, c.Why)
1156	default:
1157	parts = append(parts, id)
1158	}
1159	}
1160	if len(parts) > 0 {
1161	return strings.Join(parts, "; ")
1162	}
1163	return r.Phase
1164	}
1165
1166	// The roles a failed step can be: the sealer sealing, the target landing,
1167	// or a source box cleaning up after the open.
1168	const (
1169	failSealer = "sealer"
1170	failTarget = "target"
1171	failSource = "source"
1172	)
1173
1174	// failure says whether a box has failed its step, which role it was and why:
1175	// the items the role's report failed, else what its failed report says, else
1176	// the word failed. Before the open it is the target landing, else the sealer
1177	// sealing (both at once go by the state the move is in). Once the move is
1178	// open or finishing a failed report of any role counts, source boxes included;
1179	// while it reverts only a failed source box does, since a target or sealer
1180	// that failed earlier is being put back.
1181	func (v offlineView) failure() (kind, reason string, ok bool) {
1182	switch v.State {
1183	case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:
1184	roles := []string{moves.OfflineRoleSealer, moves.OfflineRoleTarget, moves.OfflineRoleSource}
1185	if v.State == provision.OfflineReverting {
1186	roles = roles[2:]
1187	}
1188	for _, role := range roles {
1189	if r := v.failedReport(role); r != nil {
1190	return role, r.reason(), true
1191	}
1192	}
1193	return "", "", false
1194	}
1195	sealing := v.State == provision.OfflineStopping \|\| v.State == provision.OfflineSealing \|\| v.State == provision.OfflineSealed
1196	sealReason, landReason := v.failed(moves.OfflineOwnerSeal), v.failed(moves.OfflineOwnerLanding)
1197	sealRep, landRep := v.failedReport(moves.OfflineRoleSealer), v.failedReport(moves.OfflineRoleTarget)
1198	sealFailed := sealRep != nil \|\| (sealing && sealReason != "")
1199	landFailed := landRep != nil \|\| (!sealing && landReason != "")
1200	switch {
1201	case sealFailed && (sealing \|\| !landFailed):
1202	if sealReason == "" && sealRep != nil {
1203	sealReason = sealRep.reason()
1204	}
1205	return failSealer, sealReason, true
1206	case landFailed:
1207	if landReason == "" && landRep != nil {
1208	landReason = landRep.reason()
1209	}
1210	return failTarget, landReason, true
1211	case v.Failed:
1212	if sealing {
1213	return failSealer, moves.OfflinePhaseFailed, true
1214	}
1215	return failTarget, moves.OfflinePhaseFailed, true
1216	}
1217	return "", "", false
1218	}
1219
1220	// sayFailure says a failed step: MV-O6 for the target before the open,
1221	// MV-O9 for the sealer or a target once open (abort refuses there), MV-O65
1222	// for a source box.
1223	func (w *offlineRun) sayFailure(v offlineView) bool {
1224	kind, reason, ok := v.failure()
1225	if !ok {
1226	return false
1227	}
1228	host := w.m.host
1229	if kind == failTarget && (v.State == provision.OfflineOpen \|\| v.State == provision.OfflineFinishing) {
1230	// Once the move is open, abort refuses (MV-O27), so MV-O6's advice
1231	// would send the operator to a verb that says no; MV-O9 says it.
1232	kind = failSealer
1233	}
1234	switch kind {
1235	case failTarget:
1236	moveSay(w.m.out, fmt.Sprintf("%s could not take %s: %s. Run carlos move offline abort --host %s to move it back.", v.To, host, reason, host))
1237	case failSource:
1238	for _, r := range v.Reports {
1239	if r.Role == moves.OfflineRoleSource && r.Phase == moves.OfflinePhaseFailed {
1240	moveSay(w.m.out, fmt.Sprintf("%s could not clean up %s: %s. Run carlos move offline status --host %s for each item.", r.Box, host, reason, host))
1241	return true
1242	}
1243	}
1244	default:
1245	moveSay(w.m.out, fmt.Sprintf("The move could not finish a step: %s. Run carlos move offline status --host %s for each item.", reason, host))
1246	}
1247	return true
1248	}
1249
1250	// offlineRun is one `carlos move offline` verb's conversation: the move it
1251	// follows, as the console's answers named it, and what has been said.
1252	type offlineRun struct {
1253	m *mover
1254	id, from, to string
1255	said map[string]bool
1256	lastWaiting serverText
1257	}
1258
1259	func (w *offlineRun) learn(v offlineView) {
1260	if v.ID != "" && (w.id == "" \|\| w.id == v.ID) {
1261	w.id, w.from, w.to = v.ID, v.From, v.To
1262	}
1263	}
1264
1265	func (w *offlineRun) once(key string) bool {
1266	if w.said[key] {
1267	return false
1268	}
1269	w.said[key] = true
1270	return true
1271	}
1272
1273	// read decodes a 2xx reply from an offline door.
1274	func (w *offlineRun) read(r moveReply) (offlineReply, int) {
1275	var v offlineView
1276	if json.Unmarshal(r.body, &v) != nil {
1277	moveSay(w.m.out, moveUnreadable)
1278	return offlineReply{}, moveExitError
1279	}
1280	w.learn(v)
1281	return offlineReply{v: v, waiting: r.answer.waiting, hasWaiting: r.answer.hasWaiting}, -1
1282	}
1283
1284	// poll reads the status door.
1285	func (w *offlineRun) poll(ctx context.Context) (offlineReply, int) {
1286	r, code := w.m.getReply(ctx, w.m.path)
1287	if code >= 0 {
1288	return offlineReply{}, code
1289	}
1290	return w.read(r)
1291	}
1292
1293	// post sends a verb's door. A refusal is said as the console wrote it.
1294	func (w *offlineRun) post(ctx context.Context, tail string, body any) (offlineReply, int) {
1295	r := w.m.send(ctx, http.MethodPost, tail, body)
1296	if code := w.m.refusal(r); code >= 0 {
1297	return offlineReply{}, code
1298	}
1299	return w.read(r)
1300	}
1301
1302	// follow polls until step says the verb's step has ended, and returns the
1303	// exit code it gave.
1304	func (w *offlineRun) follow(ctx context.Context, cur offlineReply, step func(offlineReply) (int, bool)) int {
1305	for {
1306	if code, done := step(cur); done {
1307	return code
1308	}
1309	if !w.m.sleep(ctx) {
1310	moveSay(w.m.out, moveStopped)
1311	return moveExitError
1312	}
1313	var code int
1314	if cur, code = w.poll(ctx); code >= 0 {
1315	return code
1316	}
1317	}
1318	}
1319
1320	// sayWaiting says the console's sentence for what the next verb waits for,
1321	// each time it changes.
1322	func (w *offlineRun) sayWaiting(r offlineReply) {
1323	if r.hasWaiting && r.waiting != w.lastWaiting {
1324	saySpoken(w.m.out, r.waiting)
1325	w.lastWaiting = r.waiting
1326	}
1327	}
1328
1329	// sayCleaning names, once each, the boxes the finish waits for.
1330	func (w *offlineRun) sayCleaning(v offlineView) {
1331	var source, sealer string
1332	switch v.State {
1333	case provision.OfflineOpen:
1334	source = moves.OfflinePhaseParked
1335	case provision.OfflineFinishing:
1336	source, sealer = moves.OfflinePhaseGone, moves.OfflinePhaseLifted
1337	case provision.OfflineReverting:
1338	source, sealer = moves.OfflinePhaseReverted, moves.OfflinePhaseReverted
1339	default:
1340	return
1341	}
1342	var boxes []string
1343	for _, b := range v.Boxes {
1344	if r := v.report(moves.OfflineRoleSource, b); r == nil \|\| r.Phase != source {
1345	boxes = append(boxes, b)
1346	}
1347	}
1348	if sealer != "" && v.Sealer != "" {
1349	if r := v.report(moves.OfflineRoleSealer, v.Sealer); r == nil \|\| r.Phase != sealer {
1350	boxes = append(boxes, v.Sealer)
1351	}
1352	}
1353	for _, b := range boxes {
1354	if w.once("clean:" + v.State + ":" + b) {
1355	moveSay(w.m.out, fmt.Sprintf("Waiting for %s to clean up %s.", b, w.m.host))
1356	}
1357	}
1358	}
1359
1360	// ended says the end of the move this run followed, from the console's
1361	// `last`, which counts only when it is that move's own.
1362	func (w *offlineRun) ended(v offlineView) int {
1363	host := w.m.host
1364	l := v.Last
1365	if w.id == "" \|\| l == nil \|\| l.ID != w.id {
1366	moveSay(w.m.out, moveNotRecorded)
1367	return moveExitWaiting
1368	}
1369	switch l.Outcome {
1370	case "done":
1371	if w.to == "" {
1372	offlineOutcomeLine(w.m, l)
1373	return moveExitDone
1374	}
1375	moveSay(w.m.out, moves.MovedLine(host, w.to, int(math.Round(l.OfflineS))))
1376	return moveExitDone
1377	case "aborted":
1378	if w.from == "" {
1379	offlineOutcomeLine(w.m, l)
1380	return moveExitRolled
1381	}
1382	moveSay(w.m.out, moves.MovedBackLine(host, w.from))
1383	return moveExitRolled
1384	}
1385	moveSay(w.m.out, moveNotRecorded)
1386	return moveExitWaiting
1387	}
1388
1389	// offlineOutcomeLine says an outcome without the regions: the move, the host
1390	// and how it ended.
1391	func offlineOutcomeLine(m mover, l moveOutcome) {
1392	moveSay(m.out, fmt.Sprintf("Move %s of %s is %s.", l.ID, m.host, l.Outcome))
1393	}
1394
1395	// served reports that the target serves the instance: its report says
1396	// opened, the move has gone on to finishing, or the next step is finish.
1397	func (w *offlineRun) served(v offlineView) bool {
1398	if v.State == provision.OfflineFinishing \|\| v.Next == "finish" {
1399	return true
1400	}
1401	for _, r := range v.Reports {
1402	if r.Role == moves.OfflineRoleTarget && r.Phase == moves.OfflinePhaseOpened {
1403	return true
1404	}
1405	}
1406	return false
1407	}
1408
1409	// sayServed is MV-O7, said once.
1410	func (w *offlineRun) sayServed(v offlineView) {
1411	if w.once("served") {
1412	host := w.m.host
1413	moveSay(w.m.out, fmt.Sprintf("%s is served from %s. Run carlos move offline finish --host %s once %s has cleaned up.", host, v.To, host, v.From))
1414	}
1415	}
1416
1417	// headline says where a move in flight stands, in the verbs' own sentences.
1418	func (w *offlineRun) headline(r offlineReply) {
1419	v, host := r.v, w.m.host
1420	if v.State == provision.OfflineStaging {
1421	moveSay(w.m.out, fmt.Sprintf("Staging %s in %s. %s keeps serving while the data is copied.", host, v.To, host))
1422	w.sayStaged(v)
1423	if v.Next == "start" {
1424	w.sayStartWhenReady(v)
1425	return
1426	}
1427	w.sayWaiting(r)
1428	return
1429	}
1430	moveSay(w.m.out, moves.MoveStartingLine(host, v.From, v.To))
1431	if w.sayFailure(v) {
1432	return
1433	}
1434	switch v.State {
1435	case provision.OfflineStopping:
1436	if v.Next == "seal" {
1437	moveSay(w.m.out, fmt.Sprintf("Every box in %s has stopped %s. Run carlos move offline seal --host %s to copy it.", v.From, host, host))
1438	return
1439	}
1440	w.sayWaiting(r)
1441	case provision.OfflineSealing:
1442	moveSay(w.m.out, fmt.Sprintf("Copying %s to %s.", host, v.To))
1443	case provision.OfflineSealed:
1444	if v.Next == "flip" {
1445	moveSay(w.m.out, fmt.Sprintf("Copied and checked. Run carlos move offline flip --host %s to start it in %s.", host, v.To))
1446	return
1447	}
1448	w.sayWaiting(r)
1449	case provision.OfflineLanding:
1450	moveSay(w.m.out, moves.FlippedLine(v.From, v.To))
1451	case provision.OfflineLanded:
1452	if v.Next == "open" {
1453	moveSay(w.m.out, fmt.Sprintf("%s holds everything %s needs. Nothing is served yet. Run carlos move offline open --host %s to serve it from %s.", v.To, host, host, v.To))
1454	return
1455	}
1456	w.sayWaiting(r)
1457	case provision.OfflineOpen, provision.OfflineFinishing:
1458	if w.served(v) {
1459	w.sayServed(v)
1460	} else {
1461	w.sayWaiting(r)
1462	}
1463	w.sayCleaning(v)
1464	case provision.OfflineReverting:
1465	moveSay(w.m.out, fmt.Sprintf("Moving %s back to %s.", host, v.From))
1466	w.sayCleaning(v)
1467	if v.Next == "finish" {
1468	moveSay(w.m.out, fmt.Sprintf("Every box has put %s back in %s. Run carlos move offline finish --host %s to end the move.", host, v.From, host))
1469	}
1470	}
1471	}
1472
1473	// offline runs one `carlos move offline` verb.
1474	func (m *mover) offline(ctx context.Context, a moveArgs) int {
1475	w := &offlineRun{m: m, said: map[string]bool{}}
1476	switch a.offline {
1477	case "status":
1478	return w.runStatus(ctx)
1479	case "stage":
1480	return w.runStage(ctx, a.to)
1481	case "start":
1482	return w.runStart(ctx, a.to)
1483	case "seal":
1484	return w.runSeal(ctx)
1485	case "flip":
1486	return w.runFlip(ctx)
1487	case "open":
1488	return w.runOpen(ctx)
1489	case "finish":
1490	return w.runFinish(ctx)
1491	default: // abort
1492	return w.runAbort(ctx)
1493	}
1494	}
1495
1496	// stray is a view the verb's step did not expect: a move that is no longer
1497	// in flight says how it ended; any other state is said where it stands, and
1498	// the operator decides.
1499	func (w *offlineRun) stray(r offlineReply) int {
1500	if r.v.State == "none" {
1501	return w.ended(r.v)
1502	}
1503	w.headline(r)
1504	return moveExitWaiting
1505	}
1506
1507	func (w *offlineRun) runStatus(ctx context.Context) int {
1508	r, code := w.poll(ctx)
1509	if code >= 0 {
1510	return code
1511	}
1512	v, host := r.v, w.m.host
1513	if v.State == "none" {
1514	if v.Last != nil {
1515	offlineOutcomeLine(w.m, v.Last)
1516	} else {
1517	moveSay(w.m.out, fmt.Sprintf("No move is in flight for %s.", host))
1518	}
1519	return moveExitDone
1520	}
1521	w.headline(r)
1522	if v.State == provision.OfflineStaging {
1523	// Nothing of the manifest is checked before the start, so the
1524	// items would all say waiting. The stage's own line said it.
1525	if v.Copying != nil && v.Copying.Phase == moves.OfflinePhaseFailed {
1526	return moveExitWaiting
1527	}
1528	return moveExitDone
1529	}
1530	failed := false
1531	for _, it := range v.Items {
1532	if it.Diagnostic == "" {
1533	moveSay(w.m.out, fmt.Sprintf("Item %s is %s.", it.Slug, it.Check))
1534	} else {
1535	moveSay(w.m.out, fmt.Sprintf("Item %s is %s: %s.", it.Slug, it.Check, it.Diagnostic))
1536	}
1537	failed = failed \|\| it.Check == "failed"
1538	}
1539	for _, rep := range v.Reports {
1540	if rep.Fresh {
1541	moveSay(w.m.out, fmt.Sprintf("%s is %s as the %s.", rep.Box, rep.Phase, rep.Role))
1542	} else {
1543	moveSay(w.m.out, fmt.Sprintf("%s is %s as the %s. Its report is not current.", rep.Box, rep.Phase, rep.Role))
1544	}
1545	}
1546	for _, l := range v.Left {
1547	moveSay(w.m.out, fmt.Sprintf("%s left %s in place: %s.", l.Holder, l.Where, l.Because))
1548	}
1549	if _, _, stepFailed := v.failure(); failed \|\| v.Failed \|\| stepFailed {
1550	return moveExitWaiting
1551	}
1552	return moveExitDone
1553	}
1554
1555	// runStage posts the stage door, then waits for the round that answers the
1556	// stage request the post made (or a later one's, if another terminal ran
1557	// stage meanwhile).
1558	func (w *offlineRun) runStage(ctx context.Context, to string) int {
1559	r, code := w.post(ctx, "/stage", map[string]string{"region": to})
1560	if code >= 0 {
1561	return code
1562	}
1563	host := w.m.host
1564	if r.v.State != provision.OfflineStaging {
1565	return w.stray(r)
1566	}
1567	moveSay(w.m.out, fmt.Sprintf("Staging %s in %s. %s keeps serving while the data is copied.", host, r.v.To, host))
1568	asked := r.v.StagedAt
1569	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1570	if r.v.State != provision.OfflineStaging {
1571	return w.stray(r), true
1572	}
1573	c := r.v.Copying
1574	if c == nil \|\| c.Answered < asked \|\| c.Answered != r.v.StagedAt {
1575	return 0, false
1576	}
1577	w.sayStaged(r.v)
1578	if c.Phase == moves.OfflinePhaseFailed {
1579	return moveExitWaiting, true
1580	}
1581	w.sayStartWhenReady(r.v)
1582	return moveExitDone, true
1583	})
1584	}
1585
1586	func (w *offlineRun) runStart(ctx context.Context, to string) int {
1587	r, code := w.post(ctx, "", map[string]string{"region": to})
1588	if code >= 0 {
1589	return code
1590	}
1591	host := w.m.host
1592	moveSay(w.m.out, moves.MoveStartingLine(host, r.v.From, r.v.To))
1593	moveSay(w.m.out, fmt.Sprintf("%s is offline from now until the move opens or is aborted.", host))
1594	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1595	switch r.v.State {
1596	case provision.OfflineStopping:
1597	if r.v.Next == "seal" {
1598	moveSay(w.m.out, fmt.Sprintf("Every box in %s has stopped %s. Run carlos move offline seal --host %s to copy it.", r.v.From, host, host))
1599	return moveExitDone, true
1600	}
1601	w.sayWaiting(r)
1602	return 0, false
1603	}
1604	return w.stray(r), true
1605	})
1606	}
1607
1608	func (w *offlineRun) runSeal(ctx context.Context) int {
1609	r, code := w.post(ctx, "/seal", struct{}{})
1610	if code >= 0 {
1611	return code
1612	}
1613	host := w.m.host
1614	moveSay(w.m.out, fmt.Sprintf("Copying %s to %s.", host, r.v.To))
1615	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1616	switch r.v.State {
1617	case provision.OfflineSealing, provision.OfflineSealed:
1618	if w.sayFailure(r.v) {
1619	return moveExitWaiting, true
1620	}
1621	if r.v.State == provision.OfflineSealed {
1622	if r.v.Next == "flip" {
1623	moveSay(w.m.out, fmt.Sprintf("Copied and checked. Run carlos move offline flip --host %s to start it in %s.", host, r.v.To))
1624	return moveExitDone, true
1625	}
1626	w.sayWaiting(r)
1627	}
1628	return 0, false
1629	}
1630	return w.stray(r), true
1631	})
1632	}
1633
1634	func (w *offlineRun) runFlip(ctx context.Context) int {
1635	r, code := w.post(ctx, "/flip", struct{}{})
1636	if code >= 0 {
1637	return code
1638	}
1639	host := w.m.host
1640	moveSay(w.m.out, moves.FlippedLine(r.v.From, r.v.To))
1641	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1642	switch r.v.State {
1643	case provision.OfflineLanding, provision.OfflineLanded:
1644	if w.sayFailure(r.v) {
1645	return moveExitWaiting, true
1646	}
1647	if r.v.State == provision.OfflineLanded {
1648	if r.v.Next == "open" {
1649	moveSay(w.m.out, fmt.Sprintf("%s holds everything %s needs. Nothing is served yet. Run carlos move offline open --host %s to serve it from %s.", r.v.To, host, host, r.v.To))
1650	return moveExitDone, true
1651	}
1652	w.sayWaiting(r)
1653	}
1654	return 0, false
1655	}
1656	return w.stray(r), true
1657	})
1658	}
1659
1660	func (w *offlineRun) runOpen(ctx context.Context) int {
1661	r, code := w.post(ctx, "/open", struct{}{})
1662	if code >= 0 {
1663	return code
1664	}
1665	// The line is said once the target serves (its report says opened, or the
1666	// next step is finish), not when the door only accepted the step.
1667	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1668	if r.v.State == provision.OfflineOpen {
1669	if w.sayFailure(r.v) {
1670	return moveExitWaiting, true
1671	}
1672	if w.served(r.v) {
1673	w.sayServed(r.v)
1674	} else {
1675	w.sayWaiting(r)
1676	}
1677	if r.v.Next == "finish" {
1678	return moveExitDone, true
1679	}
1680	if w.sayFailure(r.v) {
1681	return moveExitWaiting, true
1682	}
1683	w.sayCleaning(r.v)
1684	return 0, false
1685	}
1686	return w.stray(r), true
1687	})
1688	}
1689
1690	// finish posts until the move is cleared. The first post (from open) starts
1691	// the clean-up and the console answers offline-finishing; once the boxes have
1692	// cleaned up and the tombstone is lifted, the status shows the next verb is
1693	// finish again and the second post ends the move. A move reverting from an
1694	// abort needs only the one.
1695	func (w *offlineRun) runFinish(ctx context.Context) int {
1696	cur, code := w.poll(ctx)
1697	if code >= 0 {
1698	return code
1699	}
1700	for {
1701	// Wait until the door would accept: the boxes' clean-up shows in the
1702	// status before the post, so the door is never asked early.
1703	switch cur.v.State {
1704	case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:
1705	if cur.v.Next != "finish" {
1706	code := w.follow(ctx, cur, func(r offlineReply) (int, bool) {
1707	switch r.v.State {
1708	case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:
1709	if r.v.Next == "finish" {
1710	cur = r
1711	return -1, true
1712	}
1713	if w.sayFailure(r.v) {
1714	return moveExitWaiting, true
1715	}
1716	w.sayCleaning(r.v)
1717	return 0, false
1718	}
1719	cur = r
1720	return -1, true
1721	})
1722	if code >= 0 {
1723	return code
1724	}
1725	}
1726	}
1727	prev := cur.v.State
1728	r, code := w.post(ctx, "/finish", struct{}{})
1729	if code >= 0 {
1730	return code
1731	}
1732	if r.v.State == "none" {
1733	return w.ended(r.v)
1734	}
1735	if r.v.State == prev {
1736	return w.stray(r)
1737	}
1738	cur = r
1739	}
1740	}
1741
1742	func (w *offlineRun) runAbort(ctx context.Context) int {
1743	r, code := w.post(ctx, "/abort", struct{}{})
1744	if code >= 0 {
1745	return code
1746	}
1747	// A move aborted while staging was never offline: there is nothing to
1748	// move back, only the staged copy to remove.
1749	if r.v.NeverOffline {
1750	return w.abortStaged(ctx, r)
1751	}
1752	host := w.m.host
1753	moveSay(w.m.out, fmt.Sprintf("Moving %s back to %s.", host, r.v.From))
1754	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1755	if r.v.State == provision.OfflineReverting {
1756	if r.v.Next == "finish" {
1757	moveSay(w.m.out, fmt.Sprintf("Every box has put %s back in %s. Run carlos move offline finish --host %s to end the move.", host, r.v.From, host))
1758	return moveExitRolled, true
1759	}
1760	w.sayCleaning(r.v)
1761	return 0, false
1762	}
1763	return w.stray(r), true
1764	})
1765	}
1766
1767	// abortStaged follows an abort from staging: the stager removes the staged
1768	// copy, every source box reports, and the operator runs finish.
1769	func (w *offlineRun) abortStaged(ctx context.Context, r offlineReply) int {
1770	host := w.m.host
1771	moveSay(w.m.out, fmt.Sprintf("Removing the staged copy of %s from %s.", host, r.v.To))
1772	return w.follow(ctx, r, func(r offlineReply) (int, bool) {
1773	if r.v.State == provision.OfflineReverting {
1774	if r.v.Next == "finish" {
1775	moveSay(w.m.out, fmt.Sprintf("The staged copy of %s is removed. Run carlos move offline finish --host %s to end the move.", host, host))
1776	return moveExitRolled, true
1777	}
1778	return 0, false
1779	}
1780	return w.stray(r), true
1781	})
1782	}
1783