carlos / platform Public

Download this file

1	package console
2
3	import (
4	"context"
5	"encoding/json"
6	"errors"
7	"fmt"
8	"net/http"
9	"reflect"
10	"strings"
11	"testing"
12
13	"time"
14
15	"github.com/carlosframework/platform/internal/hostactors"
16	"github.com/carlosframework/platform/internal/hostclaim"
17	"github.com/carlosframework/platform/internal/moves"
18	"github.com/carlosframework/platform/internal/objstore"
19	"github.com/carlosframework/platform/internal/provision"
20	"github.com/carlosframework/platform/internal/release"
21	"github.com/carlosframework/platform/internal/svcrole"
22	)
23
24	// The residence-move doors (issue #331): the console writes guarded
25	// transitions and serial bumps; the state machine's 400/409 matrix is the
26	// tenant's whole interface, so every cell is pinned.
27	func TestAPIInstanceMoveVerbs(t *testing.T) {
28	fx, token := instanceFixture(t, "code-inst-move")
29	m := titoManifest()
30	m.Regions = nil
31	m.Pool = "" // pooled records are refused at the move door — separate rail, pinned below
32	enableInstances(t, fx, "titogo", m)
33	u := instancesURL(fx, "bab", "titogo")
34	put := func(key, body string) {
35	t.Helper()
36	if _, err := fx.rel.Put(context.Background(), key, []byte(body), ""); err != nil {
37	t.Fatal(err)
38	}
39	}
40	// Create BEFORE arming: once uk-box publishes a placement, the #329
41	// gate would refuse region ie (not armed) — the record predates arming,
42	// exactly like every real record on a fleet that grows regions later.
43	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
44	t.Fatalf("create = %d %s", code, out)
45	}
46	put("control/edges/uk-box.json", `{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","data_bucket":"data-uk","data_bucket_region":"eu-west-2","hosts":{}}`)
47	moveURL := u + "/acme.go.tito.io/move"
48
49	// A target with no staging bucket → 400 naming the set that can take one.
50	// The move door's bar is the target's published data_bucket, because
51	// that is exactly what internal/host's moveTargets demands before a move
52	// can even be prepared.
53	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"se"}`); code != http.StatusBadRequest || !strings.Contains(out, "Regions that can: uk") {
54	t.Fatalf("move to a region with no staging bucket = %d %s, want 400 naming the set", code, out)
55	}
56	// Same region → 409 (the transition's own conflict).
57	if code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"ie"}`); code != http.StatusBadRequest && code != http.StatusConflict {
58	t.Fatalf("move to current region = %d %s, want a refusal", code, out)
59	} else if code == http.StatusBadRequest && !strings.Contains(out, "armed") {
60	// "ie" is not armed either — refused at the armed gate, also fine.
61	_ = out
62	}
63	// Cut before any move → 409.
64	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict {
65	t.Fatalf("cut with no move = %d %s, want 409", code, out)
66	}
67	// The poll door answers "none" before any move — 200, never a 404.
68	code, _, out := rawReq(t, http.MethodGet, moveURL, token, "")
69	if code != http.StatusOK || !strings.Contains(out, `"state":"none"`) {
70	t.Fatalf("move status with no move = %d %s, want 200 none", code, out)
71	}
72	// Happy path: request → the FLAT view (titogo's consumer contract):
73	// state, target, updated_at — never the record.
74	code, _, out = rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`)
75	if code != http.StatusAccepted {
76	t.Fatalf("move request = %d %s, want 202", code, out)
77	}
78	var mv struct {
79	State string `json:"state"`
80	Target string `json:"target"`
81	UpdatedAt int64 `json:"updated_at"`
82	}
83	if err := json.Unmarshal([]byte(out), &mv); err != nil {
84	t.Fatal(err)
85	}
86	if mv.State != provision.MovePreparing || mv.Target != "uk" || mv.UpdatedAt == 0 {
87	t.Fatalf("after request: %+v", mv)
88	}
89	// The record itself still carries the nested truth (and region ie).
90	if code, _, out := rawReq(t, http.MethodGet, u+"/acme.go.tito.io", token, ""); code != http.StatusOK || !strings.Contains(out, `"region":"ie"`) || !strings.Contains(out, provision.MovePreparing) {
91	t.Fatalf("record after request: %d %s", code, out)
92	}
93	// The poll door mirrors it.
94	if code, _, out := rawReq(t, http.MethodGet, moveURL, token, ""); code != http.StatusOK || !strings.Contains(out, `"target":"uk"`) {
95	t.Fatalf("move status mid-move = %d %s", code, out)
96	}
97	// Second request while in flight → 409.
98	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict {
99	t.Fatalf("second move = %d %s, want 409", code, out)
100	}
101	// Cut from preparing → 409 (only ready cuts).
102	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict {
103	t.Fatalf("cut from preparing = %d %s, want 409", code, out)
104	}
105	// Abort from preparing → 202 aborting (two-phase: the SOURCE box
106	// deletes staging, then clears the field).
107	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) {
108	t.Fatalf("abort = %d %s, want 202 aborting", code, out)
109	}
110	// A second abort while aborting → 409; a new request while aborting →
111	// 409 (still in flight).
112	if code, _, _ := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict {
113	t.Fatalf("double abort = %d, want 409", code)
114	}
115	if code, _, _ := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict {
116	t.Fatalf("request while aborting = %d, want 409", code)
117	}
118	// The box's half clears it; then a fresh request proceeds.
119	if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveAbortComplete()); err != nil {
120	t.Fatal(err)
121	}
122	code, _, out = rawReq(t, http.MethodGet, u+"/acme.go.tito.io", token, "")
123	if code != http.StatusOK || strings.Contains(out, `"move"`) {
124	t.Fatalf("after abort-complete: %d %s, want no move key", code, out)
125	}
126	if code, _, _ := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted {
127	t.Fatal("re-request")
128	}
129	if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MovePromoteReady(time.Now())); err != nil {
130	t.Fatal(err)
131	}
132	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusAccepted {
133	t.Fatalf("cut from ready = %d %s, want 202", code, out)
134	}
135	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict {
136	t.Fatalf("abort from cutting = %d %s, want 409 (completes forward)", code, out)
137	}
138	// The serial was bumped by the successful writes.
139	if _, _, err := fx.rel.Get(context.Background(), "control/serial"); err != nil {
140	t.Fatalf("control/serial not written: %v", err)
141	}
142	// 404 discipline: unknown host is the byte-identical not-found.
143	if code, _, _ := rawReq(t, http.MethodPost, u+"/ghost.go.tito.io/move", token, `{"region":"uk"}`); code != http.StatusNotFound {
144	t.Fatalf("move on unknown host = %d, want 404", code)
145	}
146	}
147
148	// The class rails: only an exec-backed instance route moves.
149	func TestAPIInstanceMoveClassRails(t *testing.T) {
150	fx, token := instanceFixture(t, "code-inst-move-rails")
151	letStateUpstreams(t, fx, "alice@keymail.dev")
152	m := titoManifest()
153	m.Regions = nil
154	m.Pool = ""
155	enableInstances(t, fx, "titogo", m)
156	u := instancesURL(fx, "bab", "titogo")
157	if _, err := fx.rel.Put(context.Background(), "control/edges/uk-box.json",
158	[]byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`), ""); err != nil {
159	t.Fatal(err)
160	}
161	mk := func(body string) string {
162	t.Helper()
163	code, _, out := rawReq(t, http.MethodPost, u, token, body)
164	if code != http.StatusCreated {
165	t.Fatalf("create %s = %d %s", body, code, out)
166	}
167	var rec provision.Record
168	if err := json.Unmarshal([]byte(out), &rec); err != nil {
169	t.Fatal(err)
170	}
171	return rec.Host
172	}
173	for _, tc := range []struct {
174	name, body, wantIn string
175	}{
176	{"static", `{"host":"st.go.tito.io","kind":"static"}`, "kind static"},
177	{"addr-backed", `{"host":"ad.go.tito.io","addrs":["10.0.0.1:80"],"kind":"service"}`, "kind service"},
178	} {
179	t.Run(tc.name, func(t *testing.T) {
180	host := mk(tc.body)
181	code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`)
182	if code != http.StatusBadRequest || !strings.Contains(out, tc.wantIn) {
183	t.Fatalf("move %s = %d %s, want 400 mentioning %q", tc.name, code, out, tc.wantIn)
184	}
185	})
186	}
187	// Fleet-placed: region carries fleet/<name>.
188	host := mk(`{"host":"fl.go.tito.io"}`)
189	if err := provision.SetRecordRegion(context.Background(), fx.rel, "bab", "titogo", host, "fleet/pi"); err != nil {
190	t.Fatal(err)
191	}
192	code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`)
193	if code != http.StatusBadRequest || !strings.Contains(out, "fleet") {
194	t.Fatalf("move fleet-placed = %d %s, want 400 mentioning fleet", code, out)
195	}
196	_ = fmt.Sprint()
197	_ = provisionPooledMoveRefusal(t, fx, token, u)
198
199	// A replica (G14): refused at the request, while the status door still
200	// answers, since the gate it shares must not strand a replica's move.
201	rep := provision.Record{Schema: 1, Host: "rep.go.tito.io", Slug: "rep", App: "titogo",
202	Account: "bab", Region: "ie", Role: "replica", Channel: "stable", CreatedAt: time.Now().UTC()}
203	if err := provision.CreateRecord(context.Background(), fx.rel, rep); err != nil {
204	t.Fatal(err)
205	}
206	code, _, out = rawReq(t, http.MethodPost, u+"/rep.go.tito.io/move", token, `{"region":"uk"}`)
207	if code != http.StatusBadRequest || !strings.Contains(out, "replica") {
208	t.Fatalf("move replica = %d %s, want 400 mentioning the replica rail", code, out)
209	}
210	if code, _, out = rawReq(t, http.MethodGet, u+"/rep.go.tito.io/move", token, ""); code != http.StatusOK {
211	t.Fatalf("move status of a replica = %d %s, want 200", code, out)
212	}
213	}
214
215	// provisionPooledMoveRefusal pins the pool rail: a pooled record's data
216	// lives on the pool's box, and v1 refuses the class at the door.
217	func provisionPooledMoveRefusal(t *testing.T, fx *consoleFixture, token, u string) bool {
218	t.Helper()
219	rec := provision.Record{Schema: 1, Host: "pooled.go.tito.io", Slug: "pooled", App: "titogo",
220	Account: "bab", Pool: "dedicated-x", Channel: "stable", CreatedAt: time.Now().UTC()}
221	if err := provision.CreateRecord(context.Background(), fx.rel, rec); err != nil {
222	t.Fatal(err)
223	}
224	code, _, out := rawReq(t, http.MethodPost, u+"/pooled.go.tito.io/move", token, `{"region":"uk"}`)
225	if code != http.StatusBadRequest || !strings.Contains(out, "pool") {
226	t.Fatalf("move pooled = %d %s, want 400 mentioning the pool rail", code, out)
227	}
228	return true
229	}
230
231	// The armed-regions endpoint (issue #346): codes + armed flags from the
232	// edges files (the create door's own predicate), the home region under its
233	// NAME, and the deployment default — the picker contract.
234	func TestAPIRegions(t *testing.T) {
235	fx, token := instanceFixture(t, "code-inst-regions")
236	m := titoManifest()
237	m.Regions = nil
238	m.Pool = ""
239	enableInstances(t, fx, "titogo", m)
240	base := fx.ts.URL + "/api/cli/apps/bab/titogo/regions"
241	put := func(key, body string) {
242	t.Helper()
243	if _, err := fx.rel.Put(context.Background(), key, []byte(body), ""); err != nil {
244	t.Fatal(err)
245	}
246	}
247
248	// No named home, no placement-publishing boxes: empty list, empty
249	// default — consumers hide their pickers.
250	code, _, out := rawReq(t, http.MethodGet, base, token, "")
251	if code != http.StatusOK {
252	t.Fatalf("regions = %d %s", code, out)
253	}
254	var rl struct {
255	Regions []struct {
256	Code string `json:"code"`
257	Armed bool `json:"armed"`
258	} `json:"regions"`
259	Default string `json:"default"`
260	}
261	if err := json.Unmarshal([]byte(out), &rl); err != nil {
262	t.Fatal(err)
263	}
264	if len(rl.Regions) != 0 || rl.Default != "" {
265	t.Fatalf("empty deployment answered %+v", rl)
266	}
267
268	// A named home + one armed box + one EIP-less placement.
269	fx.srv.cfg.HomeRegion = "ie"
270	put("control/edges/uk-box.json", `{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`)
271	put("control/edges/se-box.json", `{"v":1,"box":"se-box","eip":"","placement":"se","hosts":{}}`)
272	code, _, out = rawReq(t, http.MethodGet, base, token, "")
273	if code != http.StatusOK {
274	t.Fatalf("regions = %d %s", code, out)
275	}
276	if err := json.Unmarshal([]byte(out), &rl); err != nil {
277	t.Fatal(err)
278	}
279	want := "[{ie true} {se false} {uk true}]"
280	got := fmt.Sprint(func() (r []struct {
281	Code string
282	Armed bool
283	}) {
284	for _, x := range rl.Regions {
285	r = append(r, struct {
286	Code string
287	Armed bool
288	}{x.Code, x.Armed})
289	}
290	return
291	}())
292	if got != want || rl.Default != "ie" {
293	t.Fatalf("regions = %s default=%q, want %s / ie", got, rl.Default, want)
294	}
295	}
296
297	// The home region's NAME normalizes to the record's "" at the create door
298	// (issue #346): a picker offering "ie" creates exactly what an unconstrained
299	// create always made, and the fleet's scope rules never see the code.
300	func TestAPIInstanceCreateNormalizesHomeRegion(t *testing.T) {
301	fx, token := instanceFixture(t, "code-inst-homenorm")
302	m := titoManifest()
303	m.Regions = nil
304	m.Pool = ""
305	enableInstances(t, fx, "titogo", m)
306	fx.srv.cfg.HomeRegion = "ie"
307	if _, err := fx.rel.Put(context.Background(), "control/edges/uk-box.json",
308	[]byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`), ""); err != nil {
309	t.Fatal(err)
310	}
311	u := instancesURL(fx, "bab", "titogo")
312	// "ie" (the home's name, NOT in the armed edges set) is accepted and
313	// stored as "".
314	code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"home.go.tito.io","region":"ie"}`)
315	if code != http.StatusCreated {
316	t.Fatalf("create region ie (home name) = %d %s, want 201", code, out)
317	}
318	rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "home.go.tito.io")
319	if err != nil {
320	t.Fatal(err)
321	}
322	if rec.Region != "" {
323	t.Fatalf("record region = %q, want the normalized unconstrained value", rec.Region)
324	}
325	// A move targeting the home name is refused plainly (no staging lane).
326	if code, _, out := rawReq(t, http.MethodPost, u+"/home.go.tito.io/move", token, `{"region":"ie"}`); code != http.StatusBadRequest || !strings.Contains(out, "home region") {
327	t.Fatalf("move to home = %d %s, want the plain refusal", code, out)
328	}
329	}
330
331	// ---- Phase 1 (CARLOS_MOVES_PHASE1): the operator door and the claim CAS ----
332
333	// operatorInstanceFixture is instanceFixture with the token's address an
334	// operator and the Phase 1 switch on.
335	func operatorInstanceFixture(t *testing.T, code string) (*consoleFixture, string) {
336	t.Helper()
337	fx, token := instanceFixture(t, code)
338	if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
339	t.Fatal(err)
340	}
341	enableMovesPhase1(fx)
342	return fx, token
343	}
344
345	// enableMovesPhase1 turns the switch on for a fixture built with it off:
346	// the config field, and the routes NewServer registers only when it is on.
347	func enableMovesPhase1(fx *consoleFixture) {
348	fx.srv.cfg.MovesEnabled = true
349	fx.srv.registerMovesPhase1Routes()
350	}
351
352	// errorText is a JSON error body's sentence, "" when the body is not one.
353	func errorText(body string) string {
354	var e struct {
355	Error string `json:"error"`
356	}
357	if json.Unmarshal([]byte(body), &e) != nil {
358	return ""
359	}
360	return e.Error
361	}
362
363	// writeServingClaim writes the host's first claim: serving under box, held
364	// by the titogo instance of incarnation inc, at the root location the
365	// activator's lease lives at (the only one Phase 1 moves, ruling T5-lease). Update validates what it
366	// writes, so an invalid fixture dies here and not in the door.
367	func writeServingClaim(t *testing.T, rel release.Store, host, inc string, created time.Time, box, region string) {
368	t.Helper()
369	_, err := hostclaim.Update(context.Background(), rel, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error {
370	c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "titogo", host), Account: "bab", App: "titogo",
371	Incarnation: inc, CreatedAt: created, Location: "instances/" + host, Stamp: 1}
372	c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box}
373	c.Touched = hostclaim.Touched{Complete: true, Regions: []string{region}, Boxes: []string{box}}
374	return nil
375	})
376	if err != nil {
377	t.Fatal(err)
378	}
379	}
380
381	// claimFor writes acme's serving claim for the record's own incarnation.
382	func claimFor(t *testing.T, fx *consoleFixture, host, box, region string) (inc string) {
383	t.Helper()
384	rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", host)
385	if err != nil {
386	t.Fatal(err)
387	}
388	inc, err = hostclaim.Incarnation("bab", "titogo", host, rec.CreatedAt)
389	if err != nil {
390	t.Fatal(err)
391	}
392	writeServingClaim(t, fx.rel, host, inc, rec.CreatedAt, box, region)
393	return inc
394	}
395
396	func claimBytes(t *testing.T, fx *consoleFixture, host string) string {
397	t.Helper()
398	b, _, err := fx.rel.Get(context.Background(), "control/hosts/"+host+".json")
399	if err != nil {
400	return ""
401	}
402	return string(b)
403	}
404
405	// The switch-off contract (index D13): with CARLOS_MOVES_PHASE1 off the
406	// door is main's, byte for byte. A member's token still moves, the legacy
407	// sentences keep their wording, no claim is written, and the resolve route
408	// answers the family's 404.
409	func TestMoveDoorSwitchOffIsLegacy(t *testing.T) {
410	fx, token := instanceFixture(t, "code-inst-move-off")
411	m := titoManifest()
412	m.Regions = nil
413	m.Pool = ""
414	enableInstances(t, fx, "titogo", m)
415	u := instancesURL(fx, "bab", "titogo")
416	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
417	t.Fatalf("create = %d %s", code, out)
418	}
419	if _, err := fx.rel.Put(context.Background(), "control/edges/uk-box.json",
420	[]byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","data_bucket":"data-uk","hosts":{}}`), ""); err != nil {
421	t.Fatal(err)
422	}
423	moveURL := u + "/acme.go.tito.io/move"
424	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"se"}`); code != http.StatusBadRequest ||
425	!strings.Contains(out, "cannot take a move in this deployment \u2014 it publishes no staging bucket") {
426	t.Fatalf("legacy refusal = %d %s, want main's sentence", code, out)
427	}
428	code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`)
429	if code != http.StatusAccepted || !strings.Contains(out, provision.MovePreparing) {
430	t.Fatalf("member request with the switch off = %d %s, want 202", code, out)
431	}
432	rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io")
433	if err != nil {
434	t.Fatal(err)
435	}
436	if !rec.Move.Legacy() || rec.Move.Cut != "" {
437	t.Fatalf("switch-off move = %+v, want main's id-less journal", rec.Move)
438	}
439	if claimBytes(t, fx, "acme.go.tito.io") != "" {
440	t.Fatal("the legacy door wrote a host claim")
441	}
442	// The transition refusals keep main's bytes, prefix and all.
443	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict || errorText(out) != "provision: move state conflict: cut is only legal from ready" {
444	t.Fatalf("legacy cut from preparing = %d %s, want main's sentence", code, out)
445	}
446	// The resolve route does not exist with the switch off: every way in
447	// answers exactly what an unrouted sibling path answers on main.
448	unrouted := moveURL + "/no-such-door"
449	same := func(what, method, bearer, body string) {
450	t.Helper()
451	wc, wh, wb := rawReq(t, method, unrouted, bearer, body)
452	gc, gh, gb := rawReq(t, method, moveURL+"/resolve", bearer, body)
453	if gc != wc || gb != wb || gh.Get("Content-Type") != wh.Get("Content-Type") || gh.Get("Allow") != wh.Get("Allow") {
454	t.Errorf("%s: resolve = %d %q %q, an unrouted path = %d %q %q", what, gc, gh.Get("Content-Type"), gb, wc, wh.Get("Content-Type"), wb)
455	}
456	}
457	same("unauthenticated POST", http.MethodPost, "", `{"retry":true}`)
458	same("member POST", http.MethodPost, token, `{"retry":true}`)
459	same("member GET", http.MethodGet, token, "")
460	if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
461	t.Fatal(err)
462	}
463	same("operator POST", http.MethodPost, token, `{"retry":true}`)
464	same("unauthenticated GET", http.MethodGet, "", "")
465	}
466
467	// The fleet's refusals (§7.1, §11, D15), each before the claim CAS.
468	func TestMoveDoorRefusesTheFleet(t *testing.T) {
469	floor := `{"min":1,"min_release":"v0.23.0"}`
470	unarmed := ukBox("uk-box")
471	unarmed.FloorUnarmed = true
472	unverified := ukBox("uk-box")
473	unverified.Unverified = true
474	// A box from before moves: its edges file has no moves_protocol, which
475	// decodes as 0, the same as an explicit 0 (§14.3).
476	preProtocol := ukBox("uk-box")
477	preProtocol.NoProtocol = true
478	for _, c := range []struct {
479	name, floor string
480	boxes []fleetBox
481	broken string // a key whose read fails
482	code int
483	want string
484	}{
485	{"two boxes in the target", "", []fleetBox{ieBox("ie-box"), ukBox("uk-a"), ukBox("uk-b")}, "",
486	http.StatusBadRequest, "uk has more than one box. Moves between regions with several boxes are not supported yet."},
487	{"a box that has not armed the floor", floor, []fleetBox{ieBox("ie-box"), unarmed}, "",
488	http.StatusConflict, "a box in ie or uk is below the moves protocol floor. roll the platform first"},
489	{"a pre-protocol box, no floor published", "", []fleetBox{ieBox("ie-box"), preProtocol}, "",
490	http.StatusConflict, "a box in ie or uk is below the moves protocol floor. roll the platform first"},
491	{"no verified box in the target", "", []fleetBox{ieBox("ie-box"), unverified}, "",
492	http.StatusBadRequest, "no box in uk publishes an edges file yet"},
493	{"no eligible box in the source", "", []fleetBox{ukBox("uk-box"), {Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"}}, "",
494	http.StatusBadRequest, "no eligible box in ie publishes an edges file"},
495	{"an unreadable edges file", "", []fleetBox{ieBox("ie-box"), ukBox("uk-box"), {Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"}}, edgesFilePrefix + "se-box.json",
496	http.StatusConflict, "the fleet's edges files could not all be read. try again"},
497	{"a floor above this console", `{"min":2,"min_release":"v0.23.0"}`, []fleetBox{ieBox2("ie-box"), ukBox2("uk-box")}, "",
498	http.StatusConflict, "this console is below the moves protocol floor. roll the console first"},
499	{"an unreadable floor", floor, []fleetBox{ieBox("ie-box"), ukBox("uk-box")}, moves.FloorKey,
500	http.StatusServiceUnavailable, "could not read the moves protocol floor. try again"},
501	} {
502	t.Run(c.name, func(t *testing.T) {
503	fx, token := operatorInstanceFixture(t, "code-inst-move-fleet")
504	m := titoManifest()
505	m.Regions = nil
506	m.Pool = ""
507	enableInstances(t, fx, "titogo", m)
508	u := instancesURL(fx, "bab", "titogo")
509	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
510	t.Fatalf("create = %d %s", code, out)
511	}
512	if c.floor != "" {
513	if _, err := fx.rel.Put(context.Background(), moves.FloorKey, []byte(c.floor), ""); err != nil {
514	t.Fatal(err)
515	}
516	}
517	armMoveFleet(t, fx, c.boxes...)
518	claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
519	before := claimBytes(t, fx, "acme.go.tito.io")
520	if c.broken != "" {
521	fx.srv.cfg.Releases = failingGet{Store: fx.rel, key: c.broken}
522	}
523	code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`)
524	fx.srv.cfg.Releases = fx.rel
525	if code != c.code || !strings.Contains(out, c.want) {
526	t.Fatalf("request = %d %s, want %d %q", code, out, c.code, c.want)
527	}
528	if claimBytes(t, fx, "acme.go.tito.io") != before {
529	t.Fatal("a refused request wrote the claim")
530	}
531	})
532	}
533	}
534
535	// Mutations are operator-only (§12, codex 11); a member's token and the
536	// Instance-role token keep the status GET only (M18).
537	func TestMoveMutationsAreOperatorOnly(t *testing.T) {
538	fx, token := instanceFixture(t, "code-inst-move-op")
539	enableMovesPhase1(fx)
540	m := titoManifest()
541	m.Regions = nil
542	m.Pool = ""
543	enableInstances(t, fx, "titogo", m)
544	u := instancesURL(fx, "bab", "titogo")
545	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
546	t.Fatalf("create = %d %s", code, out)
547	}
548	router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
549	moveURL := u + "/acme.go.tito.io/move"
550	want := canonicalNotFound(t)
551	for _, who := range []struct{ name, token string }{{"a member", token}, {"an instance token", router}} {
552	for _, c := range []struct{ method, path, body string }{
553	{http.MethodPost, moveURL, `{"region":"uk"}`},
554	{http.MethodPost, moveURL + "/cut", ""},
555	{http.MethodDelete, moveURL, ""},
556	{http.MethodPost, moveURL + "/resolve", `{"retry":true}`},
557	} {
558	code, h, body := rawReq(t, c.method, c.path, who.token, c.body)
559	assertNotFound(t, c.method+" "+c.path+" as "+who.name, want, code, h, body)
560	}
561	if code, _, out := rawReq(t, http.MethodGet, moveURL, who.token, ""); code != http.StatusOK || !strings.Contains(out, `"state":"none"`) {
562	t.Fatalf("status as %s = %d %s", who.name, code, out)
563	}
564	}
565	if claimBytes(t, fx, "acme.go.tito.io") != "" {
566	t.Fatal("a refused mutation wrote a claim")
567	}
568	// An operator reaches the resolve door. A retry with no move in flight
569	// is the guarded transition's own 409; a body that asks for nothing the
570	// door does yet is "nothing to resolve yet".
571	if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
572	t.Fatal(err)
573	}
574	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`); code != http.StatusConflict || errorText(out) != "no journaled move in flight" {
575	t.Fatalf("resolve --retry as an operator with no move = %d %s", code, out)
576	}
577	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{}`); code != http.StatusConflict || errorText(out) != "nothing to resolve yet" {
578	t.Fatalf("resolve with nothing asked as an operator = %d %s", code, out)
579	}
580	}
581
582	// Review Focus 4: a refused request writes no claim.
583	func TestMoveDoorRefusesWithoutClaimWrite(t *testing.T) {
584	fx, token := operatorInstanceFixture(t, "code-inst-move-ref")
585	m := titoManifest()
586	m.Regions = []string{"ie", "uk"}
587	m.Pool = ""
588	enableInstances(t, fx, "titogo", m)
589	u := instancesURL(fx, "bab", "titogo")
590	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
591	t.Fatalf("create = %d %s", code, out)
592	}
593	if _, err := fx.rel.Put(context.Background(), moves.FloorKey, []byte(`{"min":1,"min_release":"v0.23.0"}`), ""); err != nil {
594	t.Fatal(err)
595	}
596	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"),
597	fleetBox{Box: "se-box", Placement: "se", AWS: "eu-north-1", EIP: "1.2.3.5", Bucket: "data-se"})
598	moveURL := u + "/acme.go.tito.io/move"
599	// No claim yet: 409, no claim written.
600	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict || !strings.Contains(out, "no host claim") {
601	t.Fatalf("claimless = %d %s", code, out)
602	}
603	if claimBytes(t, fx, "acme.go.tito.io") != "" {
604	t.Fatal("a refused request wrote a claim")
605	}
606	inc := claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
607	before := claimBytes(t, fx, "acme.go.tito.io")
608	// Current region, a region outside the app's, a bad cut: refused, and
609	// none writes.
610	for _, c := range []struct {
611	body, want string
612	code int
613	}{
614	{`{"region":"ie"}`, "already in ie", http.StatusConflict},
615	{`{"region":"se"}`, "region se is not one of this app's regions: ie, uk", http.StatusBadRequest},
616	{`{"region":"uk","cut":"later"}`, `cut must be \"auto\" or \"manual\"`, http.StatusBadRequest},
617	} {
618	code, _, out := rawReq(t, http.MethodPost, moveURL, token, c.body)
619	if code != c.code || !strings.Contains(out, c.want) {
620	t.Fatalf("%s = %d %s, want %d %q", c.body, code, out, c.code, c.want)
621	}
622	if claimBytes(t, fx, "acme.go.tito.io") != before {
623	t.Fatalf("%s wrote the claim", c.body)
624	}
625	}
626	// The real thing: the claim goes moving with the id the record carries.
627	code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`)
628	if code != http.StatusAccepted {
629	t.Fatalf("request = %d %s", code, out)
630	}
631	rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io")
632	if err != nil {
633	t.Fatal(err)
634	}
635	obs, err := hostclaim.Read(context.Background(), fx.rel, "acme.go.tito.io")
636	if err != nil {
637	t.Fatal(err)
638	}
639	op := obs.Claim.Op
640	if op.Kind != hostclaim.KindMoving || op.ID != rec.Move.ID || !moves.ValidOpID(op.ID) || op.Exec != "ie-box" || op.Intent["target"] != "uk-box" || op.By != "alice@keymail.dev" {
641	t.Fatalf("claim op %+v", op)
642	}
643	if rec.Move.Cut != "manual" || rec.Move.Incarnation != inc || rec.Move.From != "ie" || rec.Move.To != "uk" || rec.Move.RequestedBy != "alice@keymail.dev" {
644	t.Fatalf("record move %+v", rec.Move)
645	}
646	// A second request: the claim is not serving, 409 naming the kind.
647	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict || !strings.Contains(out, "under a moving operation") {
648	t.Fatalf("second request = %d %s", code, out)
649	}
650	}
651
652	// §7.1 binds the source to the claim's executor (round-5 finding 7), and
653	// the claim's holder must be this incarnation.
654	func TestMoveDoorBindsTheClaimToTheSourceAndTheIncarnation(t *testing.T) {
655	for _, c := range []struct {
656	name, exec, want string
657	otherInc, free bool
658	}{
659	{name: "an executor that is not the source's eligible box", exec: "old-box", want: "the source box old-box is not the eligible box of ie"},
660	{name: "another incarnation", exec: "ie-box", otherInc: true, want: "another incarnation"},
661	{name: "a free claim with no holder", free: true, want: "the instance has no host claim yet. it is adopted on its next reconcile pass"},
662	} {
663	t.Run(c.name, func(t *testing.T) {
664	fx, token := operatorInstanceFixture(t, "code-inst-move-bind")
665	m := titoManifest()
666	m.Regions = nil
667	m.Pool = ""
668	enableInstances(t, fx, "titogo", m)
669	u := instancesURL(fx, "bab", "titogo")
670	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
671	t.Fatalf("create = %d %s", code, out)
672	}
673	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
674	rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io")
675	if err != nil {
676	t.Fatal(err)
677	}
678	created := rec.CreatedAt
679	if c.otherInc {
680	created = created.Add(-time.Hour)
681	}
682	inc, err := hostclaim.Incarnation("bab", "titogo", "acme.go.tito.io", created)
683	if err != nil {
684	t.Fatal(err)
685	}
686	if c.free {
687	if _, err := hostclaim.Update(context.Background(), fx.rel, hostclaim.Observed{Host: "acme.go.tito.io", Absent: true}, func(*hostclaim.Claim) error { return nil }); err != nil {
688	t.Fatal(err)
689	}
690	} else {
691	writeServingClaim(t, fx.rel, "acme.go.tito.io", inc, created, c.exec, "ie")
692	}
693	before := claimBytes(t, fx, "acme.go.tito.io")
694	code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`)
695	if code != http.StatusConflict || !strings.Contains(out, c.want) {
696	t.Fatalf("request = %d %s, want 409 %q", code, out, c.want)
697	}
698	if claimBytes(t, fx, "acme.go.tito.io") != before {
699	t.Fatal("a refused request wrote the claim")
700	}
701	if rec, _ := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); rec.Move != nil {
702	t.Fatalf("a refused request wrote the record: %+v", rec.Move)
703	}
704	})
705	}
706	}
707
708	// moveDoorWithClaim is a Phase 1 door fixture that has everything but the
709	// request: the instance in ie, the fleet armed (and enrolled in the roster),
710	// and a serving claim under ie-box. It returns the move URL.
711	func moveDoorWithClaim(t *testing.T, code string) (fx *consoleFixture, token, moveURL string) {
712	t.Helper()
713	fx, token = operatorInstanceFixture(t, code)
714	m := titoManifest()
715	m.Regions = nil
716	m.Pool = ""
717	enableInstances(t, fx, "titogo", m)
718	u := instancesURL(fx, "bab", "titogo")
719	if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); c != http.StatusCreated {
720	t.Fatalf("create = %d %s", c, out)
721	}
722	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
723	claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
724	return fx, token, u + "/acme.go.tito.io/move"
725	}
726
727	// Without the host claim the door's start is main's plain BeginMove: the op
728	// carries the move's own three keys and nothing else. The admission and
729	// both enrollments are the host-claim door's (TestMoveDoorAdmitsTheMoveUnderTheHostClaim);
730	// a console with no claim runtime has no admission to record, and no C7
731	// conversion to pin.
732	func TestMoveDoorPlainStartCarriesTheMoveKeys(t *testing.T) {
733	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-enr")
734	if c, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`); c != http.StatusAccepted {
735	t.Fatalf("request = %d %s", c, out)
736	}
737	obs, err := hostclaim.Read(context.Background(), fx.rel, "acme.go.tito.io")
738	if err != nil {
739	t.Fatal(err)
740	}
741	want := map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}
742	if !reflect.DeepEqual(obs.Claim.Op.Intent, want) {
743	t.Fatalf("claim intent = %v, want %v", obs.Claim.Op.Intent, want)
744	}
745	}
746
747	// brokenFenceStore fails every read of a termination fence and nothing else.
748	type brokenFenceStore struct{ release.Store }
749
750	func (b brokenFenceStore) Get(ctx context.Context, key string) ([]byte, string, error) {
751	if strings.HasPrefix(key, "control/box-fences/") {
752	return nil, "", errors.New("secret-detail: fence store down")
753	}
754	return b.Store.Get(ctx, key)
755	}
756
757	// writeMoveFence stores a termination fence for box, as its own regional
758	// worker would have: observed terminated, for the roster's cloud identity.
759	// A box never witnesses its own termination, so the fence is written by a
760	// separate active peer in the same region and account, which this adds to
761	// the current roster. A body key "written_by" overrides the writer.
762	func writeMoveFence(t *testing.T, fx *consoleFixture, b hostactors.Box, body map[string]any) {
763	t.Helper()
764	ctx := context.Background()
765	raw, _, err := fx.rel.Get(ctx, hostactors.Key)
766	if err != nil {
767	t.Fatal(err)
768	}
769	var roster hostactors.Roster
770	if err := release.VerifyRootEnvelope(fx.pub, raw, &roster); err != nil {
771	t.Fatal(err)
772	}
773	writer := moveRosterBox(fleetBox{Box: "fence-writer-" + b.BoxID, Placement: b.Region, AWS: b.Cloud.Region})
774	writer.Cloud.Account = b.Cloud.Account
775	roster.Boxes[writer.BoxID] = writer
776	putMoveRoster(t, fx, roster)
777	f := map[string]any{"v": 1, "box": b.BoxID, "enrollment": b.Enrollment, "cloud": b.Cloud, "state": "terminated",
778	"observed_at": time.Now().UTC(), "via": "describe", "request_id": "request",
779	"written_by": map[string]string{"box": writer.BoxID, "enrollment": writer.Enrollment}}
780	for k, v := range body {
781	f[k] = v
782	}
783	raw, err = json.Marshal(f)
784	if err != nil {
785	t.Fatal(err)
786	}
787	if err := release.CreateExact(ctx, fx.rel, "control/box-fences/"+b.Enrollment+".json", raw); err != nil {
788	t.Fatal(err)
789	}
790	}
791
792	// Ruling E2: from ONE roster observation, a source must be active, carry an
793	// enrollment, and have no termination fence, or the door refuses 409. A
794	// roster or fence that cannot be read is a 503 with a fixed sentence, never
795	// the error's own text. Every refusal writes neither the claim nor the
796	// record.
797	func TestMoveDoorRefusesAnIneligibleSource(t *testing.T) {
798	const ineligible = "the source box is not eligible to move from"
799	const unreadable = "could not read the host roster. try again"
800	ie, uk := ieBox("ie-box"), ukBox("uk-box")
801	rosterWith := func(mut func(b *hostactors.Box)) hostactors.Roster {
802	b := moveRosterBox(ie)
803	mut(&b)
804	return hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": b, "uk-box": moveRosterBox(uk)}}
805	}
806	for _, c := range []struct {
807	name string
808	setup func(t *testing.T, fx *consoleFixture)
809	code int
810	want string
811	}{
812	{"no roster entry for the source", func(t *testing.T, fx *consoleFixture) {
813	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"uk-box": moveRosterBox(uk)}})
814	}, http.StatusConflict, ineligible},
815	{"an entry for another box only", func(t *testing.T, fx *consoleFixture) {
816	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"old-box": moveRosterBox(fleetBox{Box: "old-box", Placement: "ie", AWS: "eu-west-1"})}})
817	}, http.StatusConflict, ineligible},
818	{"a retired source", func(t *testing.T, fx *consoleFixture) {
819	putMoveRoster(t, fx, rosterWith(func(b *hostactors.Box) { b.State = "retired" }))
820	}, http.StatusConflict, ineligible},
821	{"a quarantined source", func(t *testing.T, fx *consoleFixture) {
822	putMoveRoster(t, fx, rosterWith(func(b *hostactors.Box) {
823	b.State, b.QuarantineRevision, b.LossIDs = "quarantined", 1, []string{"loss-1"}
824	}))
825	}, http.StatusConflict, ineligible},
826	{"a fenced source", func(t *testing.T, fx *consoleFixture) {
827	writeMoveFence(t, fx, moveRosterBox(ie), nil)
828	}, http.StatusConflict, ineligible},
829	{"a fence that cannot be read", func(t *testing.T, fx *consoleFixture) {
830	fx.srv.cfg.Releases = brokenFenceStore{fx.srv.cfg.Releases}
831	}, http.StatusServiceUnavailable, unreadable},
832	{"a fence record that does not validate", func(t *testing.T, fx *consoleFixture) {
833	writeMoveFence(t, fx, moveRosterBox(ie), map[string]any{"state": "stopped"})
834	}, http.StatusServiceUnavailable, unreadable},
835	{"no roster object", func(t *testing.T, fx *consoleFixture) {
836	if err := fx.rel.Delete(context.Background(), hostactors.Key); err != nil {
837	t.Fatal(err)
838	}
839	}, http.StatusServiceUnavailable, unreadable},
840	{"a roster that does not verify", func(t *testing.T, fx *consoleFixture) {
841	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ie)}})
842	if _, err := fx.rel.Put(context.Background(), hostactors.Key, []byte(`{"secret-detail":"ed25519: bad"}`), mustETag(t, fx, hostactors.Key)); err != nil {
843	t.Fatal(err)
844	}
845	}, http.StatusServiceUnavailable, unreadable},
846	{"a roster whose source has an empty enrollment", func(t *testing.T, fx *consoleFixture) {
847	putMoveRoster(t, fx, rosterWith(func(b *hostactors.Box) { b.Enrollment = "" }))
848	}, http.StatusServiceUnavailable, unreadable},
849	{"a roster for another bucket", func(t *testing.T, fx *consoleFixture) {
850	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ie)}})
851	fx.srv.cfg.Bucket = "another-bucket"
852	}, http.StatusServiceUnavailable, unreadable},
853	} {
854	t.Run(c.name, func(t *testing.T) {
855	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-enr-closed")
856	c.setup(t, fx)
857	before := claimBytes(t, fx, "acme.go.tito.io")
858	code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`)
859	if code != c.code || errorText(out) != c.want {
860	t.Fatalf("request = %d %s, want %d %q", code, out, c.code, c.want)
861	}
862	if strings.Contains(out, "secret-detail") || strings.Contains(out, "roster authority") {
863	t.Fatalf("the body leaks an error's text: %s", out)
864	}
865	if claimBytes(t, fx, "acme.go.tito.io") != before {
866	t.Fatal("a refused request wrote the claim")
867	}
868	if rec, _ := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io"); rec.Move != nil {
869	t.Fatalf("a refused request wrote the record: %+v", rec.Move)
870	}
871	})
872	}
873	}
874
875	// The lookup the door uses: by box reference (never by label), active only,
876	// with a non-empty enrollment. A verified roster cannot carry an empty
877	// enrollment, so that case is proved here and answers 503 at the door.
878	func TestSourceEnrollmentOfTheRoster(t *testing.T) {
879	r := hostactors.Roster{Boxes: map[string]hostactors.Box{
880	"ie-box": {BoxID: "ie-box", Label: "dublin", Enrollment: "e1", State: "active"},
881	"blank": {BoxID: "blank", State: "active"},
882	"retired": {BoxID: "retired", Enrollment: "e2", State: "retired"},
883	"quar": {BoxID: "quar", Enrollment: "e3", State: "quarantined"},
884	"nostate": {BoxID: "nostate", Enrollment: "e4"},
885	}}
886	if got, ok := sourceEnrollment(r, "ie-box"); !ok || got != "e1" {
887	t.Fatalf("enrolled source = %q, %v", got, ok)
888	}
889	for _, box := range []string{"blank", "retired", "quar", "nostate", "dublin", "absent", ""} {
890	if got, ok := sourceEnrollment(r, box); ok || got != "" {
891	t.Fatalf("source %q = %q, %v, want none", box, got, ok)
892	}
893	}
894	}
895
896	func mustETag(t *testing.T, fx *consoleFixture, key string) string {
897	t.Helper()
898	_, etag, err := fx.rel.Get(context.Background(), key)
899	if err != nil {
900	t.Fatal(err)
901	}
902	return etag
903	}
904
905	// Private storage is the instance's BINDING (round-4 finding 13), not the
906	// app's create-time policy, and both policy reads fail closed.
907	func TestMoveDoorRefusesABoundStoreNotAPolicy(t *testing.T) {
908	fx, token := operatorInstanceFixture(t, "code-inst-move-store")
909	m := titoManifest()
910	m.Regions = nil
911	m.Pool = ""
912	enableInstances(t, fx, "titogo", m)
913	u := instancesURL(fx, "bab", "titogo")
914	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
915	t.Fatalf("create = %d %s", code, out)
916	}
917	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
918	claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
919	before := claimBytes(t, fx, "acme.go.tito.io")
920	moveURL := u + "/acme.go.tito.io/move"
921	ctx := context.Background()
922	if _, err := objstore.PutIn(ctx, fx.rel, "bab", "titogo", "acme", &objstore.Record{}, ""); err != nil {
923	t.Fatal(err)
924	}
925	rel := fx.srv.cfg.Releases
926	for _, c := range []struct{ name, key string }{
927	{"the storage binding", objstore.KeyIn("bab", "titogo", "acme")},
928	{"the manifest", provision.ManifestKey("bab", "titogo")},
929	} {
930	fx.srv.cfg.Releases = failingGet{Store: rel, key: c.key}
931	code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`)
932	fx.srv.cfg.Releases = rel
933	if code != http.StatusServiceUnavailable {
934	t.Fatalf("%s unreadable = %d %s, want 503", c.name, code, out)
935	}
936	if claimBytes(t, fx, "acme.go.tito.io") != before {
937	t.Fatalf("%s unreadable: the claim was written", c.name)
938	}
939	}
940	// A store under the slug with the policy off: M-C17, no claim written.
941	code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`)
942	if code != http.StatusBadRequest || !strings.Contains(out, "acme.go.tito.io uses private storage. Instances with private storage cannot move yet.") {
943	t.Fatalf("bound store = %d %s, want M-C17", code, out)
944	}
945	if claimBytes(t, fx, "acme.go.tito.io") != before {
946	t.Fatal("a refused request wrote the claim")
947	}
948	// The policy on and no store: admitted.
949	if err := fx.rel.Delete(ctx, objstore.KeyIn("bab", "titogo", "acme")); err != nil {
950	t.Fatal(err)
951	}
952	m.PrivateStorage = true
953	enableInstances(t, fx, "titogo", m)
954	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted {
955	t.Fatalf("policy on, no store = %d %s, want 202", code, out)
956	}
957	}
958
959	// The legacy matrix with the Phase 1 rules: an operator's token, a serving
960	// claim, a production-shaped fleet, guarded cut and abort.
961	func TestAPIInstanceMoveVerbsPhase1(t *testing.T) {
962	fx, token := operatorInstanceFixture(t, "code-inst-move-p1")
963	m := titoManifest()
964	m.Regions = nil
965	m.Pool = ""
966	enableInstances(t, fx, "titogo", m)
967	u := instancesURL(fx, "bab", "titogo")
968	ctx := context.Background()
969	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
970	t.Fatalf("create = %d %s", code, out)
971	}
972	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
973	inc := claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
974	moveURL := u + "/acme.go.tito.io/move"
975	guard := func() provision.MoveGuard {
976	t.Helper()
977	rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io")
978	if err != nil || rec.Move == nil {
979	t.Fatalf("record move: %v %+v", err, rec)
980	}
981	return provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation}
982	}
983
984	// A target with no staging bucket: 400 naming the set that can take one.
985	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"se"}`); code != http.StatusBadRequest || !strings.Contains(out, "it publishes no staging bucket. Regions that can: ie, uk") {
986	t.Fatalf("move to a region with no staging bucket = %d %s", code, out)
987	}
988	// Cut and abort before any move: the guarded transitions' own 409.
989	for _, c := range []struct{ method, path string }{{http.MethodPost, moveURL + "/cut"}, {http.MethodDelete, moveURL}} {
990	if code, _, out := rawReq(t, c.method, c.path, token, ""); code != http.StatusConflict || errorText(out) != "no journaled move in flight" {
991	t.Fatalf("%s %s with no move = %d %s", c.method, c.path, code, out)
992	}
993	}
994	if code, _, out := rawReq(t, http.MethodGet, moveURL, token, ""); code != http.StatusOK || !strings.Contains(out, `"state":"none"`) {
995	t.Fatalf("status with no move = %d %s", code, out)
996	}
997	// Request: the flat view.
998	code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`)
999	if code != http.StatusAccepted || !strings.Contains(out, `"state":"preparing"`) || !strings.Contains(out, `"target":"uk"`) {
1000	t.Fatalf("request = %d %s", code, out)
1001	}
1002	g := guard()
1003	if g.Incarnation != inc {
1004	t.Fatalf("guard %+v, want incarnation %s", g, inc)
1005	}
1006	// Second request while in flight, cut from preparing: 409.
1007	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict {
1008	t.Fatalf("second request = %d %s", code, out)
1009	}
1010	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict || errorText(out) != "cut is only legal from ready" {
1011	t.Fatalf("cut from preparing = %d %s", code, out)
1012	}
1013	// Abort from preparing: 202 aborting; again: 409; a request: 409.
1014	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) {
1015	t.Fatalf("abort = %d %s", code, out)
1016	}
1017	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != "already aborting" {
1018	t.Fatalf("double abort = %d %s", code, out)
1019	}
1020	if code, _, _ := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict {
1021	t.Fatalf("request while aborting = %d", code)
1022	}
1023	// The source's half (Task 14): the record clears, the claim is released.
1024	releaseMove := func(g provision.MoveGuard) {
1025	t.Helper()
1026	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedAbortComplete(g, time.Now())); err != nil {
1027	t.Fatal(err)
1028	}
1029	obs, err := hostclaim.Read(ctx, fx.rel, "acme.go.tito.io")
1030	if err != nil {
1031	t.Fatal(err)
1032	}
1033	if _, err := hostclaim.Update(ctx, fx.rel, obs, hostclaim.ReleaseMove(g.ID, "ie-box")); err != nil {
1034	t.Fatal(err)
1035	}
1036	}
1037	releaseMove(g)
1038	// A fresh request proceeds with a new id; ready; cut; abort from
1039	// cutting/requested is legal under the guard.
1040	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted {
1041	t.Fatalf("re-request = %d %s", code, out)
1042	}
1043	g2 := guard()
1044	if g2.ID == g.ID {
1045	t.Fatal("a second move reused the first move's id")
1046	}
1047	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedPromoteReady(g2, time.Now(), provision.MoveInputs{})); err != nil {
1048	t.Fatal(err)
1049	}
1050	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveCutting) {
1051	t.Fatalf("cut from ready = %d %s", code, out)
1052	}
1053	if rec, _ := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io"); rec.Move.CutPhase != provision.CutRequested {
1054	t.Fatalf("cut phase %q, want requested", rec.Move.CutPhase)
1055	}
1056	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) {
1057	t.Fatalf("abort from cutting/requested = %d %s, want 202", code, out)
1058	}
1059	releaseMove(g2)
1060	// Past the admission: abort is refused with M-C13 (preflight L5), the
1061	// cut completes forward or moves back on its own.
1062	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusAccepted {
1063	t.Fatalf("third request = %d %s", code, out)
1064	}
1065	g3 := guard()
1066	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error {
1067	rec.Move.State, rec.Move.CutPhase = provision.MoveCutting, provision.CutAdmitted
1068	return nil
1069	}); err != nil {
1070	t.Fatal(err)
1071	}
1072	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != "This move has started its cut. It will finish or move back on its own." {
1073	t.Fatalf("abort after admission = %d %s, want 409", code, out)
1074	}
1075	_ = g3
1076	if _, _, err := fx.rel.Get(ctx, "control/serial"); err != nil {
1077	t.Fatalf("control/serial not written: %v", err)
1078	}
1079	if code, _, _ := rawReq(t, http.MethodPost, u+"/ghost.go.tito.io/move", token, `{"region":"uk"}`); code != http.StatusNotFound {
1080	t.Fatalf("move on unknown host = %d, want 404", code)
1081	}
1082	}
1083
1084	// A move journaled by an older console has no id: with the switch on, the
1085	// cut names it (MV-N7) and abort takes the legacy transition (F7(a)).
1086	func TestMoveDoorLegacyRecordUnderTheSwitch(t *testing.T) {
1087	fx, token := operatorInstanceFixture(t, "code-inst-move-leg")
1088	m := titoManifest()
1089	m.Regions = nil
1090	m.Pool = ""
1091	enableInstances(t, fx, "titogo", m)
1092	u := instancesURL(fx, "bab", "titogo")
1093	ctx := context.Background()
1094	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
1095	t.Fatalf("create = %d %s", code, out)
1096	}
1097	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveRequest("uk", time.Now())); err != nil {
1098	t.Fatal(err)
1099	}
1100	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MovePromoteReady(time.Now())); err != nil {
1101	t.Fatal(err)
1102	}
1103	moveURL := u + "/acme.go.tito.io/move"
1104	// A request while the legacy move is in flight is refused by the
1105	// record's own transition, tried before the claim CAS: no claim write.
1106	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
1107	claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
1108	before := claimBytes(t, fx, "acme.go.tito.io")
1109	if code, _, out := rawReq(t, http.MethodPost, moveURL, token, `{"region":"uk"}`); code != http.StatusConflict || errorText(out) != "a move to uk is already ready" {
1110	t.Fatalf("request over a legacy move = %d %s, want 409", code, out)
1111	}
1112	if claimBytes(t, fx, "acme.go.tito.io") != before {
1113	t.Fatal("a request refused by the record wrote the claim")
1114	}
1115	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/cut", token, ""); code != http.StatusConflict || !strings.Contains(out, "this move was started by an older console and has no id. abort it and request it again") {
1116	t.Fatalf("cut of a legacy move = %d %s, want MV-N7", code, out)
1117	}
1118	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusAccepted || !strings.Contains(out, provision.MoveAborting) {
1119	t.Fatalf("abort of a legacy move = %d %s, want 202", code, out)
1120	}
1121	// A legacy move past its cut: the legacy refusal, reworded under the
1122	// switch (no prefix, no em dash).
1123	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveAbortComplete()); err != nil {
1124	t.Fatal(err)
1125	}
1126	for _, mut := range []func(*provision.Record) error{provision.MoveRequest("uk", time.Now()), provision.MovePromoteReady(time.Now()), provision.MoveMarkCutting()} {
1127	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", mut); err != nil {
1128	t.Fatal(err)
1129	}
1130	}
1131	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != "past the point of no return. the cut completes forward" {
1132	t.Fatalf("abort of a cutting legacy move = %d %s", code, out)
1133	}
1134	}
1135
1136	// interleaveOnGet runs fn once, just before the at'th read of key: a
1137	// deterministic stand-in for another writer landing between two reads.
1138	type interleaveOnGet struct {
1139	release.Store
1140	key string
1141	at, n int
1142	fn func()
1143	}
1144
1145	func (f *interleaveOnGet) Get(ctx context.Context, key string) ([]byte, string, error) {
1146	if key == f.key {
1147	f.n++
1148	if f.n == f.at {
1149	f.fn()
1150	}
1151	}
1152	return f.Store.Get(ctx, key)
1153	}
1154
1155	// F7(a)'s fallback is bound to the legacy move the door read: if that move
1156	// finished and another began before the write, the abort is refused and
1157	// the replacement is left alone.
1158	func TestMoveDoorLegacyAbortRefusesAReplacement(t *testing.T) {
1159	fx, token := operatorInstanceFixture(t, "code-inst-move-legrep")
1160	m := titoManifest()
1161	m.Regions = nil
1162	m.Pool = ""
1163	enableInstances(t, fx, "titogo", m)
1164	u := instancesURL(fx, "bab", "titogo")
1165	ctx := context.Background()
1166	const host = "acme.go.tito.io"
1167	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
1168	t.Fatalf("create = %d %s", code, out)
1169	}
1170	first := time.Now().Add(-time.Hour).UTC()
1171	second := first.Add(time.Minute)
1172	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", host, provision.MoveRequest("uk", first)); err != nil {
1173	t.Fatal(err)
1174	}
1175	rel := fx.rel
1176	// The door's first read sees the first move; before its write reads the
1177	// record again, that move is aborted and cleared and a second begins.
1178	fx.srv.cfg.Releases = &interleaveOnGet{Store: rel, key: provision.RecordKey("bab", "titogo", host), at: 2, fn: func() {
1179	for _, mut := range []func(*provision.Record) error{provision.MoveAbort(), provision.MoveAbortComplete(), provision.MoveRequest("uk", second)} {
1180	if _, err := provision.SetRecordMove(ctx, rel, "bab", "titogo", host, mut); err != nil {
1181	t.Error(err)
1182	}
1183	}
1184	}}
1185	code, _, out := rawReq(t, http.MethodDelete, u+"/"+host+"/move", token, "")
1186	fx.srv.cfg.Releases = rel
1187	if code != http.StatusConflict || errorText(out) != "the move changed since this request read it. read its state and try again" {
1188	t.Fatalf("delayed legacy abort = %d %s, want 409", code, out)
1189	}
1190	rec, err := provision.GetRecord(ctx, rel, "bab", "titogo", host)
1191	if err != nil {
1192	t.Fatal(err)
1193	}
1194	if rec.Move == nil || rec.Move.State != provision.MovePreparing || !rec.Move.RequestedAt.Equal(second) {
1195	t.Fatalf("the replacement move = %+v, want it preparing and untouched", rec.Move)
1196	}
1197	}
1198
1199	// recordWriteFault fails the first write of key. With land, the write
1200	// reaches the store first, so only the answer is lost. With blind, every
1201	// later read of key fails too, so the door cannot tell which happened.
1202	type recordWriteFault struct {
1203	release.Store
1204	key string
1205	land, blind bool
1206	failed bool
1207	}
1208
1209	func (f *recordWriteFault) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
1210	if key == f.key && !f.failed {
1211	f.failed = true
1212	if f.land {
1213	if _, err := f.Store.Put(ctx, key, body, ifMatch); err != nil {
1214	return "", err
1215	}
1216	}
1217	return "", errors.New("write timeout")
1218	}
1219	return f.Store.Put(ctx, key, body, ifMatch)
1220	}
1221
1222	func (f *recordWriteFault) Get(ctx context.Context, key string) ([]byte, string, error) {
1223	if key == f.key && f.failed && f.blind {
1224	return nil, "", errors.New("read timeout")
1225	}
1226	return f.Store.Get(ctx, key)
1227	}
1228
1229	// The claim CAS succeeded and the record write did not: the door rereads
1230	// the record and either releases the claim (the move is definitely not
1231	// journaled), answers success (it is), or, when it cannot tell, leaves the
1232	// claim and promises nothing.
1233	func TestMoveDoorCompensatesAFailedRecordWrite(t *testing.T) {
1234	const host = "acme.go.tito.io"
1235	for _, c := range []struct {
1236	name string
1237	land, blind bool
1238	code int
1239	want string // the error sentence; "" for success
1240	claimKind string
1241	}{
1242	{name: "not written: the claim is released", code: http.StatusBadGateway, want: "could not record the move. try again", claimKind: hostclaim.KindServing},
1243	{name: "written, answer lost: success", land: true, code: http.StatusAccepted, claimKind: hostclaim.KindMoving},
1244	{name: "unknown: the claim is left", blind: true, code: http.StatusBadGateway,
1245	want: "could not confirm whether the move was recorded. read its state before trying again", claimKind: hostclaim.KindMoving},
1246	} {
1247	t.Run(c.name, func(t *testing.T) {
1248	fx, token := operatorInstanceFixture(t, "code-inst-move-comp")
1249	m := titoManifest()
1250	m.Regions = nil
1251	m.Pool = ""
1252	enableInstances(t, fx, "titogo", m)
1253	u := instancesURL(fx, "bab", "titogo")
1254	ctx := context.Background()
1255	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
1256	t.Fatalf("create = %d %s", code, out)
1257	}
1258	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
1259	claimFor(t, fx, host, "ie-box", "ie")
1260	rel := fx.rel
1261	fault := &recordWriteFault{Store: rel, key: provision.RecordKey("bab", "titogo", host), land: c.land, blind: c.blind}
1262	fx.srv.cfg.Releases = fault
1263	code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`)
1264	fx.srv.cfg.Releases = rel
1265	if !fault.failed {
1266	t.Fatal("the fault never fired")
1267	}
1268	if code != c.code || (c.want != "" && errorText(out) != c.want) {
1269	t.Fatalf("request = %d %s, want %d %q", code, out, c.code, c.want)
1270	}
1271	obs, err := hostclaim.Read(ctx, rel, host)
1272	if err != nil {
1273	t.Fatal(err)
1274	}
1275	if obs.Claim.Op.Kind != c.claimKind || obs.Claim.Op.Exec != "ie-box" {
1276	t.Fatalf("claim op = %+v, want %s under ie-box", obs.Claim.Op, c.claimKind)
1277	}
1278	rec, err := provision.GetRecord(ctx, rel, "bab", "titogo", host)
1279	if err != nil {
1280	t.Fatal(err)
1281	}
1282	switch c.claimKind {
1283	case hostclaim.KindServing:
1284	// Released with evidence, and the host can be asked again.
1285	if rec.Move != nil || obs.Claim.Op.Journal["released_move"] == "" {
1286	t.Fatalf("record move %+v, claim op %+v", rec.Move, obs.Claim.Op)
1287	}
1288	if code, _, out := rawReq(t, http.MethodPost, u+"/"+host+"/move", token, `{"region":"uk"}`); code != http.StatusAccepted {
1289	t.Fatalf("retry after a release = %d %s", code, out)
1290	}
1291	default:
1292	if c.land && (rec.Move == nil || rec.Move.ID != obs.Claim.Op.ID) {
1293	t.Fatalf("record move %+v, claim op %+v: want one id", rec.Move, obs.Claim.Op)
1294	}
1295	}
1296	})
1297	}
1298	}
1299
1300	// The move door's own refusals and the shared gates' sentences it surfaces
1301	// (F13): with the switch off, main's bytes; with it on, the Phase 1
1302	// sentences at the move door and main's bytes at every other door.
1303	func TestMoveDoorGateSentences(t *testing.T) {
1304	const dash = "\u2014"
1305	hib := false
1306	records := []provision.Record{
1307	{Host: "st.go.tito.io", Slug: "st", Kind: "static"},
1308	{Host: "ad.go.tito.io", Slug: "ad", Addrs: []string{"10.0.0.1:80"}},
1309	{Host: "fl.go.tito.io", Slug: "fl", Region: "fleet/pi"},
1310	{Host: "po.go.tito.io", Slug: "po", Pool: "dedicated-x"},
1311	{Host: "nh.go.tito.io", Slug: "nh", Hibernate: &hib},
1312	}
1313	type sentence struct{ host, legacy, phase1 string }
1314	rails := []sentence{
1315	{"st.go.tito.io", "only exec-backed instances move " + dash + " this route is kind static",
1316	"only exec-backed instances move. this route is kind static"},
1317	{"ad.go.tito.io", "an addr-backed route's process is not platform-managed " + dash + " nothing to move",
1318	"an addr-backed route's process is not platform-managed. there is nothing to move"},
1319	{"fl.go.tito.io", "fleet-placed instances keep state-does-not-follow semantics " + dash + " move is for platform regions",
1320	"fleet-placed instances keep state-does-not-follow semantics. moves are for platform regions"},
1321	{"po.go.tito.io", "pooled instances do not move in v1 " + dash + " their data lives on the pool's own box",
1322	"pooled instances do not move yet. their data lives on the pool's own box"},
1323	{"nh.go.tito.io", "a non-hibernating instance runs outside the activator " + dash + " nothing manages its data, nothing can move it",
1324	"a non-hibernating instance runs outside the activator. nothing manages its data, so nothing can move it"},
1325	}
1326	const (
1327	limitLegacy = "too many instance requests " + dash + " wait a minute and try again"
1328	limitPhase1 = "too many instance requests. wait a minute and try again"
1329	bucketLegacy = "no deployment bucket on this host " + dash + " the console is running without one"
1330	bucketPhase1 = "this console has no deployment bucket, so it cannot answer this"
1331	)
1332	ambiguousLegacy := `account "bab" is ambiguous ` + dash + ` pass --account-id with one of: bab ("ops"), bac ("bab")`
1333	ambiguousPhase1 := `account "bab" is ambiguous. pass --account-id with one of: bab ("ops"), bac ("bab")`
1334
1335	for _, on := range []bool{false, true} {
1336	t.Run(fmt.Sprintf("switch %v", on), func(t *testing.T) {
1337	fx, token := operatorInstanceFixture(t, fmt.Sprintf("code-inst-move-copy-%v", on))
1338	if !on {
1339	fx.srv.cfg.MovesEnabled = false
1340	}
1341	ctx := context.Background()
1342	for _, rec := range records {
1343	rec.Schema, rec.App, rec.Account, rec.Channel, rec.CreatedAt = 1, "titogo", "bab", "stable", time.Now().UTC()
1344	if err := provision.CreateRecord(ctx, fx.rel, rec); err != nil {
1345	t.Fatal(err)
1346	}
1347	}
1348	u := instancesURL(fx, "bab", "titogo")
1349	pick := func(s sentence) string {
1350	if on {
1351	return s.phase1
1352	}
1353	return s.legacy
1354	}
1355	check := func(what, method, url, body string, wantCode int, want string) {
1356	t.Helper()
1357	code, _, out := rawReq(t, method, url, token, body)
1358	if code != wantCode || errorText(out) != want {
1359	t.Errorf("%s = %d %s, want %d %q", what, code, out, wantCode, want)
1360	}
1361	}
1362	// The class rails, on every move verb that runs them.
1363	for _, rail := range rails {
1364	for _, v := range []struct{ method, suffix, body string }{
1365	{http.MethodPost, "/move", `{"region":"uk"}`},
1366	{http.MethodGet, "/move", ""},
1367	{http.MethodPost, "/move/cut", ""},
1368	{http.MethodDelete, "/move", ""},
1369	} {
1370	check(rail.host+" "+v.method+v.suffix, v.method, u+"/"+rail.host+v.suffix, v.body, http.StatusBadRequest, pick(rail))
1371	}
1372	}
1373	// The shared instance gate: no deployment bucket.
1374	rel := fx.srv.cfg.Releases
1375	fx.srv.cfg.Releases = nil
1376	check("move, no bucket", http.MethodPost, u+"/st.go.tito.io/move", `{"region":"uk"}`, http.StatusServiceUnavailable, pick(sentence{"", bucketLegacy, bucketPhase1}))
1377	check("instance GET, no bucket", http.MethodGet, u+"/st.go.tito.io", "", http.StatusServiceUnavailable, bucketLegacy)
1378	fx.srv.cfg.Releases = rel
1379	// The account resolver: an ambiguous account segment.
1380	if err := fx.st.CreateAccount(2, "bac", "bab", "alice@keymail.dev", time.Now()); err != nil {
1381	t.Fatal(err)
1382	}
1383	check("move, ambiguous account", http.MethodPost, u+"/st.go.tito.io/move", `{"region":"uk"}`, http.StatusConflict, pick(sentence{"", ambiguousLegacy, ambiguousPhase1}))
1384	check("instance GET, ambiguous account", http.MethodGet, u+"/st.go.tito.io", "", http.StatusConflict, ambiguousLegacy)
1385	check("channels GET, ambiguous account", http.MethodGet, fx.ts.URL+"/api/cli/apps/bab/titogo/channels", "", http.StatusConflict, ambiguousLegacy)
1386	check("app create, ambiguous account", http.MethodPost, fx.ts.URL+"/api/cli/apps/bab/new", `{"name":"other"}`, http.StatusConflict, ambiguousLegacy)
1387	u = u + "?exact=sqid"
1388	// The shared instance gate: the rate limit, spent last.
1389	for i := 0; i < cliInstanceBurst; i++ {
1390	fx.srv.cli.instances.allow(hashToken(token), time.Now())
1391	}
1392	base := instancesURL(fx, "bab", "titogo")
1393	check("move, rate limited", http.MethodPost, base+"/st.go.tito.io/move?exact=sqid", `{"region":"uk"}`, http.StatusTooManyRequests, pick(sentence{"", limitLegacy, limitPhase1}))
1394	check("instance GET, rate limited", http.MethodGet, base+"/st.go.tito.io?exact=sqid", "", http.StatusTooManyRequests, limitLegacy)
1395	})
1396	}
1397	}
1398
1399	// Fix round 2 of Task 5 (ruling T5-lease): Phase 1 cuts only a holder whose
1400	// location is the activator's own lease prefix, so the door refuses a
1401	// scoped holder (instances/<host>/<incarnation>) at request time with the
1402	// recorded sentence (MV-N167), before any claim write. It does so whether
1403	// or not this console knows the activator's prefix; a root holder passes.
1404	func TestMoveDoorRefusesAScopedHolder(t *testing.T) {
1405	for _, prefix := range []string{"", "instances"} {
1406	t.Run("prefix "+prefix, func(t *testing.T) {
1407	fx, token := operatorInstanceFixture(t, "code-inst-move-scoped")
1408	fx.srv.cfg.HibernatePrefix = prefix
1409	m := titoManifest()
1410	m.Regions = nil
1411	m.Pool = ""
1412	enableInstances(t, fx, "titogo", m)
1413	u := instancesURL(fx, "bab", "titogo")
1414	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
1415	t.Fatalf("create = %d %s", code, out)
1416	}
1417	armMoveFleet(t, fx, ieBox("ie-box"), ukBox("uk-box"))
1418	inc := claimFor(t, fx, "acme.go.tito.io", "ie-box", "ie")
1419	rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) { c.Holder.Location = "instances/acme.go.tito.io/" + inc })
1420	before := claimBytes(t, fx, "acme.go.tito.io")
1421	code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`)
1422	if code != http.StatusConflict || errorText(out) != provision.RefusedReplicaLayout {
1423	t.Fatalf("a scoped holder = %d %s, want 409 %q", code, out, provision.RefusedReplicaLayout)
1424	}
1425	if claimBytes(t, fx, "acme.go.tito.io") != before {
1426	t.Fatal("a refused request wrote the claim")
1427	}
1428	rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) { c.Holder.Location = "instances/acme.go.tito.io" })
1429	if code, _, out := rawReq(t, http.MethodPost, u+"/acme.go.tito.io/move", token, `{"region":"uk"}`); code != http.StatusAccepted {
1430	t.Fatalf("a root holder = %d %s, want 202", code, out)
1431	}
1432	})
1433	}
1434	}
1435
1436	// rewriteClaim edits the host's claim in place, as a test fixture.
1437	func rewriteClaim(t *testing.T, fx *consoleFixture, host string, edit func(*hostclaim.Claim)) {
1438	t.Helper()
1439	obs, err := hostclaim.Read(context.Background(), fx.rel, host)
1440	if err != nil || obs.Absent {
1441	t.Fatalf("claim: %v absent=%v", err, obs.Absent)
1442	}
1443	if _, err := hostclaim.Update(context.Background(), fx.rel, obs, func(c *hostclaim.Claim) error { edit(c); return nil }); err != nil {
1444	t.Fatal(err)
1445	}
1446	}
1447
1448	// Plan 4 Task 1 (ruling J20): the rollback, retry and declare-lost refusals
1449	// reach an operator as their recorded sentences, taken from the transitions
1450	// themselves, not the fixed sentence.
1451	func TestPhase1ConflictPassesPlan4Refusals(t *testing.T) {
1452	g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
1453	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1454	const rb = "rb_abcdefghijklmnopqrstuvwxyz"
1455	committed := &provision.Record{Region: "uk", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveCommitted}}
1456	notStuck := &provision.Record{Region: "ie", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveReverting}}
1457	cleaned := &provision.Record{Region: "ie", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveRollingBack,
1458	Rollback: &provision.MoveRollback{ID: rb, TargetPhase: provision.RbTargetCleaned}}}
1459	for want, err := range map[string]error{
1460	provision.RefusedCommitted: provision.MoveRollBack(g, rb, provision.RollbackLandingDeadline, now)(committed),
1461	provision.RefusedNotStuck: provision.MoveRetry(g)(notStuck),
1462	provision.RefusedNothingToDeclare: provision.MoveRollbackTargetLost(g, "control/losses/x.json", now)(cleaned),
1463	} {
1464	if !errors.Is(err, provision.ErrMoveConflict) {
1465	t.Fatalf("%q: %v", want, err)
1466	}
1467	if got := phase1Conflict(err); got != want {
1468	t.Fatalf("phase1Conflict(%v) = %q, want %q", err, got, want)
1469	}
1470	}
1471	// Ruling J9 (fix round 1): the four reasons Plan 4 records are spoken as
1472	// their recorded sentences too, not the fixed one.
1473	for _, reason := range []string{provision.RollbackDeclaredLost, provision.StuckBaselineMismatch, provision.StuckRecoveryProbeAttempts, provision.StuckRevertProbeBudget,
1474	provision.StuckReturnFilesOccupied, provision.StuckReturnInstallMismatch, provision.StuckReturnUnknownStamp,
1475	provision.StuckReturnTargetDatabase, provision.StuckReturnTargetChanged, provision.StuckReturnTargetFiles} {
1476	err := fmt.Errorf("%w: %s", provision.ErrMoveConflict, reason)
1477	if got := phase1Conflict(err); got != reason {
1478	t.Fatalf("phase1Conflict(%v) = %q, want %q", err, got, reason)
1479	}
1480	}
1481	}
1482
1483	// stuckMove is a move journal stuck in the given way, as the box's own
1484	// transitions leave one (MoveMarkStuck), for the resolve door to act on.
1485	func stuckMove(t *testing.T, fx *consoleFixture, setup func(*provision.MoveState), reason string) provision.MoveGuard {
1486	t.Helper()
1487	g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
1488	ctx := context.Background()
1489	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error {
1490	rec.Move = &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", Cut: "auto", RequestedAt: time.Now().UTC()}
1491	setup(rec.Move)
1492	return nil
1493	}); err != nil {
1494	t.Fatal(err)
1495	}
1496	if reason != "" {
1497	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveMarkStuck(g, reason)); err != nil {
1498	t.Fatal(err)
1499	}
1500	}
1501	return g
1502	}
1503
1504	// resolveRetryBody is the resolve door's retry request.
1505	const resolveRetryBody = `{"retry":true}`
1506
1507	// TestResolveRetryFromStuckOnly (Plan 4 Task 7, §7.10): the resolve door's
1508	// {"retry": true} runs provision.MoveRetry under the guard from the record,
1509	// and answers 409 unless the move is stuck. The retry clears the stuck
1510	// reason only; the box resumes at the phase StuckAt names. The answer is
1511	// the flat view, which carries no reason.
1512	func TestResolveRetryFromStuckOnly(t *testing.T) {
1513	fx, token := operatorInstanceFixture(t, "code-inst-move-retry")
1514	m := titoManifest()
1515	m.Regions = nil
1516	m.Pool = ""
1517	enableInstances(t, fx, "titogo", m)
1518	u := instancesURL(fx, "bab", "titogo")
1519	ctx := context.Background()
1520	if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated {
1521	t.Fatalf("create = %d %s", code, out)
1522	}
1523	resolveURL := u + "/acme.go.tito.io/move/resolve"
1524	const rb = "rb_abcdefghijklmnopqrstuvwxyz"
1525	read := func() *provision.Record {
1526	t.Helper()
1527	rec, err := provision.GetRecord(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io")
1528	if err != nil {
1529	t.Fatal(err)
1530	}
1531	return rec
1532	}
1533	moveBytes := func() string {
1534	t.Helper()
1535	b, err := json.Marshal(read().Move)
1536	if err != nil {
1537	t.Fatal(err)
1538	}
1539	return string(b)
1540	}
1541	clear := func() {
1542	t.Helper()
1543	if _, err := provision.SetRecordMove(ctx, fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error { rec.Move = nil; return nil }); err != nil {
1544	t.Fatal(err)
1545	}
1546	}
1547	serial := func() string {
1548	t.Helper()
1549	b, _, err := fx.rel.Get(ctx, "control/serial")
1550	if err != nil {
1551	return ""
1552	}
1553	return string(b)
1554	}
1555
1556	// No move in flight: the guarded transition's own 409, nothing written.
1557	if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != "no journaled move in flight" {
1558	t.Fatalf("retry with no move = %d %s", code, out)
1559	}
1560
1561	// A move that is not stuck is refused with the recorded sentence and
1562	// writes nothing, in every state a move can be in.
1563	for _, c := range []struct {
1564	name string
1565	setup func(*provision.MoveState)
1566	}{
1567	{"preparing", func(m *provision.MoveState) { m.State = provision.MovePreparing }},
1568	{"ready", func(m *provision.MoveState) { m.State = provision.MoveReady }},
1569	{"cutting", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }},
1570	{"aborting", func(m *provision.MoveState) { m.State = provision.MoveAborting }},
1571	{"landing", func(m *provision.MoveState) { m.State = provision.MoveLanding }},
1572	{"committed", func(m *provision.MoveState) { m.State = provision.MoveCommitted }},
1573	{"rolling back", func(m *provision.MoveState) {
1574	m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, SourcePhase: provision.RbSourceVerified}
1575	}},
1576	{"reverting", func(m *provision.MoveState) { m.State = provision.MoveReverting }},
1577	} {
1578	stuckMove(t, fx, c.setup, "")
1579	before, ser := moveBytes(), serial()
1580	if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck {
1581	t.Fatalf("retry of a move that is %s = %d %s, want %q", c.name, code, out, provision.RefusedNotStuck)
1582	}
1583	if moveBytes() != before || serial() != ser {
1584	t.Fatalf("a refused retry of a move that is %s wrote", c.name)
1585	}
1586	clear()
1587	}
1588
1589	// A body that asks for no retry changes nothing, and a stuck move stays
1590	// stuck. A body that is not JSON is the family's 400.
1591	stuckMove(t, fx, func(m *provision.MoveState) {
1592	m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetCleaned, SourcePhase: provision.RbSourceVerified}
1593	}, provision.StuckBaselineMismatch)
1594	before := moveBytes()
1595	for _, body := range []string{`{}`, `{"retry":false}`} {
1596	if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, body); code != http.StatusConflict || errorText(out) != "nothing to resolve yet" {
1597	t.Fatalf("resolve %s = %d %s", body, code, out)
1598	}
1599	}
1600	if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, `{"retry":`); code != http.StatusBadRequest {
1601	t.Fatalf("resolve with a broken body = %d %s", code, out)
1602	}
1603	if moveBytes() != before {
1604	t.Fatal("a resolve that asked for no retry wrote")
1605	}
1606	clear()
1607
1608	// Stuck rollbacks: each returns to the source phase StuckAt names, with
1609	// the stuck reason cleared and every other field kept.
1610	for _, c := range []struct {
1611	name, reason, at string
1612	}{
1613	{"a baseline mismatch", provision.StuckBaselineMismatch, provision.RbSourceVerified},
1614	{"a recovery probe out of attempts", provision.StuckRecoveryProbeAttempts, provision.RbSourceProbing},
1615	} {
1616	stuckMove(t, fx, func(m *provision.MoveState) {
1617	m.State, m.Reason = provision.MoveRollingBack, provision.RollbackLandingDeadline
1618	m.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetCleaned, SourcePhase: c.at}
1619	}, c.reason)
1620	stuck := read().Move
1621	if stuck.State != provision.MoveStuck || stuck.StuckReason != c.reason || stuck.Rollback.StuckAt != c.at {
1622	t.Fatalf("%s: setup %+v %+v", c.name, stuck, stuck.Rollback)
1623	}
1624	ser := serial()
1625	code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody)
1626	if code != http.StatusAccepted {
1627	t.Fatalf("retry after %s = %d %s", c.name, code, out)
1628	}
1629	var view map[string]any
1630	if err := json.Unmarshal([]byte(out), &view); err != nil {
1631	t.Fatal(err)
1632	}
1633	if len(view) != 6 || view["state"] != provision.MoveRollingBack || view["target"] != "uk" || view["updated_at"] == nil || view["id"] != "mv_abcdefghijklmnopqrstuvwxyz" || view["protocol"] != "phase1" || view["phase"] != provision.RbTargetCleaned+"/"+c.at {
1634	t.Fatalf("retry after %s answered %s, want the flat view in rolling_back", c.name, out)
1635	}
1636	got := read().Move
1637	want := *stuck
1638	want.State, want.StuckReason, want.UpdatedAt = provision.MoveRollingBack, "", got.UpdatedAt
1639	rbWant := *stuck.Rollback
1640	rbWant.SourcePhase, rbWant.StuckAt = c.at, ""
1641	want.Rollback = &rbWant
1642	if !reflect.DeepEqual(*got, want) {
1643	t.Fatalf("retry after %s:\n got %+v / %+v\nwant %+v / %+v", c.name, *got, *got.Rollback, want, rbWant)
1644	}
1645	if serial() == ser {
1646	t.Fatalf("retry after %s did not bump the serial, so the box would not notice", c.name)
1647	}
1648	// Not stuck any more: a second retry is refused.
1649	if code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck {
1650	t.Fatalf("second retry after %s = %d %s", c.name, code, out)
1651	}
1652	clear()
1653	}
1654
1655	// A stuck revert has no rollback to return to: it goes back to
1656	// reverting, and keeps the cause the revert recorded.
1657	stuckMove(t, fx, func(m *provision.MoveState) {
1658	m.State, m.Reason = provision.MoveReverting, provision.RevertCutDeadline
1659	}, provision.StuckRevertProbeBudget)
1660	if got := read().Move; got.State != provision.MoveStuck || got.StuckReason != provision.StuckRevertProbeBudget {
1661	t.Fatalf("revert setup %+v", got)
1662	}
1663	code, _, out := rawReq(t, http.MethodPost, resolveURL, token, resolveRetryBody)
1664	if code != http.StatusAccepted || !strings.Contains(out, `"state":"reverting"`) {
1665	t.Fatalf("retry of a stuck revert = %d %s", code, out)
1666	}
1667	if got := read().Move; got.State != provision.MoveReverting || got.StuckReason != "" || got.Reason != provision.RevertCutDeadline || got.Rollback != nil {
1668	t.Fatalf("a retried revert %+v", got)
1669	}
1670	}
1671
1672	// ---- Plan 6 Task 2: the status door's shape and GET …/move?detail=1 ----
1673
1674	// moveAnswer is a move door's answer as a CLI reads it: the flat view
1675	// (preflight L2: id and last; L4: the Phase 1 marker) and, for an operator's
1676	// detail=1, the detail object.
1677	type moveAnswer struct {
1678	State string `json:"state"`
1679	Target string `json:"target"`
1680	UpdatedAt int64 `json:"updated_at"`
1681	ID string `json:"id"`
1682	Last *provision.MoveOutcome `json:"last"`
1683	Protocol string `json:"protocol"`
1684	Detail *struct {
1685	Move *provision.MoveState `json:"move"`
1686	Claim *struct {
1687	Kind string `json:"kind"`
1688	Phase string `json:"phase"`
1689	Exec string `json:"exec"`
1690	Participants []string `json:"participants"`
1691	Target string `json:"target"`
1692	} `json:"claim"`
1693	TargetReport *moves.TargetReport `json:"target_report"`
1694	SourceReport *struct {
1695	Box string `json:"box"`
1696	MoveID string `json:"move_id"`
1697	S1 moves.Check `json:"s1"`
1698	S2 moves.Check `json:"s2"`
1699	Env []struct {
1700	Name string `json:"name"`
1701	Kind string `json:"kind"`
1702	} `json:"env"`
1703	} `json:"source_report"`
1704	Evidence struct {
1705	Source string `json:"source"`
1706	Target string `json:"target"`
1707	} `json:"evidence"`
1708	Log []string `json:"log"`
1709	Timings map[string]float64 `json:"timings"`
1710	WaitingForBox string `json:"waiting_for_box"`
1711	From string `json:"from"`
1712	Region string `json:"region"`
1713	} `json:"detail"`
1714	}
1715
1716	func readMoveAnswer(t *testing.T, method, u, token, body string, wantCode int) (moveAnswer, string) {
1717	t.Helper()
1718	code, _, out := rawReq(t, method, u, token, body)
1719	if code != wantCode {
1720	t.Fatalf("%s %s = %d %s, want %d", method, u, code, out, wantCode)
1721	}
1722	var a moveAnswer
1723	if err := json.Unmarshal([]byte(out), &a); err != nil {
1724	t.Fatalf("%s %s: %v in %s", method, u, err, out)
1725	}
1726	return a, out
1727	}
1728
1729	// setMove writes the record's move through the record writer, as a test
1730	// fixture (never a direct Put of the record).
1731	func setMove(t *testing.T, fx *consoleFixture, mut func(*provision.Record)) {
1732	t.Helper()
1733	if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", func(rec *provision.Record) error {
1734	mut(rec)
1735	return nil
1736	}); err != nil {
1737	t.Fatal(err)
1738	}
1739	}
1740
1741	func recordMove(t *testing.T, fx *consoleFixture) *provision.MoveState {
1742	t.Helper()
1743	rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io")
1744	if err != nil || rec.Move == nil {
1745	t.Fatalf("record move: %v", err)
1746	}
1747	return rec.Move
1748	}
1749
1750	// Preflight L2 and L4: with the switch on every move answer carries the
1751	// move id and the Phase 1 marker, and after a close the flat view carries
1752	// the outcome (D11's last), for every caller of the flat view, the
1753	// Instance-role token included.
1754	func TestMoveViewCarriesTheIDAndTheOutcome(t *testing.T) {
1755	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-view")
1756	router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
1757	a, out := readMoveAnswer(t, http.MethodGet, moveURL, token, "", http.StatusOK)
1758	if a.State != "none" || a.Protocol != "phase1" || a.ID != "" || a.Last != nil || strings.Contains(out, `"detail"`) {
1759	t.Fatalf("status before any move = %s", out)
1760	}
1761	a, out = readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
1762	id := recordMove(t, fx).ID
1763	if a.ID != id || !provision.ValidMoveID(id) || a.Protocol != "phase1" || a.State != provision.MovePreparing {
1764	t.Fatalf("request answered %s, want id %s and the marker", out, id)
1765	}
1766	for _, who := range []string{token, router} {
1767	if a, out := readMoveAnswer(t, http.MethodGet, moveURL, who, "", http.StatusOK); a.ID != id || a.Protocol != "phase1" || a.Last != nil {
1768	t.Fatalf("status in flight = %s", out)
1769	}
1770	}
1771	if a, out := readMoveAnswer(t, http.MethodDelete, moveURL, token, "", http.StatusAccepted); a.ID != id || a.State != provision.MoveAborting || a.Protocol != "phase1" {
1772	t.Fatalf("abort answered %s", out)
1773	}
1774	at := time.Now().UTC().Truncate(time.Second)
1775	if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io",
1776	provision.MoveGuardedAbortComplete(provision.MoveGuard{ID: id, Incarnation: recordMove(t, fx).Incarnation}, at)); err != nil {
1777	t.Fatal(err)
1778	}
1779	for _, who := range []string{token, router} {
1780	a, out := readMoveAnswer(t, http.MethodGet, moveURL, who, "", http.StatusOK)
1781	if a.State != "none" || a.ID != "" || a.Last == nil || a.Last.ID != id || a.Last.Outcome != "aborted" || !a.Last.At.Equal(at) || a.Protocol != "phase1" {
1782	t.Fatalf("status after the close = %s", out)
1783	}
1784	}
1785	}
1786
1787	// With the switch off the status door is main's, byte for byte: no id, no
1788	// last, no marker, and detail=1 is not read.
1789	func TestMoveViewSwitchOffIsMainsShape(t *testing.T) {
1790	fx, token := instanceFixture(t, "code-inst-move-view-off")
1791	if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
1792	t.Fatal(err)
1793	}
1794	m := titoManifest()
1795	m.Regions = nil
1796	m.Pool = ""
1797	enableInstances(t, fx, "titogo", m)
1798	u := instancesURL(fx, "bab", "titogo")
1799	if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); c != http.StatusCreated {
1800	t.Fatalf("create = %d %s", c, out)
1801	}
1802	moveURL := u + "/acme.go.tito.io/move"
1803	g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
1804	setMove(t, fx, func(rec *provision.Record) {
1805	rec.Move = &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveAborting, RequestedAt: time.Now().UTC()}
1806	})
1807	if _, err := provision.SetRecordMove(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io", provision.MoveGuardedAbortComplete(g, time.Now())); err != nil {
1808	t.Fatal(err)
1809	}
1810	for _, q := range []string{"", "?detail=1"} {
1811	code, _, out := rawReq(t, http.MethodGet, moveURL+q, token, "")
1812	if code != http.StatusOK || strings.TrimSpace(out) != `{"state":"none"}` {
1813	t.Fatalf("switch-off status%s = %d %q, want main's bytes", q, code, out)
1814	}
1815	}
1816	}
1817
1818	// M18 (Task 2 rulings): detail=1 is the operator's. A member's token and
1819	// the Instance-role token get the flat view, with no detail and no error.
1820	func TestMoveDetailIsOperatorOnly(t *testing.T) {
1821	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-dop")
1822	router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
1823	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
1824	id := recordMove(t, fx).ID
1825	a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK)
1826	if a.Detail == nil || a.Detail.Move == nil || a.Detail.Move.ID != id || a.Detail.Claim == nil || a.Detail.Claim.Kind != hostclaim.KindMoving ||
1827	a.Detail.Claim.Exec != "ie-box" || a.Detail.Claim.Target != "uk-box" || a.Protocol != "phase1" || a.ID != id {
1828	t.Fatalf("an operator's detail = %s", out)
1829	}
1830	// The CLI's sentences name the regions as the caller does (Task 1a):
1831	// the move's origin and the record's region now.
1832	if a.Detail.From != "ie" || a.Detail.Region != "ie" {
1833	t.Fatalf("detail from %q region %q, want ie and ie: %s", a.Detail.From, a.Detail.Region, out)
1834	}
1835	flat, _ := readMoveAnswer(t, http.MethodGet, moveURL, token, "", http.StatusOK)
1836	if err := fx.st.SetOperator("alice@keymail.dev", false); err != nil {
1837	t.Fatal(err)
1838	}
1839	for _, who := range []struct{ name, token string }{{"a member", token}, {"an instance token", router}} {
1840	a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", who.token, "", http.StatusOK)
1841	if a.Detail != nil || strings.Contains(out, `"detail"`) {
1842	t.Fatalf("detail=1 as %s = %s, want the flat view", who.name, out)
1843	}
1844	if a.State != flat.State || a.ID != flat.ID || a.Target != flat.Target || a.Protocol != flat.Protocol || a.UpdatedAt != flat.UpdatedAt {
1845	t.Fatalf("detail=1 as %s = %s, want the flat view %+v", who.name, out, flat)
1846	}
1847	}
1848	}
1849
1850	// Review Focus 2: the detail answer carries the reports' check ids, the
1851	// evidence keys and the log keys, and never a launch environment value, an
1852	// evidence or log body, a CARLOS_ value or a secret.
1853	func TestMoveDetailLeaksNothing(t *testing.T) {
1854	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-leak")
1855	ctx := context.Background()
1856	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
1857	mv := recordMove(t, fx)
1858	prefix := moves.ReportPrefix("bab", "titogo", "acme.go.tito.io", mv.ID)
1859	srcKey, dstKey := moves.StoppedKey(prefix, "source"), moves.StoppedKey(prefix, "target")
1860	const rb = "rb_abcdefghijklmnopqrstuvwxyz"
1861	setMove(t, fx, func(rec *provision.Record) {
1862	rec.Move.State, rec.Move.CutPhase = provision.MoveRollingBack, provision.CutArchived
1863	rec.Move.Source = &provision.MoveSource{FinalTXID: 7, DBSHA256: strings.Repeat("d", 64), Evidence: srcKey}
1864	rec.Move.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetReleased, Evidence: dstKey}
1865	})
1866
1867	secretsKey := strings.Repeat("A", 43) + "="
1868	env := map[string]string{
1869	"DATABASE_URL": "postgres://app:hunter2-db-canary@db.internal/app",
1870	"STRIPE_SECRET_KEY": "sk_live_stripe_canary_value",
1871	"CARLOS_SECRETS_KEY": secretsKey,
1872	"CARLOS_APP_TOKEN": "carlos-app-token-canary",
1873	}
1874	hk, err := moves.ReportKeyFromSecrets(secretsKey)
1875	if err != nil {
1876	t.Fatal(err)
1877	}
1878	src := moves.SourceReport{V: 1, Box: "ie-box", MoveID: mv.ID, At: time.Now().UTC(), Env: moves.ReportEnv(hk, env),
1879	S1: moves.Check{OK: true}, S2: moves.Check{OK: true}, Release: "v0.25.0"}
1880	// Malformed reports: raw values under a name, one of them a secret that
1881	// is 64 hex characters (the shape of an HMAC), a digest that is a raw
1882	// secret, and a target report whose env input is one. The console can
1883	// tell none of them from an HMAC, so it passes no env value at all.
1884	const hexSecret = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
1885	const hexDigestSecret = "sha256:60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"
1886	const hexTargetSecret = "fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13"
1887	src.Env.Vars["PLAIN_LEAK"] = "raw-env-value-canary"
1888	src.Env.Vars["HEX_API_SECRET"] = hexSecret
1889	malformed := src
1890	malformed.Env.Digest = hexDigestSecret
1891	dst := moves.TargetReport{V: 1, Box: "uk-box", Region: "uk", MoveID: mv.ID, At: time.Now().UTC(), Checks: map[string]moves.Check{
1892	moves.CheckProtocol: {OK: true}, moves.CheckEnv: {OK: false, Reason: "the launch environment differs"},
1893	}, Inputs: provision.MoveInputs{EnvReport: hexTargetSecret, Release: "v0.25.0"}}
1894	putJSON := func(key string, v any) {
1895	t.Helper()
1896	b, err := json.Marshal(v)
1897	if err != nil {
1898	t.Fatal(err)
1899	}
1900	if _, err := fx.rel.Put(ctx, key, b, ""); err != nil {
1901	t.Fatal(err)
1902	}
1903	}
1904	putJSON(moves.SourceReportKey(prefix), malformed)
1905	putJSON(moves.TargetReportKey(prefix), dst)
1906	putJSON(srcKey, map[string]any{"v": 1, "role": "source", "note": "evidence-body-canary-source", "env": env["DATABASE_URL"]})
1907	putJSON(dstKey, map[string]any{"v": 1, "role": "target", "note": "evidence-body-canary-target"})
1908	logKeys := []string{moves.LogKeyFor(prefix, "requested"), moves.LogKeyFor(prefix, "rolling_back")}
1909	for _, k := range logKeys {
1910	putJSON(k, map[string]any{"event": "x", "note": "log-body-canary", "actor": "alice@keymail.dev"})
1911	}
1912
1913	a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK)
1914	if a.Detail == nil || a.Detail.TargetReport == nil || a.Detail.SourceReport == nil {
1915	t.Fatalf("detail = %s, want both reports", out)
1916	}
1917	leaks := []string{
1918	env["DATABASE_URL"], "hunter2-db-canary", env["STRIPE_SECRET_KEY"], secretsKey, env["CARLOS_APP_TOKEN"],
1919	"evidence-body-canary", "log-body-canary", token, hexDigestSecret, hexTargetSecret,
1920	strings.TrimPrefix(hexDigestSecret, "sha256:"), src.Env.Digest,
1921	}
1922	// No value of either report's env section: every one of the source's
1923	// (the HMACs, the raw canary, the hex secret) and the target's input.
1924	for _, v := range malformed.Env.Vars {
1925	leaks = append(leaks, v)
1926	}
1927	for _, leak := range leaks {
1928	if strings.Contains(out, leak) {
1929	t.Errorf("the detail answer holds %q:\n%s", leak, out)
1930	}
1931	}
1932	for _, want := range []string{moves.CheckProtocol, moves.CheckEnv, `"s1"`, `"s2"`, srcKey, dstKey, "v0.25.0"} {
1933	if !strings.Contains(out, want) {
1934	t.Errorf("the detail answer lacks %q:\n%s", want, out)
1935	}
1936	}
1937	// Each env entry is its name and its kind, nothing else.
1938	got := map[string]string{}
1939	for _, e := range a.Detail.SourceReport.Env {
1940	got[e.Name] = e.Kind
1941	}
1942	want := map[string]string{}
1943	for name := range malformed.Env.Vars {
1944	want[name] = moves.Class(name)
1945	}
1946	if !reflect.DeepEqual(got, want) {
1947	t.Errorf("source env = %v, want the names and kinds %v", got, want)
1948	}
1949	var raw struct {
1950	Detail struct {
1951	SourceReport map[string]json.RawMessage `json:"source_report"`
1952	} `json:"detail"`
1953	}
1954	if err := json.Unmarshal([]byte(out), &raw); err != nil {
1955	t.Fatal(err)
1956	}
1957	var entries []map[string]any
1958	if err := json.Unmarshal(raw.Detail.SourceReport["env"], &entries); err != nil {
1959	t.Fatalf("source env is not a list of entries: %v in %s", err, raw.Detail.SourceReport["env"])
1960	}
1961	for _, e := range entries {
1962	if len(e) != 2 {
1963	t.Errorf("an env entry carries more than its name and kind: %v", e)
1964	}
1965	}
1966	if a.Detail.Evidence.Source != srcKey || a.Detail.Evidence.Target != dstKey {
1967	t.Errorf("evidence = %+v, want the two keys", a.Detail.Evidence)
1968	}
1969	if !reflect.DeepEqual(a.Detail.Log, logKeys) {
1970	t.Errorf("log = %v, want %v", a.Detail.Log, logKeys)
1971	}
1972	}
1973
1974	// Plan 6 Task 4: once a move has closed done or rolled_back, detail=1 shows
1975	// its timings, read from the terminal log event's Fields: the box or the
1976	// console wrote them from the record's own journal times when it built the
1977	// entry (moves.Timings), and the door never works them out a second way.
1978	// Only the timing fields are shown, never the rest of the event; a move with
1979	// no terminal event logged yet, or one still in flight, shows none; a log
1980	// read that fails answers the reports' fixed sentence.
1981	func TestMoveDetailShowsTheTimings(t *testing.T) {
1982	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-timings")
1983	ctx := context.Background()
1984	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
1985	mv := recordMove(t, fx)
1986	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || a.Detail.Timings != nil {
1987	t.Fatalf("a move in flight shows timings: %s", out)
1988	}
1989	prefix := moves.ReportPrefix("bab", "titogo", "acme.go.tito.io", mv.ID)
1990	setMove(t, fx, func(rec *provision.Record) {
1991	rec.Move, rec.LastMove = nil, &provision.MoveOutcome{ID: mv.ID, Outcome: "done", At: time.Now().UTC(), OfflineS: 70.5}
1992	})
1993	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || a.Detail.Timings != nil {
1994	t.Fatalf("timings before the terminal event is logged: %s", out)
1995	}
1996	const digest = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"
1997	ev := moves.LogEvent{V: 1, Op: mv.ID, Host: "acme.go.tito.io", Event: "done", Identity: "done", Actor: "uk-box", At: time.Now().UTC(),
1998	Fields: map[string]string{"from": "ie", "to": "uk", "db_sha256": digest, "prepare_s": "300", "cut_s": "40", "cut_budget_s": "95",
1999	"land_s": "30.5", "land_budget_s": "50", "offline_s": "70.5"}}
2000	if _, err := moves.WriteLog(ctx, fx.rel, prefix, ev); err != nil {
2001	t.Fatal(err)
2002	}
2003	a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK)
2004	want := map[string]float64{"prepare_s": 300, "cut_s": 40, "cut_budget_s": 95, "land_s": 30.5, "land_budget_s": 50, "offline_s": 70.5}
2005	if a.Detail == nil || !reflect.DeepEqual(a.Detail.Timings, want) {
2006	t.Fatalf("detail timings = %s, want %v", out, want)
2007	}
2008	if strings.Contains(out, digest) {
2009	t.Fatalf("the detail shows the event's other fields: %s", out)
2010	}
2011	// A rolled_back close reads its own event, not the done one.
2012	setMove(t, fx, func(rec *provision.Record) { rec.LastMove.Outcome = "rolled_back" })
2013	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil || a.Detail.Timings != nil {
2014	t.Fatalf("a rolled_back close shows the done event's timings: %s", out)
2015	}
2016	rb := ev
2017	rb.Event, rb.Identity, rb.Fields = "rolled_back", "rolled_back", map[string]string{"offline_s": "210", "rollback_s": "135", "release_budget_s": "900"}
2018	if _, err := moves.WriteLog(ctx, fx.rel, prefix, rb); err != nil {
2019	t.Fatal(err)
2020	}
2021	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.Detail == nil ||
2022	!reflect.DeepEqual(a.Detail.Timings, map[string]float64{"offline_s": 210, "rollback_s": 135, "release_budget_s": 900}) {
2023	t.Fatalf("rolled_back timings = %s", out)
2024	}
2025	}
2026
2027	// Relocating (spec rev 7.20 §7.10, D16) and Paul's J3 (preflight L18): a
2028	// move whose claim is relocating/pinning with exec "none" in a region with
2029	// no live enrolled box reports waiting_for_box = that region: the source's
2030	// before the flip, the target's after a committed landing was converted. A
2031	// converted committed landing answers state relocating, in the flat view
2032	// too. "Live enrolled" is boxchoice.Eligible over the live edges snapshot,
2033	// with the roster read through hostactors.Directory: an active entry with
2034	// an enrollment and no termination fence.
2035	// waitingFor is the detail's waiting_for_box, or a marker when the answer
2036	// has no detail at all.
2037	func waitingFor(a moveAnswer) string {
2038	if a.Detail == nil {
2039	return "<no detail>"
2040	}
2041	return a.Detail.WaitingForBox
2042	}
2043
2044	func TestMoveDetailShowsWaitingForBox(t *testing.T) {
2045	relocate := func(region, box string) func(*hostclaim.Claim) {
2046	return func(c *hostclaim.Claim) {
2047	c.Op.Kind, c.Op.Phase, c.Op.Exec, c.Op.Participants = hostclaim.KindRelocating, "pinning", "none", nil
2048	c.Op.Intent = map[string]string{"region": region, "source_box": box}
2049	}
2050	}
2051	// Every phase hostclaim lets run on no box (validateRelocating).
2052	noBoxPhases := []string{"starting", "releasing", "pinning"}
2053	inPhase := func(phase string) func(*hostclaim.Claim) {
2054	return func(c *hostclaim.Claim) { c.Op.Phase, c.Op.Exec = phase, "none" }
2055	}
2056	t.Run("before the flip", func(t *testing.T) {
2057	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-wait-src")
2058	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
2059	mv := recordMove(t, fx)
2060	setMove(t, fx, func(rec *provision.Record) {
2061	rec.Move.Relocation = &provision.MoveRelocationDecision{SourceBox: "ie-box", Region: "ie", PreviousState: rec.Move.State, DecidedAt: time.Now().UTC()}
2062	rec.Move.State = provision.MoveRelocating
2063	})
2064	rewriteClaim(t, fx, "acme.go.tito.io", relocate("ie", "ie-box"))
2065	// ie-box was terminated: its fence is verified, so ie has no live
2066	// enrolled box even though its edges file is fresh.
2067	writeMoveFence(t, fx, moveRosterBox(ieBox("ie-box")), nil)
2068	router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
2069	if a, out := readMoveAnswer(t, http.MethodGet, moveURL, router, "", http.StatusOK); a.State != provision.MoveRelocating || a.ID != mv.ID {
2070	t.Fatalf("flat view = %s", out)
2071	}
2072	for _, phase := range noBoxPhases {
2073	rewriteClaim(t, fx, "acme.go.tito.io", inPhase(phase))
2074	a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK)
2075	if a.State != provision.MoveRelocating || waitingFor(a) != "ie" {
2076	t.Fatalf("detail in %s/none with no box in ie = %s, want waiting_for_box ie", phase, out)
2077	}
2078	}
2079	// A live enrolled box joins ie: nothing is waiting.
2080	armMoveFleet(t, fx, ieBox("ie-box2"))
2081	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box2": moveRosterBox(ieBox("ie-box2")), "uk-box": moveRosterBox(ukBox("uk-box"))}})
2082	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" {
2083	t.Fatalf("detail with a live box in ie = %s", out)
2084	}
2085	// The same box, live but not enrolled in the roster, is not one.
2086	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"uk-box": moveRosterBox(ukBox("uk-box"))}})
2087	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); waitingFor(a) != "ie" {
2088	t.Fatalf("detail with an unenrolled box in ie = %s, want waiting_for_box ie", out)
2089	}
2090	})
2091	t.Run("after the commit (J3)", func(t *testing.T) {
2092	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-wait-dst")
2093	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
2094	now := time.Now().UTC()
2095	setMove(t, fx, func(rec *provision.Record) {
2096	rec.Region = "uk"
2097	rec.Move.State, rec.Move.CutPhase, rec.Move.FlippedAt, rec.Move.CommittedAt = provision.MoveCommitted, provision.CutFlipped, &now, &now
2098	})
2099	// Not converted: the record's own state.
2100	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveCommitted || waitingFor(a) != "" {
2101	t.Fatalf("a committed landing = %s", out)
2102	}
2103	rewriteClaim(t, fx, "acme.go.tito.io", relocate("uk", "uk-box"))
2104	writeMoveFence(t, fx, moveRosterBox(ukBox("uk-box")), nil)
2105	router := serviceToken(t, fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
2106	if a, out := readMoveAnswer(t, http.MethodGet, moveURL, router, "", http.StatusOK); a.State != provision.MoveRelocating {
2107	t.Fatalf("flat view of a converted landing = %s, want relocating", out)
2108	}
2109	for _, phase := range noBoxPhases {
2110	rewriteClaim(t, fx, "acme.go.tito.io", inPhase(phase))
2111	a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK)
2112	if a.State != provision.MoveRelocating || waitingFor(a) != "uk" || (a.Detail != nil && a.Detail.Move.State != provision.MoveCommitted) {
2113	t.Fatalf("detail of a converted landing in %s/none with no box in uk = %s, want waiting_for_box uk", phase, out)
2114	}
2115	}
2116	// A live enrolled box joins uk: still relocating, nothing waiting.
2117	armMoveFleet(t, fx, ukBox("uk-box2"))
2118	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ieBox("ie-box")), "uk-box2": moveRosterBox(ukBox("uk-box2"))}})
2119	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" {
2120	t.Fatalf("detail with a live box in uk = %s", out)
2121	}
2122	// A phase that runs on a box is not waiting for one.
2123	putMoveRoster(t, fx, hostactors.Roster{Boxes: map[string]hostactors.Box{"ie-box": moveRosterBox(ieBox("ie-box"))}})
2124	rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) {
2125	c.Op.Phase, c.Op.Exec, c.Op.Intent["target_box"] = "installing", "uk-box2", "uk-box2"
2126	})
2127	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" {
2128	t.Fatalf("detail of a relocation that is installing = %s", out)
2129	}
2130	rewriteClaim(t, fx, "acme.go.tito.io", func(c *hostclaim.Claim) {
2131	c.Op.Phase, c.Op.Exec = "releasing", "uk-box"
2132	delete(c.Op.Intent, "target_box")
2133	})
2134	if a, out := readMoveAnswer(t, http.MethodGet, moveURL+"?detail=1", token, "", http.StatusOK); a.State != provision.MoveRelocating || waitingFor(a) != "" {
2135	t.Fatalf("detail of a relocation releasing on its source box = %s", out)
2136	}
2137	})
2138	}
2139
2140	// Preflight L5 (claim-off half): once the cut has started (cutting past
2141	// requested, landing, committed, rolling back, reverting) the abort door
2142	// answers M-C13 and writes nothing. Before it, abort is the guarded
2143	// transition; a stuck move keeps provision's own refusal.
2144	func TestMoveAbortAfterTheCutStartedSaysSo(t *testing.T) {
2145	const mc13 = "This move has started its cut. It will finish or move back on its own."
2146	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-mc13")
2147	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
2148	for _, c := range []struct {
2149	name string
2150	mut func(*provision.MoveState)
2151	}{
2152	{"cutting/admitted", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }},
2153	{"cutting/flipped", func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutFlipped }},
2154	{"landing", func(m *provision.MoveState) { m.State = provision.MoveLanding }},
2155	{"committed", func(m *provision.MoveState) { m.State = provision.MoveCommitted }},
2156	{"rolling back", func(m *provision.MoveState) { m.State = provision.MoveRollingBack }},
2157	{"reverting", func(m *provision.MoveState) { m.State = provision.MoveReverting }},
2158	} {
2159	setMove(t, fx, func(rec *provision.Record) { c.mut(rec.Move) })
2160	before, _ := json.Marshal(recordMove(t, fx))
2161	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) != mc13 {
2162	t.Fatalf("abort in %s = %d %s, want M-C13", c.name, code, out)
2163	}
2164	if after, _ := json.Marshal(recordMove(t, fx)); string(after) != string(before) {
2165	t.Fatalf("a refused abort in %s wrote", c.name)
2166	}
2167	}
2168	setMove(t, fx, func(rec *provision.Record) {
2169	rec.Move.State, rec.Move.CutPhase, rec.Move.StuckReason = provision.MoveStuck, provision.CutArchived, provision.StuckBaselineMismatch
2170	})
2171	if code, _, out := rawReq(t, http.MethodDelete, moveURL, token, ""); code != http.StatusConflict || errorText(out) == mc13 {
2172	t.Fatalf("abort of a stuck move = %d %s, want provision's refusal", code, out)
2173	}
2174	setMove(t, fx, func(rec *provision.Record) {
2175	rec.Move.State, rec.Move.CutPhase, rec.Move.StuckReason = provision.MoveCutting, provision.CutRequested, ""
2176	})
2177	if a, out := readMoveAnswer(t, http.MethodDelete, moveURL, token, "", http.StatusAccepted); a.State != provision.MoveAborting {
2178	t.Fatalf("abort from cutting/requested = %s", out)
2179	}
2180	}
2181
2182	// Task 1a fix round 1 (L19): with the switch on the flat view carries the
2183	// move's phase, so a poller reads detail=1 only when the state or the phase
2184	// changes.
2185	func TestMoveViewCarriesThePhase(t *testing.T) {
2186	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-phase")
2187	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
2188	const rb = "rb_abcdefghijklmnopqrstuvwxyz"
2189	for _, c := range []struct {
2190	mut func(*provision.MoveState)
2191	want string
2192	}{
2193	{func(m *provision.MoveState) { m.State = provision.MovePreparing }, ""},
2194	{func(m *provision.MoveState) { m.State, m.CutPhase = provision.MoveCutting, provision.CutAdmitted }, provision.CutAdmitted},
2195	{func(m *provision.MoveState) { m.State, m.LandingPhase = provision.MoveLanding, provision.LandProbing }, provision.LandProbing},
2196	{func(m *provision.MoveState) {
2197	m.State, m.Rollback = provision.MoveRollingBack, &provision.MoveRollback{ID: rb, TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceVerified}
2198	}, provision.RbTargetReleasing + "/" + provision.RbSourceVerified},
2199	} {
2200	setMove(t, fx, func(rec *provision.Record) { c.mut(rec.Move) })
2201	code, _, out := rawReq(t, http.MethodGet, moveURL, token, "")
2202	var v map[string]any
2203	if code != http.StatusOK || json.Unmarshal([]byte(out), &v) != nil {
2204	t.Fatalf("status = %d %s", code, out)
2205	}
2206	got, has := v["phase"]
2207	if (c.want == "" && has) || (c.want != "" && got != c.want) {
2208	t.Fatalf("phase = %v (present %v), want %q: %s", got, has, c.want, out)
2209	}
2210	}
2211	}
2212
2213	// Task 1b (preflight L7, MV-N11): a console without the host claim cannot run
2214	// the claim's loss protocol, so the resolve door answers a declare-lost with
2215	// MV-N11 and writes nothing; a retry is unchanged.
2216	func TestResolveDeclareLostWithoutTheClaim(t *testing.T) {
2217	fx, token, moveURL := moveDoorWithClaim(t, "code-inst-move-dl-off")
2218	readMoveAnswer(t, http.MethodPost, moveURL, token, `{"region":"uk","cut":"manual"}`, http.StatusAccepted)
2219	before, _ := json.Marshal(recordMove(t, fx))
2220	code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"declare_lost":"uk-box"}`)
2221	if code != http.StatusConflict || errorText(out) != "declare-lost is not enabled in this deployment yet" {
2222	t.Fatalf("declare-lost without the claim = %d %s", code, out)
2223	}
2224	if after, _ := json.Marshal(recordMove(t, fx)); string(after) != string(before) {
2225	t.Fatal("a refused declare-lost wrote")
2226	}
2227	if code, _, out := rawReq(t, http.MethodPost, moveURL+"/resolve", token, `{"retry":true}`); code != http.StatusConflict || errorText(out) != provision.RefusedNotStuck {
2228	t.Fatalf("retry = %d %s", code, out)
2229	}
2230	}
2231