carlos / platform Public

Download this file

1	package host
2
3	import (
4	"context"
5	"encoding/json"
6	"errors"
7	"fmt"
8	"os"
9	"path/filepath"
10	"sort"
11	"strconv"
12	"strings"
13	"sync"
14	"testing"
15	"time"
16
17	"github.com/carlosframework/platform/internal/activator"
18	"github.com/carlosframework/platform/internal/genselect"
19	"github.com/carlosframework/platform/internal/hostactors"
20	"github.com/carlosframework/platform/internal/hostbox"
21	"github.com/carlosframework/platform/internal/hostclaim"
22	"github.com/carlosframework/platform/internal/hostfiles"
23	"github.com/carlosframework/platform/internal/hostops"
24	"github.com/carlosframework/platform/internal/instancekey"
25	"github.com/carlosframework/platform/internal/lease"
26	"github.com/carlosframework/platform/internal/moves"
27	"github.com/carlosframework/platform/internal/park"
28	"github.com/carlosframework/platform/internal/provision"
29	"github.com/carlosframework/platform/internal/registry"
30	"github.com/carlosframework/platform/internal/release"
31	"github.com/superfly/ltx"
32	)
33
34	// startIntoHomeT3 writes the record and claim for a move from the harness's
35	// named source (ie-box in ie) into the home placement, whose box is
36	// home-box: the record's move names the home's code, "", and the claim's
37	// target region is the placement-derived engine region, @home. The home box
38	// publishes its edges file with no placement, as a home box does.
39	func startIntoHomeT3(t *testing.T, h *phase1Harness) {
40	t.Helper()
41	ctx := context.Background()
42	if h.src == "" {
43	t.Fatal("startIntoHomeT3 needs a named source")
44	}
45	if _, err := h.s.Put(ctx, "control/edges/home-box.json",
46	[]byte(`{"v":1,"box":"home-box","eip":"1.2.3.9","data_bucket":"data-home","data_bucket_region":"eu-west-1","hosts":{}}`), ""); err != nil {
47	t.Fatal(err)
48	}
49	rec := provision.Record{Schema: 1, Host: p1Host, Slug: "s", App: "app", Account: "bab", Region: h.src, Channel: "stable", CreatedAt: p1Created}
50	if err := provision.CreateRecord(ctx, h.s, rec); err != nil {
51	t.Fatal(err)
52	}
53	if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveStartTransition(provision.MoveStart{
54	ID: p1ID, Incarnation: h.inc, From: h.src, To: "", Cut: "manual", By: "op", Now: h.clock.Now()})); err != nil {
55	t.Fatal(err)
56	}
57	obs, err := hostclaim.Update(ctx, h.s, hostclaim.Observed{Host: p1Host, Absent: true}, func(c *hostclaim.Claim) error {
58	c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "app", p1Host), Account: "bab", App: "app",
59	Incarnation: h.inc, CreatedAt: p1Created, Location: "instances/" + p1Host + "/" + h.inc, Generation: "cur", Stamp: 1}
60	c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"}
61	c.Touched = hostclaim.Touched{Complete: true, Regions: []string{park.PlacementRegion(h.src)}, Boxes: []string{"ie-box"}}
62	return nil
63	})
64	if err != nil {
65	t.Fatal(err)
66	}
67	if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.BeginMove(p1ID, "op", "ie-box", "home-box", park.PlacementRegion(""), h.clock.Now())); err != nil {
68	t.Fatal(err)
69	}
70	}
71
72	// A home box sets no CARLOS_REGION: agentOwnedVars adds it only when the
73	// box has a region, and removes a tenant-provided one otherwise. T3 runs
74	// here on both sides' production compositions: the source's launch report
75	// is its ExecBackend's LaunchEnvironment with region ie, and the home box's
76	// held environment is composeHeldEnv with the home box's own region, "", as
77	// armPhase1 wires it. Before home part 2's T3 rule the target lacked the
78	// source's CARLOS_REGION and failed region-missing; the harness's default
79	// fixture sets CARLOS_REGION=uk and would have hidden that.
80	func TestHomeTargetT3FromTheRealComposition(t *testing.T) {
81	h := newPhase1Harness(t)
82	src := &activator.ExecBackend{DataDir: h.cfg.DataDir, Region: h.src}
83	h.cfg.LaunchEnv = func(string) (map[string]string, map[string]bool, error) {
84	env, instance, _, err := src.LaunchLayers(h.route)
85	return env, instance, err
86	}
87	startIntoHomeT3(t, h)
88	h.pass(t)
89	if _, ok := h.sourceReport(t).Env.Vars[moves.RegionVar]; !ok {
90	t.Fatal("the named source's launch report carries no CARLOS_REGION; the witness needs it")
91	}
92
93	tc := h.targetCfg()
94	tc.Region, tc.Box, tc.OwnBucket = "", "home-box", "data-home"
95	tc.ComposeHeldEnv = composeHeldEnv(tc.SecretsPriv, tc.Region, false)
96	h.targetPass(t, tc)
97	r := h.targetReport(t)
98	if r.Box != "home-box" || r.Region != "" {
99	t.Fatalf("the report is not the home box's: box %q region %q", r.Box, r.Region)
100	}
101	if t3 := r.Checks[moves.CheckEnv]; !t3.OK {
102	t.Fatalf("T3 on a home target that sets no region: %+v", t3)
103	}
104	if r.Inputs.EnvReport != h.sourceReport(t).Env.Digest {
105	t.Fatalf("a passing T3 binds the source's env report: %q", r.Inputs.EnvReport)
106	}
107	}
108
109	// A box that sets CARLOS_REGION is not the home box, even when a move into
110	// home names it: T3 fails it region-present (Review Focus 5), here with the
111	// same production composer given a region.
112	func TestHomeTargetT3RefusesABoxThatSetsARegion(t *testing.T) {
113	h := newPhase1Harness(t)
114	src := &activator.ExecBackend{DataDir: h.cfg.DataDir, Region: h.src}
115	h.cfg.LaunchEnv = func(string) (map[string]string, map[string]bool, error) {
116	env, instance, _, err := src.LaunchLayers(h.route)
117	return env, instance, err
118	}
119	startIntoHomeT3(t, h)
120	h.pass(t)
121
122	tc := h.targetCfg()
123	tc.Region, tc.Box, tc.OwnBucket = "", "home-box", "data-home"
124	tc.ComposeHeldEnv = composeHeldEnv(tc.SecretsPriv, "uk", false)
125	h.targetPass(t, tc)
126	t3 := h.targetReport(t).Checks[moves.CheckEnv]
127	if want := t3Differs([]string{moves.RegionVar + " (region-present)"}); t3.OK || t3.Reason != want {
128	t.Fatalf("T3 = %+v, want %q", t3, want)
129	}
130	}
131
132	// Home part 2, Task 3: the commit into home, through the production owners.
133	// Every witness below runs the move ca -> home on the intoHome fixture: the
134	// source ca-box in ca, the target home-box whose roster name is ie, whose
135	// placement code is "" and whose engine region for the move is @home.
136
137	// claimWatch is the harness's control store with a recorder on the host
138	// claim: every claim a transition writes is decoded and kept, so a chore
139	// the run created is seen even after an acknowledgement removed it, and so
140	// are every touched region and box and every quiet_after key the claim ever
141	// held. It embeds the DirStore so the store's optional interfaces stay.
142	type claimWatch struct {
143	*release.DirStore
144	mu sync.Mutex
145	writes []hostclaim.Claim
146	}
147
148	func (w *claimWatch) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
149	etag, err := w.DirStore.Put(ctx, key, body, ifMatch)
150	if err == nil && key == "control/hosts/"+p1Host+".json" {
151	var c hostclaim.Claim
152	if jerr := json.Unmarshal(body, &c); jerr == nil {
153	w.mu.Lock()
154	w.writes = append(w.writes, c)
155	w.mu.Unlock()
156	}
157	}
158	return etag, err
159	}
160
161	// since is every claim written after the n-th write.
162	func (w *claimWatch) since(n int) []hostclaim.Claim {
163	w.mu.Lock()
164	defer w.mu.Unlock()
165	return append([]hostclaim.Claim(nil), w.writes[n:]...)
166	}
167
168	// watchClaims puts a claimWatch in front of the harness's store before
169	// anything captures it.
170	func watchClaims(out **claimWatch) p1Option {
171	return func(h *phase1Harness) {
172	w := &claimWatch{DirStore: h.s.(*release.DirStore)}
173	h.s = w
174	*out = w
175	}
176	}
177
178	// stagingLeaseKeys records the key of every staging lease the harness's
179	// boxes take, as the production hook keys it (stagingLeaseKey over the
180	// record code the round passes).
181	type stagingLeaseKeys struct {
182	mu sync.Mutex
183	keys []string
184	}
185
186	func (k *stagingLeaseKeys) wrap(c *moveCfg) {
187	inner := c.StagingLease
188	c.StagingLease = func(ctx context.Context, code, host string) (moves.SafeUntilGuard, func(context.Context) error, error) {
189	key, err := stagingLeaseKey(code, host)
190	if err != nil {
191	return nil, nil, err
192	}
193	k.mu.Lock()
194	k.keys = append(k.keys, key)
195	k.mu.Unlock()
196	return inner(ctx, code, host)
197	}
198	}
199
200	func (k *stagingLeaseKeys) all() []string {
201	k.mu.Lock()
202	defer k.mu.Unlock()
203	return append([]string(nil), k.keys...)
204	}
205
206	// homeLeaseKey is the home lane's staging lease for the harness's host.
207	var homeLeaseKey = "control/staging-leases/" + park.HomeRegion + "/" + p1Host
208
209	// onlyTheHomeToken is TestHomeTargetFreshStateShowsOnlyTheHomeToken's
210	// assertion over a witness that ran from clean claim state (the start CAS
211	// wrote intent.to @home and home_region ie, and nothing else names home):
212	// every chore created for the target side names @home and is owned by
213	// region:@home, touched only ever gained @home and the target box beside
214	// the source's, no quiet_after key is ie or "", and every staging lease
215	// taken for the home lane is keyed @home. A supplied ie would show here as
216	// a chore, a touched region, a quiet key or a lease key of its own.
217	func onlyTheHomeToken(t *testing.T, writes []hostclaim.Claim, leases []string, src, srcBox string) {
218	t.Helper()
219	if len(writes) == 0 {
220	t.Fatal("fixture: the witness wrote no claim")
221	}
222	seen := map[string]bool{}
223	for _, c := range writes {
224	for _, o := range c.Obligations {
225	if seen[o.ID] {
226	continue
227	}
228	seen[o.ID] = true
229	switch {
230	case o.Kind == "park-local":
231	if o.Owner != "box:"+o.Box || o.Region != "" {
232	t.Fatalf("a park-local chore %+v", o)
233	}
234	case o.Region == src:
235	if o.Owner != "region:"+src {
236	t.Fatalf("a source-side chore %+v", o)
237	}
238	case o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion:
239	t.Fatalf("a target-side chore not named %s: %+v", park.HomeRegion, o)
240	}
241	}
242	for _, r := range c.Touched.Regions {
243	if r != src && r != park.HomeRegion {
244	t.Fatalf("touched gained %q: %+v", r, c.Touched)
245	}
246	}
247	for _, b := range c.Touched.Boxes {
248	if b != srcBox && b != "home-box" {
249	t.Fatalf("touched gained box %q: %+v", b, c.Touched)
250	}
251	}
252	for k := range c.QuietAfter {
253	if k != src && k != park.HomeRegion {
254	t.Fatalf("quiet_after gained %q: %v", k, c.QuietAfter)
255	}
256	}
257	if c.Op.Intent != nil && c.Op.Intent["to"] != "" && c.Op.Intent["to"] != park.HomeRegion {
258	t.Fatalf("the op's to is %q", c.Op.Intent["to"])
259	}
260	}
261	for _, k := range leases {
262	if k != homeLeaseKey && k != "control/staging-leases/"+src+"/"+p1Host {
263	t.Fatalf("a staging lease keyed %q", k)
264	}
265	}
266	}
267
268	// laneKeysIn lists the keys under the move's staging lane in bucket.
269	func laneKeysIn(h *phase1Harness, bucket, id string) []string {
270	prefix := moves.StagingPrefix(p1Host, id)
271	var keys []string
272	for k := range h.f.buckets[bucket] {
273	if rest, ok := strings.CutPrefix(k, prefix); ok {
274	keys = append(keys, rest)
275	}
276	}
277	sort.Strings(keys)
278	return keys
279	}
280
281	// TestMoveIntoHomeStagesTheHomeLane: the source round on ca-box stages the
282	// lane into the home box's bucket, data-home, under moves/<host>/<id>/,
283	// under the staging lease of the home lane and no other. Touch is a union:
284	// from a claim that names no home it adds @home and home-box; with ie and
285	// home-box already there it writes nothing; with ie and no home-box it adds
286	// @home and home-box and keeps ie. The only region it ever adds is @home.
287	func TestMoveIntoHomeStagesTheHomeLane(t *testing.T) {
288	type tc struct {
289	name string
290	opts []p1Option
291	seedBox bool
292	writes int
293	}
294	for _, c := range []tc{
295	{name: "touched names no home", writes: 1},
296	{name: "touched holds ie and home-box", opts: []p1Option{retainTouched("ie")}, seedBox: true, writes: 0},
297	{name: "touched holds ie without home-box", opts: []p1Option{retainTouched("ie")}, writes: 1},
298	} {
299	t.Run(c.name, func(t *testing.T) {
300	var watch *claimWatch
301	h := newPhase1Harness(t, append([]p1Option{intoHome, watchClaims(&watch)}, c.opts...)...)
302	leases := &stagingLeaseKeys{}
303	leases.wrap(&h.cfg)
304	h.start(t, "manual")
305	if c.seedBox {
306	rewriteP1Claim(t, h.s, func(cl *hostclaim.Claim) { cl.Touched.Boxes = append(cl.Touched.Boxes, "home-box") })
307	}
308	before := h.claim(t)
309	beforeTouched, _ := json.Marshal(before.Touched)
310	var atFirst *hostclaim.Claim
311	h.f.failAt = func(op, key string) error {
312	if atFirst == nil {
313	atFirst = h.claim(t)
314	}
315	return nil
316	}
317	h.pass(t)
318	if atFirst == nil {
319	t.Fatal("no staging write happened")
320	}
321	if d := int(atFirst.Gen - before.Gen); d != c.writes {
322	t.Fatalf("%d claim writes before the first staging write, want %d (touched %+v -> %+v)", d, c.writes, before.Touched, atFirst.Touched)
323	}
324	after := h.claim(t)
325	for _, r := range before.Touched.Regions {
326	if !containsString(after.Touched.Regions, r) {
327	t.Fatalf("touched lost %q: %+v -> %+v", r, before.Touched, after.Touched)
328	}
329	}
330	for _, b := range before.Touched.Boxes {
331	if !containsString(after.Touched.Boxes, b) {
332	t.Fatalf("touched lost box %q: %+v -> %+v", b, before.Touched, after.Touched)
333	}
334	}
335	for _, r := range after.Touched.Regions {
336	if !containsString(before.Touched.Regions, r) && r != park.HomeRegion {
337	t.Fatalf("touched gained %q: %+v -> %+v", r, before.Touched, after.Touched)
338	}
339	}
340	if !containsString(after.Touched.Boxes, "home-box") {
341	t.Fatalf("touched has no home-box: %+v", after.Touched)
342	}
343	if c.writes == 0 {
344	if afterTouched, _ := json.Marshal(after.Touched); string(afterTouched) != string(beforeTouched) {
345	t.Fatalf("touched was rewritten: %s -> %s", beforeTouched, afterTouched)
346	}
347	} else if !containsString(after.Touched.Regions, park.HomeRegion) {
348	t.Fatalf("the Touch did not add %s: %+v", park.HomeRegion, after.Touched)
349	}
350	// The lane is in data-home under moves/<host>/<id>/, and every
351	// staging request went there.
352	keys := laneKeysIn(h, "data-home", p1ID)
353	for _, k := range []string{"move.json", "replica/0000/0000000000000001-0000000000000002.ltx", "replica/0001/0000000000000001-0000000000000002.ltx"} {
354	if !containsString(keys, k) {
355	t.Fatalf("the home lane lacks %s: %v", k, keys)
356	}
357	}
358	for _, call := range h.f.stagingCalls() {
359	if call.Bucket != "data-home" || !strings.HasPrefix(call.Key, moves.StagingPrefix(p1Host, p1ID)) {
360	t.Fatalf("a staging request outside the home lane: %+v", call)
361	}
362	}
363	// The lease of the home lane, and no other.
364	got := leases.all()
365	if len(got) == 0 {
366	t.Fatal("the round took no staging lease")
367	}
368	for _, k := range got {
369	if k != homeLeaseKey {
370	t.Fatalf("a staging lease keyed %q, want %q", k, homeLeaseKey)
371	}
372	}
373	if c.name == "touched names no home" {
374	onlyTheHomeToken(t, watch.since(0), got, "ca", "ca-box")
375	}
376	})
377	}
378	}
379
380	// TestMoveIntoHomeTargetChecksPass: on home-box, every readiness check is
381	// ok over the production composition (the home box sets no region). T8
382	// names the home by its name, ie, when it fails; T10 probes the source's
383	// bucket, data-ca; T12 refuses inside a quiet window recorded under ie, and
384	// again under @home, naming ie.
385	func TestMoveIntoHomeTargetChecksPass(t *testing.T) {
386	t.Parallel()
387	newHarness := func(t *testing.T) (*phase1Harness, moveCfg, *[]string) {
388	h := newPhase1Harness(t, intoHome, realLaunchEnv)
389	h.start(t, "manual")
390	h.pass(t)
391	tc := h.targetCfg()
392	probed := &[]string{}
393	tc.LaneProbe = func(_ context.Context, bucket, region, id string) error {
394	*probed = append(*probed, bucket+"/"+region+"/"+id)
395	return nil
396	}
397	return h, tc, probed
398	}
399	h, tc, probed := newHarness(t)
400	h.targetPass(t, tc)
401	r := h.targetReport(t)
402	if r.Box != "home-box" || r.Region != "" {
403	t.Fatalf("the report is not the home box's: box %q region %q", r.Box, r.Region)
404	}
405	for _, id := range []string{moves.CheckProtocol, moves.CheckSecrets, moves.CheckEnv, moves.CheckStorage, moves.CheckArchive, moves.CheckDisk, moves.CheckRelease,
406	moves.CheckOneOwner, moves.CheckClaim, moves.CheckLane, moves.CheckBudget, moves.CheckQuiet, moves.CheckArmed} {
407	if c, ok := r.Checks[id]; !ok || !c.OK {
408	t.Fatalf("%s on the home box: %+v (present %v)", id, c, ok)
409	}
410	}
411	if ok, why := r.Passing(); !ok {
412	t.Fatalf("the home box's report does not pass: %s", why)
413	}
414	if len(*probed) == 0 {
415	t.Fatal("T10 probed nothing")
416	}
417	for _, p := range *probed {
418	if p != "data-ca/ca-central-1/"+p1ID {
419	t.Fatalf("T10 probed %s, want the source's bucket data-ca", p)
420	}
421	}
422	// T8 names the home by its name: a second placementless box makes the
423	// home ambiguous, and the reason says ie, never @home or "".
424	if _, err := h.s.Put(t.Context(), "control/edges/home-box-2.json",
425	[]byte(`{"v":1,"box":"home-box-2","eip":"1.2.3.10","data_bucket":"data-home-2","data_bucket_region":"eu-west-1","hosts":{}}`), ""); err != nil {
426	t.Fatal(err)
427	}
428	h.clock.Advance(time.Second)
429	h.targetPass(t, tc)
430	if t8 := h.targetReport(t).Checks[moves.CheckOneOwner]; t8.OK || t8.Reason != t8Count(2, "ie") {
431	t.Fatalf("T8 with two home boxes: %+v, want %q", t8, t8Count(2, "ie"))
432	}
433
434	// T12 inside a quiet window, recorded under either spelling.
435	for _, spelling := range []string{"ie", park.HomeRegion} {
436	h, tc, _ := newHarness(t)
437	until := h.clock.now.Add(time.Hour).UTC()
438	rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.QuietAfter = map[string]time.Time{spelling: until} })
439	h.targetPass(t, tc)
440	if t12 := h.targetReport(t).Checks[moves.CheckQuiet]; t12.OK || t12.Reason != t12NotQuiet("ie", until) {
441	t.Fatalf("T12 with quiet_after under %s: %+v, want %q", spelling, t12, t12NotQuiet("ie", until))
442	}
443	}
444	}
445
446	// intoHomeClose is the close harness for the move ca -> home with the
447	// production probe mode (Task 2): the real cut, flip and landing through
448	// restored, the probe through movesProbeOwners over the claim runtime.
449	// With residue, an earlier home stay of this incarnation is seeded in
450	// data-home first (a root generation and one named generation, the
451	// landing harness's residue), so the landing's parking step has work. It
452	// returns the residue's keys.
453	func intoHomeClose(t *testing.T, residue bool, opts ...p1Option) *intoHomeRun {
454	t.Helper()
455	return intoHomeCloseWith(t, residue, nil, opts...)
456	}
457
458	// intoHomeCloseWith is intoHomeClose with afterFlip run on the landing
459	// harness between the flip and the first landing pass.
460	func intoHomeCloseWith(t *testing.T, residue bool, afterFlip func(*landingHarness), opts ...p1Option) *intoHomeRun {
461	t.Helper()
462	r := &intoHomeRun{leases: &stagingLeaseKeys{}}
463	lh := newLandingHarness(t, append([]p1Option{intoHome, realLaunchEnv, watchClaims(&r.watch)}, opts...)...)
464	// Both boxes' configurations derive from the source's (targetCfg), so
465	// one wrap records every staging lease the run takes.
466	r.leases.wrap(&lh.cfg)
467	if residue {
468	r.residue = earlierHomeStay(t, lh)
469	}
470	lh.flip(t)
471	if afterFlip != nil {
472	afterFlip(lh)
473	}
474	h := newFinalHarnessFlipped(t, lh)
475	if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, func(rec *provision.Record) error {
476	rec.Move.CutAttempts = 2
477	return nil
478	}); err != nil {
479	t.Fatal(err)
480	}
481	h.ownAuthorize = true
482	r.h, r.p = h, h.productionProbe(t)
483	// Before the probe the home box's env file gains bytes the prepared
484	// manifest does not carry (a comment line, which changes no variable
485	// and so not the env pin), and an adoption record sits beside it, as
486	// TestLandingWakesEstablishThroughC8 stages them: the establishment then
487	// has a revision to commit and a local effect to show.
488	envPath := filepath.Join(h.tdir, p1Host+".env")
489	before, err := os.ReadFile(envPath)
490	if err != nil {
491	t.Fatal(err)
492	}
493	r.env = append(append([]byte(nil), before...), []byte("# established on home-box\n")...)
494	if err := os.WriteFile(envPath, r.env, 0o600); err != nil {
495	t.Fatal(err)
496	}
497	r.adoption = filepath.Join(h.tdir, p1Host+instancekey.AdoptionSuffix)
498	if err := os.WriteFile(r.adoption, []byte(`{"v":2}`), 0o600); err != nil {
499	t.Fatal(err)
500	}
501	return r
502	}
503
504	// intoHomeRun is one into-home run: the close harness, its production
505	// probe, the residue planted, the env file and adoption record staged for
506	// C8, and the recorders of the claims written and the staging leases taken.
507	type intoHomeRun struct {
508	h *finalHarness
509	p *prodProbe
510	residue []string
511	env []byte
512	adoption string
513	watch *claimWatch
514	leases *stagingLeaseKeys
515	}
516
517	// earlierHomeStay plants an earlier stay of this incarnation in data-home
518	// at the holder's location, as the landing harness's residue does: the
519	// root generation's level file, two level files of the named generation
520	// old, and the earlier stay's generation.json, here a marker that names
521	// old (the probe reads the location's current marker, and a home box that
522	// served the instance before left a well-formed one).
523	func earlierHomeStay(t *testing.T, h *landingHarness) []string {
524	t.Helper()
525	ctx := context.Background()
526	marker, err := json.Marshal(genselect.Marker{Op: "old", Incarnation: h.inc, Host: p1Host, Box: "home-box", InstalledAt: h.clock.now.Add(-48 * time.Hour).UTC()})
527	if err != nil {
528	t.Fatal(err)
529	}
530	keys := []string{
531	cutRoot + "/g-old/0000/0000000000000001-0000000000000001.ltx",
532	cutRoot + "/g-old/0009/0000000000000001-0000000000000001.ltx",
533	cutRoot + "/0000/0000000000000001-0000000000000001.ltx",
534	cutRoot + "/generation.json",
535	}
536	for _, k := range keys {
537	body := []byte("residue " + k)
538	if strings.HasSuffix(k, "/generation.json") {
539	body = marker
540	}
541	if _, err := h.vs.PutIfAbsent(ctx, "data-home", k, body); err != nil {
542	t.Fatal(err)
543	}
544	}
545	return keys
546	}
547
548	// TestMoveIntoHomeCommitsAndCloses: the whole move ca -> home through the
549	// real cut, flip and landing, then the production probe owners and the
550	// claim runtime, as TestMoveFromHomeClosesThroughTheSourcePark does for
551	// part 1 with the fake probe.
552	func TestMoveIntoHomeCommitsAndCloses(t *testing.T) {
553	r := intoHomeClose(t, true)
554	// While the attempt is live, the owners' admission evidence (the one the
555	// writer was bound to) accepts an admission on home-box in ie against
556	// the op's @home, and refuses the same admission in another region.
557	checked := false
558	r.p.afterMaterialize = func(int) error {
559	ctx := context.Background()
560	o, err := hostclaim.Read(ctx, r.h.s, p1Host)
561	if err != nil {
562	return err
563	}
564	in, err := r.p.owners.Select(ctx, r.h.landingHarness.route, o, poProbe)
565	if err != nil {
566	return err
567	}
568	if in.Admission.Region != "ie" || o.Claim.Op.Intent["to"] != park.HomeRegion {
569	t.Fatalf("fixture: an admission in %q against the op's %q", in.Admission.Region, o.Claim.Op.Intent["to"])
570	}
571	if err := r.p.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil {
572	t.Fatalf("the evidence refused ie against %s: %v", park.HomeRegion, err)
573	}
574	bad := in.Admission
575	bad.Region = "ca"
576	if err := r.p.owners.AdmissionEvidence(ctx, o, bad, in.Baseline); !errors.Is(err, hostbox.ErrChanged) {
577	t.Fatalf("the evidence accepted an admission in ca: %v", err)
578	}
579	checked = true
580	return nil
581	}
582	intoHomeCommitsAndCloses(t, r, 1)
583	if !checked {
584	t.Fatal("the evidence was never checked")
585	}
586	intoHomeMovesOutAgain(t, r)
587	}
588
589	// homeRegional is the home box's regional runner as the claim assembly
590	// builds it (claimassembly.go): hostbox.Regional over a park engine in the
591	// box's roster region ie with the fleet's home designation ie (so it owns
592	// @home too), on data-home, with the real moves park view seen from ie. The
593	// claim's authorizer and the regional owner check stand in as allowing.
594	func homeRegional(h *finalHarness) hostbox.Regional {
595	return regionalRunner(h.phase1Harness, h.leases, "ie", "ie", "data-home")
596	}
597
598	// regionalRunner is a box's regional runner as the claim assembly builds
599	// it: hostbox.Regional over a park engine in the box's roster region with
600	// the box's verified home binding (home, "" for a named box), on its data
601	// bucket, with the real moves park view seen from its region.
602	func regionalRunner(h *phase1Harness, leases park.Leases, region, home, bucket string) hostbox.Regional {
603	return hostbox.Regional{Engine: park.Engine{HomeRegion: home, Store: h.vs, Claims: h.s, Bucket: bucket, Region: region, Leases: claimParkLeases(leases),
604	Now: h.clock.Now, NewID: obligationID, View: movesParkView(MovesRecordsFrom(h.s), h.vs, bucket, region), Authorize: allowParkAuthorize},
605	Owner: func(context.Context, hostclaim.Observed) error { return nil }}
606	}
607
608	// residueChores is the op's park-generation chores, by generation.
609	func residueChores(c *hostclaim.Claim) map[string]hostclaim.Obligation {
610	out := map[string]hostclaim.Obligation{}
611	for _, o := range c.Obligations {
612	if o.Op == p1ID && o.Kind == "park-generation" {
613	out[o.Generation] = o
614	}
615	}
616	return out
617	}
618
619	// intoHomeCommitsAndCloses drives an intoHomeClose run from restored (or
620	// probed) through the probe, the commit and the close, asserting each
621	// step. attempts is how many probe attempts the runtime admitted by the
622	// commit.
623	func intoHomeCommitsAndCloses(t *testing.T, r *intoHomeRun, attempts int) {
624	t.Helper()
625	ctx := context.Background()
626	h, p, residue, env, adoption := r.h, r.p, r.residue, r.env, r.adoption
627	// The landing's parking step queued one park-generation per residue
628	// generation, in @home, owned by region:@home, and none for the
629	// install's own.
630	if residue != nil {
631	gens := residueChores(h.claim(t))
632	if len(gens) != 2 {
633	t.Fatalf("residue chores: %+v", gens)
634	}
635	for _, g := range []string{hostclaim.RootGeneration, "old"} {
636	o, ok := gens[g]
637	if !ok || o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion || o.Park == nil || o.Park.Location != cutRoot {
638	t.Fatalf("the residue chore for %s: %+v", g, o)
639	}
640	}
641	if _, ok := gens[p1ID]; ok {
642	t.Fatal("the landing queued a park of its own generation")
643	}
644	}
645
646	// The probe, the commit and the open, through the production owners.
647	h.land(t)
648	h.committedAndOpen(t)
649	if len(p.admissions) != attempts {
650	t.Fatalf("admissions through the claim runtime: %+v", p.admissions)
651	}
652	a := p.admissions[len(p.admissions)-1]
653	if a.Box != "home-box" || a.Region != "ie" || a.Mode != "cold" || (attempts == 1 && a.Generation != p1ID) {
654	t.Fatalf("the probe's admission: %+v", a)
655	}
656	c := h.claim(t)
657	if c.Op.Intent["to"] != park.HomeRegion || c.Op.Intent["home_region"] != "ie" || c.Holder.WriterBox != "home-box" {
658	t.Fatalf("the claim after the probe: to %q home %q writer %q", c.Op.Intent["to"], c.Op.Intent["home_region"], c.Holder.WriterBox)
659	}
660	// The writer asked the owners' admission evidence for each admission,
661	// and it passed: the admission's ie and the op's @home are one region
662	// under home_region ie (SameRegion). The same evidence refuses the
663	// admission in a region that is not the target's.
664	if p.evidence != attempts {
665	t.Fatalf("the writer's admission evidence ran %d times, want %d", p.evidence, attempts)
666	}
667	if !park.SameRegion(c.Op.Intent["home_region"], a.Region, c.Op.Intent["to"]) || a.Region == c.Op.Intent["to"] {
668	t.Fatalf("fixture: the admission's %q and the op's %q are not two spellings of one region", a.Region, c.Op.Intent["to"])
669	}
670
671	// C8 established the prepared generation's files manifest in data-home
672	// under <location>/files/g-<id>/.
673	m, found, err := hostfiles.Load(ctx, p1FilesStore{vs: h.vs, bucket: "data-home"}, moves.GenerationFilesPrefix(cutRoot, p1ID))
674	if err != nil || !found || m.Seq != 1 {
675	t.Fatalf("the generation's files manifest in data-home: %+v %v %v", m, found, err)
676	}
677	if e, ok := m.Entries["upload.txt"]; !ok || !strings.HasPrefix(e.Object, moves.GenerationFilesPrefix(cutRoot, p1ID)) {
678	t.Fatalf("the files manifest does not carry the instance's files from data-home: %+v", m.Entries)
679	}
680	if m.Platform[hostfiles.EnvItem].SHA256 != sha(env) {
681	t.Fatalf("the established env item is not the env file on disk: %+v", m.Platform)
682	}
683	if _, has := m.Platform[instancekey.AdoptionItem]; has {
684	t.Fatal("the adoption item was carried")
685	}
686	if _, err := os.Stat(adoption); !errors.Is(err, os.ErrNotExist) {
687	t.Fatalf("the adoption record remains: %v", err)
688	}
689	for k := range h.f.buckets["data-ca"] {
690	if strings.Contains(k, "/files/g-"+p1ID+"/") {
691	t.Fatalf("the generation's files were written to the source's bucket: %s", k)
692	}
693	}
694
695	// The commit: the source's location park in ca, the lane park in
696	// @home, the source's predecessor covered, quiet_after gains ca.
697	src := h.chore(t, "ca", false)
698	if src == nil || src.Owner != "region:ca" || src.Park.Location != cutRoot {
699	t.Fatalf("the source's location park: %+v", src)
700	}
701	lane := h.chore(t, park.HomeRegion, true)
702	if lane == nil || lane.Owner != "region:"+park.HomeRegion || lane.Park.Location != moves.ScopeLocation(p1Host, p1ID) {
703	t.Fatalf("the lane park: %+v (%+v)", lane, c.Obligations)
704	}
705	if h.chore(t, "ie", true) != nil || h.chore(t, "ie", false) != nil {
706	t.Fatalf("a chore named ie: %+v", c.Obligations)
707	}
708	covered := false
709	for _, ad := range c.Holder.Admissions {
710	if ad.Generation == p1ID && ad.Predecessor == "cur" {
711	if ad.Region != "ie" || ad.PredecessorRegion != "ca" || !ad.Covered {
712	t.Fatalf("the probe's admission after the commit: %+v", ad)
713	}
714	covered = true
715	}
716	}
717	if !covered {
718	t.Fatalf("no admission retires the source's generation: %+v", c.Holder.Admissions)
719	}
720
721	// After the horizon the home box's engine acknowledges the lane park
722	// and the residue parks; the source parks its location; the record
723	// closes in home.
724	h.clock.Advance(24 * time.Hour)
725	h.land(t)
726	if h.chore(t, park.HomeRegion, true) != nil {
727	t.Fatalf("the lane park was not acknowledged: %+v", h.claim(t).Obligations)
728	}
729	if residue != nil {
730	// The residue parks are the home region's regional runner's: the
731	// real hostbox.Regional on home-box, its engine in the roster's ie
732	// with the @home alias, over the real park view seen from ie.
733	r := homeRegional(h)
734	for _, o := range residueChores(h.claim(t)) {
735	if err := r.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID}); err != nil {
736	t.Fatalf("the home box's regional runner on the residue park of %s: %v", o.Generation, err)
737	}
738	}
739	if gens := residueChores(h.claim(t)); len(gens) != 0 {
740	t.Fatalf("the residue parks were not acknowledged: %+v", gens)
741	}
742	for _, k := range residue {
743	if _, ok := h.f.buckets["data-home"][k]; ok && !strings.HasSuffix(k, "/generation.json") {
744	t.Fatalf("the earlier stay's %s is still at the location", k)
745	}
746	}
747	}
748	h.notClosed(t, "the source has not parked")
749	h.sourcePass(t)
750	h.done(t)
751	rec := h.read(t, p1Host)
752	if rec.Region != "" || rec.LastMove == nil || rec.LastMove.Outcome != "done" {
753	t.Fatalf("the record after the close: region %q last %+v", rec.Region, rec.LastMove)
754	}
755	for k := range h.f.buckets["data-ca"] {
756	if strings.HasPrefix(k, cutRoot+"/") {
757	t.Fatalf("still at the source's location: %s", k)
758	}
759	}
760	// The acknowledged parks set quiet_after: ca for the source's location,
761	// and never the unset region.
762	q := h.claim(t).QuietAfter
763	if _, ok := q["ca"]; !ok {
764	t.Fatalf("quiet_after %v names no ca", q)
765	}
766	if _, ok := q[""]; ok {
767	t.Fatalf("quiet_after %v names the unset region", q)
768	}
769	}
770
771	// intoHomeMovesOutAgain: the closed move leaves the instance in home, and
772	// it can move out again by part 1's path. The roster places a move from
773	// home-box (roster ie, the fleet's home) to ca-box as "" -> ca; the start
774	// CAS begins it on the closed claim; and the home box's source round
775	// accepts it: it touches ca and ca-box, takes ca's staging lease and
776	// stages the new move's lane into data-ca.
777	func intoHomeMovesOutAgain(t *testing.T, r *intoHomeRun) {
778	t.Helper()
779	h := r.h
780	ctx := context.Background()
781	roster := hostactors.Roster{V: 1, DeploymentBucket: "deploy", Revision: 1, HomeRegion: "ie", Boxes: map[string]hostactors.Box{
782	"home-box": {Region: "ie", State: "active", Enrollment: p1DstEnrollment},
783	"ca-box": {Region: "ca", State: "active", Enrollment: p1SrcEnrollment},
784	}}
785	if !hostops.ClaimMoveRosterPlaces(roster, "home-box", "ca-box", "", "ca", "ie") {
786	t.Fatal("the roster does not place home-box -> ca-box as a move out of home")
787	}
788	if hostops.ClaimMoveRosterPlaces(roster, "home-box", "ca-box", "ie", "ca", "ie") {
789	t.Fatal("the roster placed the home box's record as the named region ie")
790	}
791	rec := h.read(t, p1Host)
792	if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveStartTransition(provision.MoveStart{
793	ID: p1ID2, Incarnation: h.inc, From: rec.Region, To: "ca", Cut: "manual", By: "op", Now: h.clock.Now()})); err != nil {
794	t.Fatal(err)
795	}
796	obs, err := hostclaim.Read(ctx, h.s, p1Host)
797	if err != nil {
798	t.Fatal(err)
799	}
800	if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.BeginAdmittedMove(p1ID2, "op", "home-box", "ca-box", park.PlacementRegion("ca"), h.clock.Now(),
801	hostclaim.MoveAdmission{HomeRegion: "ie", AppID: "app_" + p1Host, AdmissionID: p1ID2, SourceEnrollment: p1DstEnrollment, TargetEnrollment: p1SrcEnrollment})); err != nil {
802	t.Fatalf("the start CAS of the move out of home: %v", err)
803	}
804	// The home box as the source: its own bucket, data directory, route
805	// and launch environment.
806	c := h.landingCfg(t)
807	c.LaunchEnv = func(string) (map[string]string, map[string]bool, error) {
808	env, instance, _, err := (&activator.ExecBackend{DataDir: h.tdir, Region: ""}).LaunchLayers(h.landingHarness.route)
809	return env, instance, err
810	}
811	c.ReadyProbe, c.StrictNext = h.cfg.ReadyProbe, h.cfg.StrictNext
812	before := len(r.leases.all())
813	h.lease.mu.Lock()
814	h.lease.until = h.clock.now.Add(moves.StagingLeaseTTL)
815	h.lease.mu.Unlock()
816	w := &movesWorker{cfg: c, mirrors: map[string]lease.MirrorStore{}}
817	if err := movesPass(ctx, h.s, w); err != nil {
818	t.Fatal(err)
819	}
820	m := h.read(t, p1Host).Move
821	if m == nil || m.ID != p1ID2 || m.State != provision.MovePreparing {
822	t.Fatalf("the move out of home after the source round: %+v", m)
823	}
824	keys := laneKeysIn(h.phase1Harness, "data-ca", p1ID2)
825	replica := false
826	for _, k := range keys {
827	replica = replica || strings.HasPrefix(k, "replica/")
828	}
829	if !containsString(keys, "move.json") || !replica {
830	t.Fatalf("the lane into ca: %v", keys)
831	}
832	if got := r.leases.all()[before:]; len(got) == 0 || got[0] != "control/staging-leases/ca/"+p1Host {
833	t.Fatalf("the move out of home's staging leases: %v", got)
834	}
835	cl := h.claim(t)
836	if !containsString(cl.Touched.Regions, "ca") || !containsString(cl.Touched.Boxes, "ca-box") || cl.Op.Intent["to"] != "ca" || cl.Op.Exec != "home-box" {
837	t.Fatalf("the claim of the move out of home: %+v %+v", cl.Op, cl.Touched)
838	}
839	}
840
841	// TestMoveIntoHomeRestartedProbe: the first probe attempt is admitted on
842	// home-box and the agent dies before readiness. The restarted attempt is
843	// admitted into a fresh generation whose predecessor is the first
844	// attempt's, in the region that admission recorded, ie; it is published
845	// fenced (TestMovesProbeOwnersPublishARestartedAttemptFenced's shape), on
846	// home-box into data-home; and the commit and the close proceed as for one
847	// attempt.
848	func TestMoveIntoHomeRestartedProbe(t *testing.T) {
849	r := intoHomeClose(t, true)
850	h, p := r.h, r.p
851	published := false
852	p.afterMaterialize = func(n int) error {
853	switch n {
854	case 1:
855	return errors.New("test: the agent died before readiness")
856	case 2:
857	restartedAttemptPublishesFenced(t, h, p)
858	published = true
859	}
860	return nil
861	}
862	h.holdProbed = true
863	h.land(t)
864	if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbing || len(p.admissions) != 1 || p.admissions[0].Generation != p1ID {
865	t.Fatalf("the first attempt: %+v admissions %+v", m, p.admissions)
866	}
867	first := p.admissions[0]
868	if first.Box != "home-box" || first.Region != "ie" || first.PredecessorRegion != "ca" || first.Predecessor != "cur" {
869	t.Fatalf("the first attempt's admission: %+v", first)
870	}
871
872	// The agent restarts: a fresh worker, and the claim runtime's second
873	// attempt, published fenced while it is live.
874	h.freshTarget()
875	h.land(t)
876	if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandProbed || !published {
877	t.Fatalf("the restarted attempt: %+v published %v", m, published)
878	}
879	if len(p.admissions) != 2 {
880	t.Fatalf("admissions: %+v", p.admissions)
881	}
882
883	// The commit and the close, as for one attempt.
884	h.holdProbed = false
885	intoHomeCommitsAndCloses(t, r, 2)
886	}
887
888	// restartedAttemptPublishesFenced runs inside the restarted attempt, after
889	// Materialize: the admission is a fresh generation whose predecessor is the
890	// first attempt's generation in the first admission's spelling, ie, and
891	// the fresh generation is published fenced, never unfenced and never
892	// before its own first root, then over the location's marker in data-home.
893	func restartedAttemptPublishesFenced(t *testing.T, h *finalHarness, p *prodProbe) {
894	t.Helper()
895	ctx := context.Background()
896	first, again := p.admissions[0], p.admissions[1]
897	if again.Box != "home-box" || again.Region != "ie" || again.Mode != "cold" || again.Generation == p1ID || again.Predecessor != p1ID || again.PredecessorRegion != first.Region {
898	t.Fatalf("the restarted attempt's admission: %+v (first %+v)", again, first)
899	}
900	admitted, in := p.materialized, p.materializedIn
901	if g := admitted.Claim.Holder.Generation; g != again.Generation || in.Admission.NewGeneration != g {
902	t.Fatalf("fixture: the materialized holder is at %s, its inputs admit %s", g, in.Admission.NewGeneration)
903	}
904	if err := p.owners.Publish(ctx, admitted, in); !errors.Is(err, hostbox.ErrChanged) {
905	t.Fatalf("an unfenced marker for a fresh generation: %v", err)
906	}
907	synced := 0
908	pub := ClaimPublication{Sync: func(context.Context) error { synced++; return nil }, Check: func(context.Context) error { return nil }}
909	if _, err := p.owners.PublishFenced(ctx, admitted, in, pub); !errors.Is(err, hostbox.ErrChanged) || synced != 1 {
910	t.Fatalf("a marker before the root: %v synced=%d", err, synced)
911	}
912	root := cutRoot + "/g-" + again.Generation + "/0009/" + ltx.FormatFilename(1, ltx.TXID(h.n))
913	hdr, err := (&ltx.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, Commit: 4, MinTXID: 1, MaxTXID: ltx.TXID(h.n)}).MarshalBinary()
914	if err != nil {
915	t.Fatal(err)
916	}
917	if _, err := h.vs.PutIfAbsent(ctx, "data-home", root, hdr); err != nil {
918	t.Fatal(err)
919	}
920	receipt, err := p.owners.PublishFenced(ctx, admitted, in, pub)
921	if err != nil || receipt.Root.Key != root || receipt.MarkerVersion == "" {
922	t.Fatalf("publish: %+v %v", receipt, err)
923	}
924	if err := p.owners.PublicationEvidence(ctx, admitted); err != nil {
925	t.Fatal(err)
926	}
927	raw, err := h.vs.GetVersion(ctx, "data-home", cutRoot+"/generation.json", receipt.MarkerVersion)
928	if err != nil {
929	t.Fatal(err)
930	}
931	var marker genselect.Marker
932	if err := json.Unmarshal(raw, &marker); err != nil || marker.Op != again.Generation || marker.Box != "home-box" || marker.Incarnation != h.inc {
933	t.Fatalf("the published marker: %+v %v", marker, err)
934	}
935	for k := range h.f.buckets["data-ca"] {
936	if strings.HasPrefix(k, cutRoot+"/g-"+again.Generation+"/") {
937	t.Fatalf("the restarted attempt wrote into the source's bucket: %s", k)
938	}
939	}
940	}
941
942	// cleanHomeStart asserts the claim a witness starts from: the start CAS
943	// wrote intent.to @home and home_region ie, and nothing else in the claim
944	// names home in either spelling (no touched entry, no quiet key, no chore).
945	func cleanHomeStart(t *testing.T, c *hostclaim.Claim) {
946	t.Helper()
947	if c.Op.Intent["to"] != park.HomeRegion || c.Op.Intent["home_region"] != "ie" {
948	t.Fatalf("the start CAS: %+v", c.Op.Intent)
949	}
950	for _, r := range c.Touched.Regions {
951	if r == "ie" || r == park.HomeRegion {
952	t.Fatalf("touched names home at the start: %+v", c.Touched)
953	}
954	}
955	if containsString(c.Touched.Boxes, "home-box") || len(c.QuietAfter) != 0 || len(c.Obligations) != 0 {
956	t.Fatalf("the start is not clean: touched %+v quiet %v obligations %+v", c.Touched, c.QuietAfter, c.Obligations)
957	}
958	}
959
960	// homeTokenSeen counts what a witness created for the target side, so the
961	// fresh-state proof is never vacuous: chores in @home, and whether touched
962	// ever held @home and home-box.
963	func homeTokenSeen(writes []hostclaim.Claim) (chores int, touched bool) {
964	seen := map[string]bool{}
965	for _, c := range writes {
966	for _, o := range c.Obligations {
967	if o.Region == park.HomeRegion && !seen[o.ID] {
968	seen[o.ID] = true
969	chores++
970	}
971	}
972	if containsString(c.Touched.Regions, park.HomeRegion) && containsString(c.Touched.Boxes, "home-box") {
973	touched = true
974	}
975	}
976	return chores, touched
977	}
978
979	// TestHomeTargetFreshStateShowsOnlyTheHomeToken (Review Focus 1, supplied
980	// regions are @home; the runtime proof): each box-side into-home witness of
981	// Tasks 3 and 4 runs once from clean claim state, where a wrong spelling
982	// cannot hide (the hostops witnesses, TestMoveIntoHomeDeclareLost and
983	// TestMoveIntoHomeFencedTargetRelease, assert the same from their own clean
984	// state with onlyHomeToken). A
985	// chore queued with ie would be a new obligation named ie; a Touch with ie
986	// would add ie to touched; a staging lease keyed with ie would show in the
987	// recorded keys; an exact comparison with intent.to (Abort, Revert,
988	// TargetReleased and the fenced and lost releases) would refuse and stop
989	// the witness. hostclaim.Commit compares nothing with intent.to: given ie
990	// it queues the lane park under ie, and only this check's chore assertion
991	// catches it, as it does QueueStagingPark's. After each, every
992	// target-side chore the run ever created is @home and owned by
993	// region:@home, touched gained only @home and home-box, quiet_after never
994	// named ie or "", and every home lane lease is keyed @home (a dirty
995	// return's lane is the source's, under ca's lease).
996	func TestHomeTargetFreshStateShowsOnlyTheHomeToken(t *testing.T) {
997	type run struct {
998	writes []hostclaim.Claim
999	leases []string
1000	chores int // the least number of @home chores the witness creates
1001	// noHomeLease: the witness's recorder starts after the cut, so it
1002	// sees no lease of the home lane; returnLane: the witness writes a
1003	// dirty return's lane into the source's region, under ca's lease.
1004	noHomeLease, returnLane bool
1005	}
1006	for name, witness := range map[string]func(t *testing.T) run{
1007	"the source round stages the home lane": func(t *testing.T) run {
1008	var watch *claimWatch
1009	h := newPhase1Harness(t, intoHome, watchClaims(&watch))
1010	leases := &stagingLeaseKeys{}
1011	leases.wrap(&h.cfg)
1012	h.start(t, "manual")
1013	cleanHomeStart(t, h.claim(t))
1014	h.pass(t)
1015	h.targetPass(t, h.targetCfg())
1016	h.pass(t)
1017	if m := h.read(t, p1Host).Move; m.State != provision.MoveReady {
1018	t.Fatalf("the move did not reach ready: %+v", m)
1019	}
1020	return run{writes: watch.since(0), leases: leases.all()}
1021	},
1022	"the landing, the commit and the close": func(t *testing.T) run {
1023	r := intoHomeClose(t, true)
1024	intoHomeCommitsAndCloses(t, r, 1)
1025	// Two residue parks and the lane park.
1026	return run{writes: r.watch.since(0), leases: r.leases.all(), chores: 3}
1027	},
1028	"the restarted probe": func(t *testing.T) run {
1029	r := intoHomeClose(t, false)
1030	r.p.afterMaterialize = func(n int) error {
1031	if n == 1 {
1032	return errors.New("test: the agent died before readiness")
1033	}
1034	return nil
1035	}
1036	r.h.land(t)
1037	r.h.freshTarget()
1038	intoHomeCommitsAndCloses(t, r, 2)
1039	return run{writes: r.watch.since(0), leases: r.leases.all(), chores: 1}
1040	},
1041	// Task 4.
1042	"the abort": func(t *testing.T) run {
1043	writes, leases := intoHomeAbort(t)
1044	return run{writes: writes, leases: leases, chores: 1}
1045	},
1046	"the pre-flip revert": func(t *testing.T) run {
1047	writes, leases := intoHomePreFlipRevert(t)
1048	return run{writes: writes, leases: leases, chores: 1}
1049	},
1050	"the clean rollback": func(t *testing.T) run {
1051	writes, leases := intoHomeCleanRollback(t)
1052	// The home location's cleaning park and the lane park.
1053	return run{writes: writes, leases: leases, chores: 2, noHomeLease: true}
1054	},
1055	"the dirty return": func(t *testing.T) run {
1056	writes, leases := intoHomeDirtyReturn(t)
1057	return run{writes: writes, leases: leases, chores: 1, noHomeLease: true, returnLane: true}
1058	},
1059	} {
1060	t.Run(name, func(t *testing.T) {
1061	got := witness(t)
1062	// The claim the start CAS left is the first written with the
1063	// move's op: clean, as cleanHomeStart requires.
1064	for i, c := range got.writes {
1065	if c.Op.Kind == hostclaim.KindMoving {
1066	cleanHomeStart(t, &got.writes[i])
1067	break
1068	}
1069	}
1070	onlyTheHomeToken(t, got.writes, got.leases, "ca", "ca-box")
1071	chores, touched := homeTokenSeen(got.writes)
1072	if chores < got.chores || !touched {
1073	t.Fatalf("the witness created %d chores in %s (want at least %d), touched held it %v", chores, park.HomeRegion, got.chores, touched)
1074	}
1075	home := 0
1076	for _, k := range got.leases {
1077	switch {
1078	case k == homeLeaseKey:
1079	home++
1080	case got.returnLane && k == "control/staging-leases/ca/"+p1Host:
1081	default:
1082	t.Fatalf("a staging lease keyed %q", k)
1083	}
1084	}
1085	if home == 0 && !got.noHomeLease {
1086	t.Fatal("the witness took no staging lease of the home lane")
1087	}
1088	})
1089	}
1090	}
1091
1092	// obligationsJSON is the claim's obligation list as stored, for byte
1093	// comparison.
1094	func obligationsJSON(t *testing.T, obs []hostclaim.Obligation) string {
1095	t.Helper()
1096	b, err := json.Marshal(obs)
1097	if err != nil {
1098	t.Fatal(err)
1099	}
1100	return string(b)
1101	}
1102
1103	// otherThan is the obligations not in region.
1104	func otherThan(obs []hostclaim.Obligation, region string) []hostclaim.Obligation {
1105	var out []hostclaim.Obligation
1106	for _, o := range obs {
1107	if o.Region != region {
1108	out = append(out, o)
1109	}
1110	}
1111	return out
1112	}
1113
1114	// retainedID is a fixed obligation id for a seeded chore.
1115	func retainedID(n int) string { return fmt.Sprintf("%032x", 0xa000+n) }
1116
1117	// TestHomeTargetRetainedChoresKeepTheirSpelling (Review Focus 1, retained
1118	// state keeps its spelling): the residue chores and the lane chore are
1119	// already owed when the landing and the commit queue theirs, under ie and
1120	// then under @home. The landing's QueueResidueParks(@home) leaves the
1121	// obligation list byte-identical; the commit's QueueStagingPark(@home)
1122	// leaves every chore but the source's new location park byte-identical. No
1123	// second chore, no rewrite of ie to @home. The home box's engines then
1124	// acknowledge each seeded chore once, and the move closes in home.
1125	func TestHomeTargetRetainedChoresKeepTheirSpelling(t *testing.T) {
1126	for _, spelling := range []string{"ie", park.HomeRegion} {
1127	t.Run(spelling, func(t *testing.T) {
1128	ctx := context.Background()
1129	r, ids := retainedChoresRun(t, spelling)
1130	h := r.h
1131
1132	// Each retained chore is acknowledged once, in its spelling: the
1133	// lane by the home box's moves engine, the residue by its
1134	// regional runner.
1135	h.clock.Advance(24 * time.Hour)
1136	h.land(t)
1137	reg := homeRegional(h)
1138	for _, g := range []string{hostclaim.RootGeneration, "old"} {
1139	if err := reg.Run(ctx, park.Work{Host: p1Host, ID: ids[g].ID, Op: p1ID}); err != nil {
1140	t.Fatalf("the regional runner on the residue park of %s owed under %s: %v", g, spelling, err)
1141	}
1142	}
1143	ackedOnce(t, h, ids, spelling)
1144	// A second run of an acknowledged chore writes nothing.
1145	gen := h.claim(t).Gen
1146	if err := reg.Run(ctx, park.Work{Host: p1Host, ID: ids["old"].ID, Op: p1ID}); err != nil || h.claim(t).Gen != gen {
1147	t.Fatalf("a second acknowledgement: %v (claim gen %d to %d)", err, gen, h.claim(t).Gen)
1148	}
1149	h.sourcePass(t)
1150	h.done(t)
1151	})
1152	}
1153	}
1154
1155	// ackedOnce asserts each chore in ids is no longer owed and its retirement
1156	// holds exactly one park manifest in data-home, and that no home chore is
1157	// still owed.
1158	func ackedOnce(t *testing.T, h *finalHarness, ids map[string]hostclaim.Obligation, spelling string) {
1159	t.Helper()
1160	c := h.claim(t)
1161	for name, o := range ids {
1162	for _, owed := range c.Obligations {
1163	if owed.ID == o.ID {
1164	t.Fatalf("the %s chore owed under %s was not acknowledged: %+v", name, spelling, owed)
1165	}
1166	}
1167	n := 0
1168	for k := range h.f.buckets["data-home"] {
1169	if strings.HasPrefix(k, "parked/"+h.inc+"/"+p1Host+"/"+o.Retirement+"/") && strings.HasSuffix(k, "/manifest.json") {
1170	n++
1171	}
1172	}
1173	if n != 1 {
1174	t.Fatalf("the %s chore's retirement %s holds %d manifests, want 1", name, o.Retirement, n)
1175	}
1176	}
1177	for _, o := range c.Obligations {
1178	if o.Region == park.HomeRegion || o.Region == "ie" {
1179	t.Fatalf("a home chore is still owed: %+v", o)
1180	}
1181	}
1182	}
1183
1184	// retainedChoresRun is an into-home run whose residue chores and lane
1185	// chore are already owed under spelling when the landing and the commit
1186	// queue theirs; it asserts both queues leave them as they are, and returns
1187	// at landing/committed with the seeded chores by name (@root, old, lane).
1188	func retainedChoresRun(t *testing.T, spelling string) (*intoHomeRun, map[string]hostclaim.Obligation) {
1189	t.Helper()
1190	ids := map[string]hostclaim.Obligation{}
1191	var seeded, atQueue string
1192	r := intoHomeCloseWith(t, true, func(lh *landingHarness) {
1193	// Owed before the landing's parking step: one park-generation
1194	// per residue generation, with the landing's own scope.
1195	rec := lh.read(t, p1Host)
1196	rewriteP1Claim(t, lh.s, func(c *hostclaim.Claim) {
1197	scope := moves.ResidueScope(*c.Holder, *rec.Move.FlippedAt)
1198	for i, g := range []string{hostclaim.RootGeneration, "old"} {
1199	s := scope
1200	o := hostclaim.Obligation{ID: retainedID(2 * i), Op: p1ID, Kind: "park-generation", Region: spelling, Generation: g, Retirement: retainedID(2*i + 1), Owner: "region:" + spelling, Park: &s}
1201	ids[g] = o
1202	c.Obligations = append(c.Obligations, o)
1203	}
1204	})
1205	seeded = obligationsJSON(t, lh.claim(t).Obligations)
1206	lh.landFault = func(point string) error {
1207	if point == "after-residue-queued" && atQueue == "" {
1208	atQueue = obligationsJSON(t, lh.claim(t).Obligations)
1209	}
1210	return nil
1211	}
1212	})
1213	h, p := r.h, r.p
1214	if atQueue == "" {
1215	t.Fatal("the landing never queued its residue parks")
1216	}
1217	if atQueue != seeded {
1218	t.Fatalf("QueueResidueParks(%s) over chores owed under %s changed the obligations:\n%s\nwas\n%s", park.HomeRegion, spelling, atQueue, seeded)
1219	}
1220
1221	// Owed before the commit: the lane's park with the commit's own
1222	// scope.
1223	var beforeCommit, afterCommit []hostclaim.Obligation
1224	h.landFault = func(point string) error {
1225	switch point {
1226	case "before-commit":
1227	if beforeCommit != nil {
1228	return nil
1229	}
1230	rec := h.read(t, p1Host)
1231	rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
1232	_, staging, err := moves.CommitScopes(c, rec.Move.FlippedAt)
1233	if err != nil {
1234	t.Fatal(err)
1235	}
1236	o := hostclaim.Obligation{ID: retainedID(10), Op: p1ID, Kind: "park-replica", Region: spelling, Retirement: retainedID(11), Owner: "region:" + spelling, Park: &staging}
1237	ids["lane"] = o
1238	c.Obligations = append(c.Obligations, o)
1239	})
1240	beforeCommit = h.claim(t).Obligations
1241	case "after-claim-committed":
1242	afterCommit = h.claim(t).Obligations
1243	}
1244	return nil
1245	}
1246	h.land(t)
1247	h.committedAndOpen(t)
1248	if len(p.admissions) != 1 || afterCommit == nil {
1249	t.Fatalf("the commit: admissions %+v, claim captured %v", p.admissions, afterCommit != nil)
1250	}
1251	if got, want := obligationsJSON(t, otherThan(afterCommit, "ca")), obligationsJSON(t, beforeCommit); got != want {
1252	t.Fatalf("the commit over a lane park owed under %s changed the home chores:\n%s\nwas\n%s", spelling, got, want)
1253	}
1254	if src := afterCommit[len(afterCommit)-1]; len(afterCommit) != len(beforeCommit)+1 || src.Region != "ca" || src.Kind != "park-replica" || src.Park.Location != cutRoot {
1255	t.Fatalf("the commit queued %+v beside the retained chores", afterCommit[len(beforeCommit):])
1256	}
1257	return r, ids
1258	}
1259
1260	// TestHomeRegionalAcknowledgesBothSpellings (Ask A2's witness): the real
1261	// hostbox.Regional on home-box (roster region ie, home ie) acknowledges a
1262	// lane park-replica and the residue park-generations owed under @home, and
1263	// the same owed under ie, each exactly once; ca-box's runner (region ca, no
1264	// home) acknowledges none of them.
1265	func TestHomeRegionalAcknowledgesBothSpellings(t *testing.T) {
1266	for _, spelling := range []string{park.HomeRegion, "ie"} {
1267	t.Run(spelling, func(t *testing.T) {
1268	ctx := context.Background()
1269	r, ids := retainedChoresRun(t, spelling)
1270	h := r.h
1271	h.clock.Advance(24 * time.Hour)
1272	ca := regionalRunner(h.phase1Harness, h.leases, "ca", "", "data-ca")
1273	home := homeRegional(h)
1274	for name, o := range ids {
1275	gen := h.claim(t).Gen
1276	if err := ca.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID}); !errors.Is(err, hostbox.ErrUnsupported) || h.claim(t).Gen != gen {
1277	t.Fatalf("ca's runner on the %s chore owed under %s: %v (claim gen %d to %d)", name, spelling, err, gen, h.claim(t).Gen)
1278	}
1279	}
1280	for name, o := range ids {
1281	if err := home.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID}); err != nil {
1282	t.Fatalf("home-box's runner on the %s chore owed under %s: %v", name, spelling, err)
1283	}
1284	gen := h.claim(t).Gen
1285	if err := home.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID}); err != nil || h.claim(t).Gen != gen {
1286	t.Fatalf("a second acknowledgement of the %s chore: %v (claim gen %d to %d)", name, err, gen, h.claim(t).Gen)
1287	}
1288	}
1289	ackedOnce(t, h, ids, spelling)
1290	})
1291	}
1292	}
1293
1294	// Home part 2, Task 4: revert, rollback, abort, loss and conversion with a
1295	// home target. Each witness also runs from clean claim state in
1296	// TestHomeTargetFreshStateShowsOnlyTheHomeToken.
1297
1298	// homeLaneCfg is home-box's configuration with its moves park engine armed
1299	// (region @home, ParkHomeRegion ie) over its own data directory and lease
1300	// table.
1301	func homeLaneCfg(h *phase1Harness, c moveCfg, dir string, leases *fakeParkLeases) moveCfg {
1302	c.DataDir = dir
1303	armParkFakes(&c, h.s, leases, park.PlacementRegion(c.Region), roleTarget)
1304	return c
1305	}
1306
1307	// homeLaneChore is the op's lane park owed in @home.
1308	func homeLaneChore(c *hostclaim.Claim) *hostclaim.Obligation {
1309	for _, o := range c.Obligations {
1310	if o.Op == p1ID && o.Kind == "park-replica" && o.Park != nil && o.Park.Location == moves.ScopeLocation(p1Host, p1ID) && o.Region != "ca" {
1311	return &o
1312	}
1313	}
1314	return nil
1315	}
1316
1317	// TestMoveIntoHomeAbortParksTheHomeLane (Ask A2's witness): an abort before
1318	// the cut. Abort queues the lane park in @home, owned by region:@home;
1319	// home-box's moves engine archives the lane in data-home; past the horizon
1320	// a named box's regional runner (ca) refuses the chore and home-box's
1321	// hostbox.Regional (roster region ie, home ie) acknowledges it; the record
1322	// clears only after that.
1323	func TestMoveIntoHomeAbortParksTheHomeLane(t *testing.T) {
1324	intoHomeAbort(t)
1325	}
1326
1327	func intoHomeAbort(t *testing.T) ([]hostclaim.Claim, []string) {
1328	t.Helper()
1329	ctx := context.Background()
1330	var watch *claimWatch
1331	h := newPhase1Harness(t, intoHome, watchClaims(&watch))
1332	leases := &stagingLeaseKeys{}
1333	leases.wrap(&h.cfg)
1334	h.start(t, "manual")
1335	cleanHomeStart(t, h.claim(t))
1336	h.pass(t)
1337	staged := laneKeysIn(h, "data-home", p1ID)
1338	if len(staged) == 0 {
1339	t.Fatal("fixture: the round staged nothing in data-home")
1340	}
1341	g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
1342	if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {
1343	t.Fatal(err)
1344	}
1345	h.pass(t)
1346	c := h.claim(t)
1347	if c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseAborting || c.Op.Exec != "ca-box" || c.Op.Intent["to"] != park.HomeRegion {
1348	t.Fatalf("the claim after the abort: %+v", c.Op)
1349	}
1350	o := homeLaneChore(c)
1351	if o == nil || o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion || o.Park.Incarnation != h.inc {
1352	t.Fatalf("the abort's lane park: %+v (%+v)", o, c.Obligations)
1353	}
1354	waiting := func(why string) {
1355	t.Helper()
1356	if m := h.read(t, p1Host).Move; m == nil || m.State != provision.MoveAborting {
1357	t.Fatalf("%s: the record cleared: %+v", why, m)
1358	}
1359	}
1360	// home-box's moves engine archives the lane; the horizon has not passed.
1361	tleases := &fakeParkLeases{}
1362	tc := homeLaneCfg(h, h.targetCfg(), t.TempDir(), tleases)
1363	h.targetPass(t, tc)
1364	if left := laneKeysIn(h, "data-home", p1ID); len(left) != 0 {
1365	t.Fatalf("the home lane still holds %v", left)
1366	}
1367	if homeLaneChore(h.claim(t)) == nil {
1368	t.Fatal("the lane park was acknowledged before its horizon")
1369	}
1370	h.pass(t)
1371	waiting("before the horizon")
1372	// Past the horizon: a named box's runner refuses the chore, the home
1373	// box's acknowledges it.
1374	until := o.Park.EvidenceAt.Add(park.SignatureAge + o.Park.RequestLimit)
1375	h.clock.Advance(until.Sub(h.clock.now) + time.Second)
1376	work := park.Work{Host: p1Host, ID: o.ID, Op: p1ID}
1377	if err := regionalRunner(h, tleases, "ca", "", "data-ca").Run(ctx, work); !errors.Is(err, hostbox.ErrUnsupported) {
1378	t.Fatalf("ca's regional runner on the home lane park: %v", err)
1379	}
1380	if homeLaneChore(h.claim(t)) == nil {
1381	t.Fatal("ca's runner acknowledged the home lane park")
1382	}
1383	h.pass(t)
1384	waiting("ca's runner refused")
1385	if err := regionalRunner(h, tleases, "ie", "ie", "data-home").Run(ctx, work); err != nil {
1386	t.Fatalf("home-box's regional runner on the lane park: %v", err)
1387	}
1388	if homeLaneChore(h.claim(t)) != nil {
1389	t.Fatal("the lane park was not acknowledged")
1390	}
1391	if q := h.claim(t).QuietAfter; q["ie"] != (time.Time{}) || q[""] != (time.Time{}) {
1392	t.Fatalf("quiet_after %v", q)
1393	}
1394	// Only now does the record clear.
1395	h.pass(t)
1396	rec := h.read(t, p1Host)
1397	if rec.Move != nil || rec.Region != "ca" || rec.LastMove == nil || rec.LastMove.Outcome != "aborted" {
1398	t.Fatalf("the record after the abort: region %q move %+v last %+v", rec.Region, rec.Move, rec.LastMove)
1399	}
1400	if c := h.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ca-box" || len(c.Obligations) != 0 {
1401	t.Fatalf("the claim after the abort: %+v %+v", c.Op, c.Obligations)
1402	}
1403	for k := range h.f.buckets["data-ca"] {
1404	if strings.HasPrefix(k, "parked/") {
1405	t.Fatalf("the abort parked in the source's bucket: %s", k)
1406	}
1407	}
1408	return watch.since(0), leases.all()
1409	}
1410
1411	// TestMoveIntoHomeCleanRollback: the home target's probe, through the
1412	// production probe owners, is admitted on home-box and fails before any
1413	// write, until its attempts are spent. The rollback releases clean:
1414	// TargetReleased names @home, which matches op.intent.to exactly, and the
1415	// cleaning park of the home location is queued in @home. The source then
1416	// reopens on ca-box through the production recovery owners, in two stages:
1417	// before the admission the clean ReturnToSource has put the holder back on
1418	// src_gen and the owners' baseline restores from src_gen; after it the
1419	// holder is on the fresh generation the admission named, the admission is
1420	// clean-return with no writes lost, in ca, and its predecessor is the failed
1421	// home target's last admission, in the spelling that admission recorded:
1422	// ie. The target's cleaning then queues the lane park in @home, home-box's
1423	// regional runner acknowledges both after the horizon, and the record ends
1424	// rolled_back in ca with nothing parked in data-ca.
1425	func TestMoveIntoHomeCleanRollback(t *testing.T) {
1426	intoHomeCleanRollback(t)
1427	}
1428
1429	func intoHomeCleanRollback(t *testing.T) ([]hostclaim.Claim, []string) {
1430	t.Helper()
1431	ctx := context.Background()
1432	var watch *claimWatch
1433	h := newRecoverHarness(t, false, intoHome, realLaunchEnv, watchClaims(&watch))
1434	leases := &stagingLeaseKeys{}
1435	leases.wrap(&h.cfg)
1436	cleanHomeStart(t, firstMoving(t, watch.since(0)))
1437	// Two attempts, each admitted through the production owners and the
1438	// claim runtime and failing before any write. The second is a fresh
1439	// generation with its own stamp: its restore retires the first's
1440	// installation and re-tags its inventory (RetireSuperseded).
1441	if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, func(rec *provision.Record) error {
1442	rec.Move.CutAttempts = 2
1443	return nil
1444	}); err != nil {
1445	t.Fatal(err)
1446	}
1447	p := h.productionProbe(t)
1448	p.afterMaterialize = func(int) error { return errors.New("test: the probe failed") }
1449	for i := 0; i < 8 && h.read(t, p1Host).Move.State != provision.MoveRollingBack; i++ {
1450	h.rollbackPass(t)
1451	}
1452	if len(p.admissions) != 2 || p.stopped != 1 {
1453	t.Fatalf("the probe's attempts: admissions %+v, predecessors stopped %d", p.admissions, p.stopped)
1454	}
1455	failed := p.admissions[len(p.admissions)-1]
1456	if failed.Box != "home-box" || failed.Region != "ie" || failed.Generation == p1ID || failed.Predecessor != p1ID {
1457	t.Fatalf("the home target's last admission: %+v", failed)
1458	}
1459	for i := 0; i < 4 && h.rb(t).TargetPhase != provision.RbTargetReleased; i++ {
1460	h.rollbackPass(t)
1461	}
1462	rb := h.rb(t)
1463	if rb.TargetPhase != provision.RbTargetReleased || rb.Dirty {
1464	t.Fatalf("the target did not release clean: %+v", rb)
1465	}
1466	c := h.claim(t)
1467	if c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseReleased || c.Op.Intent["to"] != park.HomeRegion || c.Op.Intent["rollback"] != rb.ID {
1468	t.Fatalf("the claim after TargetReleased: %+v", c.Op)
1469	}
1470	location := false
1471	for _, o := range c.Obligations {
1472	if o.Op == p1ID && o.Kind == "park-replica" && o.Park != nil && o.Park.Location == cutRoot {
1473	if o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion {
1474	t.Fatalf("the cleaning park of the home location: %+v", o)
1475	}
1476	location = true
1477	}
1478	}
1479	if !location {
1480	t.Fatalf("TargetReleased queued no park of the home location: %+v", c.Obligations)
1481	}
1482
1483	// The source's recovery through the production recovery owners. Its
1484	// bucket is versioned, as production's is: the harness stored the
1485	// source's replica directly, so each object gets its version here.
1486	srcGen := c.Op.Journal[hostclaim.JournalSrcGen]
1487	src := sourceRecovery(t, h)
1488	src.beforeAdmit = func(o hostclaim.Observed, in ClaimLaunchInputs) {
1489	// Stage one: the clean ReturnToSource put the holder back on
1490	// src_gen, and the owners restore from src_gen.
1491	if o.Claim.Holder.Generation != srcGen || srcGen != "cur" || in.Baseline == nil || in.Baseline.Generation != srcGen || in.Prepared {
1492	t.Fatalf("before the admission: holder at %q (src_gen %q), baseline %+v", o.Claim.Holder.Generation, srcGen, in.Baseline)
1493	}
1494	if a := in.Admission; a.Coverage != "clean-return" || a.Box != "ca-box" || a.Region != "ca" || a.Mode != "cold" || a.PredecessorRegion != failed.Region {
1495	t.Fatalf("the selected admission: %+v", a)
1496	}
1497	}
1498	h.srcPass(t)
1499	if src.wakes != 1 || src.materialized.Claim == nil {
1500	t.Fatalf("the recovery's starts: %d, materialized %v", src.wakes, src.materialized.Claim != nil)
1501	}
1502	// Stage two: the holder is on the fresh generation the runtime admitted
1503	// and restored; the admission is clean-return, in ca, after the home
1504	// target's last admission in ie.
1505	ah := src.admitted.Claim.Holder
1506	last := ah.Admissions[len(ah.Admissions)-1]
1507	if ah.Generation != src.materializedIn.Admission.NewGeneration || ah.Generation == srcGen || ah.WriterBox != "ca-box" {
1508	t.Fatalf("the holder after the admission: %+v", ah)
1509	}
1510	if gen, err := hostbox.ReadGenerationMarker(h.srcRoute.DBPath + ".gen"); err != nil || gen != ah.Generation {
1511	t.Fatalf("the restore's generation marker %q %v, want %s", gen, err, ah.Generation)
1512	}
1513	if last.Coverage != "clean-return" || last.WritesLost != nil || len(last.AdditionalWritesLost) != 0 || last.Region != "ca" || last.Box != "ca-box" {
1514	t.Fatalf("the clean return's admission: %+v", last)
1515	}
1516	if last.Predecessor != failed.Generation || last.PredecessorRegion != "ie" {
1517	t.Fatalf("the clean return's predecessor %q in %q, want the home target's %q in ie", last.Predecessor, last.PredecessorRegion, failed.Generation)
1518	}
1519	h.opened(t)
1520
1521	// The target's cleaning: the lane park in @home, then both parks
1522	// acknowledged by home-box's regional runner after their horizon.
1523	h.rollbackPass(t)
1524	if rb := h.rb(t); rb.TargetPhase != provision.RbTargetCleaned {
1525	t.Fatalf("the target did not clean: %+v", rb)
1526	}
1527	owed := h.claim(t).Obligations
1528	var lane *hostclaim.Obligation
1529	for i, o := range owed {
1530	if o.Op == p1ID && o.Park != nil && o.Park.Location == moves.ScopeLocation(p1Host, p1ID) {
1531	lane = &owed[i]
1532	}
1533	if o.Op == p1ID && (o.Region != park.HomeRegion || o.Owner != "region:"+park.HomeRegion) {
1534	t.Fatalf("a chore of the rollback outside @home: %+v", o)
1535	}
1536	}
1537	if lane == nil {
1538	t.Fatalf("no lane park: %+v", owed)
1539	}
1540	h.clock.Advance(24 * time.Hour)
1541	home := homeRegionalOn(h.phase1Harness, h.leases)
1542	for _, o := range owed {
1543	if o.Op != p1ID {
1544	continue
1545	}
1546	if err := home.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID}); err != nil {
1547	t.Fatalf("home-box's regional runner on %+v: %v", o, err)
1548	}
1549	}
1550	h.srcPass(t)
1551	final := h.read(t, p1Host)
1552	if final.Move != nil || final.Region != "ca" || final.LastMove == nil || final.LastMove.Outcome != "rolled_back" {
1553	t.Fatalf("the record after the rollback: region %q move %+v last %+v", final.Region, final.Move, final.LastMove)
1554	}
1555	if c := h.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ca-box" || len(c.Obligations) != 0 {
1556	t.Fatalf("the claim after the rollback: %+v %+v", c.Op, c.Obligations)
1557	}
1558	// Nothing the source serves was parked: data-ca holds no archive, and
1559	// src_gen's objects are where they were.
1560	cur := 0
1561	for k := range h.f.buckets["data-ca"] {
1562	if strings.HasPrefix(k, "parked/") {
1563	t.Fatalf("the rollback parked in the source's bucket: %s", k)
1564	}
1565	if strings.HasPrefix(k, cutLineage) {
1566	cur++
1567	}
1568	}
1569	if cur == 0 {
1570	t.Fatal("src_gen's objects are gone from data-ca")
1571	}
1572	return watch.since(0), leases.all()
1573	}
1574
1575	// TestMoveIntoHomeDirtyReturn: the home target's probe, through the
1576	// production probe owners, is admitted on home-box, writes, and fails. The
1577	// rollback releases dirty: home-box seals its installation and writes the
1578	// return lane into data-ca under the staging lease
1579	// control/staging-leases/ca/<host>. The source installs the return through
1580	// the shared installer, and the recovery owners admit it into the return
1581	// generation, sealed, establishing its files from the return's prepared
1582	// manifest through the real C8 provider. probed queues the return lane's
1583	// park in ca. The pre-move generations park after the return (D6): no
1584	// park-generation is owed before the target has returned, and the source's
1585	// install queues them in ca.
1586	func TestMoveIntoHomeDirtyReturn(t *testing.T) {
1587	intoHomeDirtyReturn(t)
1588	}
1589
1590	func intoHomeDirtyReturn(t *testing.T) ([]hostclaim.Claim, []string) {
1591	t.Helper()
1592	ctx := context.Background()
1593	var watch *claimWatch
1594	h := newRecoverHarness(t, false, intoHome, realLaunchEnv, watchClaims(&watch))
1595	leases := &stagingLeaseKeys{}
1596	leases.wrap(&h.cfg)
1597	cleanHomeStart(t, firstMoving(t, watch.since(0)))
1598	if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, func(rec *provision.Record) error {
1599	rec.Move.CutAttempts = 2
1600	return nil
1601	}); err != nil {
1602	t.Fatal(err)
1603	}
1604	// Each attempt is admitted through the production owners and the claim
1605	// runtime, the app writes, and the attempt fails. The second's restore
1606	// retires the first's installation (RetireSuperseded).
1607	p := h.productionProbe(t)
1608	p.afterMaterialize = func(int) error {
1609	h.targetChanged(t)
1610	return errors.New("test: the probe failed after a write")
1611	}
1612	residueBefore := func(why string) {
1613	t.Helper()
1614	for _, o := range h.claim(t).Obligations {
1615	if o.Kind == "park-generation" {
1616	t.Fatalf("%s: a pre-move generation's park is owed already: %+v", why, o)
1617	}
1618	}
1619	}
1620	for i := 0; i < 6 && h.read(t, p1Host).Move.State != provision.MoveRollingBack; i++ {
1621	h.rollbackPass(t)
1622	}
1623	if len(p.admissions) != 2 || p.stopped != 1 || p.admissions[1].Box != "home-box" || p.admissions[1].Region != "ie" || p.admissions[1].Predecessor != p1ID {
1624	t.Fatalf("the home target's admissions: %+v (predecessors stopped %d)", p.admissions, p.stopped)
1625	}
1626	before := len(leases.all())
1627	for i := 0; i < 6 && h.rb(t).TargetPhase != provision.RbTargetReturned; i++ {
1628	h.rollbackPass(t)
1629	residueBefore("the target's release")
1630	}
1631	rb := h.rb(t)
1632	if !rb.Dirty || rb.TargetPhase != provision.RbTargetReturned {
1633	t.Fatalf("the target did not return dirty: %+v", rb)
1634	}
1635	// The return lane in data-ca, under ca's staging lease.
1636	lane := moves.ScopeLocation(p1Host, rb.ID) + "/"
1637	inCA := 0
1638	for k := range h.f.buckets["data-ca"] {
1639	if strings.HasPrefix(k, lane) {
1640	inCA++
1641	}
1642	}
1643	for k := range h.f.buckets["data-home"] {
1644	if strings.HasPrefix(k, lane) {
1645	t.Fatalf("the return lane was written into data-home: %s", k)
1646	}
1647	}
1648	if inCA == 0 {
1649	t.Fatal("the return lane holds nothing in data-ca")
1650	}
1651	if got := leases.all()[before:]; len(got) == 0 || slicesAll(got, "control/staging-leases/ca/"+p1Host) != len(got) {
1652	t.Fatalf("the return lane's staging leases: %v", got)
1653	}
1654
1655	// The source's recovery through the claim runtime on ca-box: the
1656	// production recovery owners, the real C8 provider establishing from the
1657	// return's prepared manifest.
1658	rec := sourceRecovery(t, h)
1659	// Under CARLOS_HOST_CLAIM (the probe mode sets it) the source's
1660	// configured step publishes the live release through the held release;
1661	// the stand-in answers as that publication would.
1662	h.releaseLive = func(context.Context, release.Store, provision.Record, string) moves.Check {
1663	return moves.Check{OK: true}
1664	}
1665	rec.beforeAdmit = func(o hostclaim.Observed, in ClaimLaunchInputs) {
1666	residueQueued := false
1667	for _, ob := range o.Claim.Obligations {
1668	if ob.Kind == "park-generation" && ob.Region == "ca" && ob.Owner == "region:ca" {
1669	residueQueued = true
1670	}
1671	}
1672	if !residueQueued {
1673	t.Fatalf("the source's install queued no park of the pre-move generations: %+v", o.Claim.Obligations)
1674	}
1675	if a := in.Admission; !in.Prepared || a.NewGeneration != rb.ID || a.Coverage != "sealed" || a.Box != "ca-box" || a.Region != "ca" || a.PredecessorRegion != "ie" {
1676	t.Fatalf("the dirty return's selected admission: %+v prepared %v", a, in.Prepared)
1677	}
1678	}
1679	h.srcPass(t)
1680	if rec.wakes != 1 || rec.materialized.Claim == nil || rec.materializedIn.Admission.NewGeneration != rb.ID {
1681	t.Fatalf("the recovery's starts: %d, materialized %+v", rec.wakes, rec.materializedIn.Admission)
1682	}
1683	got := h.opened(t)
1684	if got.ID != rb.ID {
1685	t.Fatalf("opened %+v", got)
1686	}
1687	// C8: the return generation's prepared manifest in data-ca is the one
1688	// the establishment opened.
1689	m, found, err := hostfiles.Load(ctx, p1FilesStore{vs: h.vs, bucket: "data-ca"}, moves.GenerationFilesPrefix(cutRoot, rb.ID))
1690	if err != nil || !found {
1691	t.Fatalf("the return generation's files manifest in data-ca: %v %v", found, err)
1692	}
1693	if _, ok := m.Entries["upload.txt"]; !ok || m.Platform[hostfiles.EnvItem].SHA256 == "" {
1694	t.Fatalf("the return's files manifest: %+v", m)
1695	}
1696	// probed queued the return lane's park in ca.
1697	returnLane := 0
1698	for _, o := range h.claim(t).Obligations {
1699	if o.Kind == "park-replica" && o.Park != nil && o.Park.Location == moves.ScopeLocation(p1Host, rb.ID) {
1700	if o.Region != "ca" || o.Owner != "region:ca" {
1701	t.Fatalf("the return lane's park: %+v", o)
1702	}
1703	returnLane++
1704	}
1705	}
1706	if returnLane != 1 {
1707	t.Fatalf("return lane parks: %d", returnLane)
1708	}
1709	return watch.since(0), leases.all()
1710	}
1711
1712	// slicesAll counts the entries of xs equal to x.
1713	func slicesAll(xs []string, x string) int {
1714	n := 0
1715	for _, s := range xs {
1716	if s == x {
1717	n++
1718	}
1719	}
1720	return n
1721	}
1722
1723	// TestMoveIntoHomePreFlipRevert: the cut stops the source on ca-box and
1724	// fails before the flip (TestCutDeadlinePassedReverts' shape, stopped with
1725	// execution given up). Revert(id, ca-box, @home, &laneScope) matches
1726	// op.intent.to exactly and queues the lane park in @home, owned by
1727	// region:@home. The source's revert probe re-admits on ca-box through the
1728	// production recovery owners, uncovered (TestRevertProbeAdmitsUncovered's
1729	// shape), in ca, after the source's own generation in ca. home-box's moves
1730	// engine archives the lane in data-home; past the horizon home-box's
1731	// hostbox.Regional acknowledges it, and the revert closes. Nothing in data-ca
1732	// is parked.
1733	func TestMoveIntoHomePreFlipRevert(t *testing.T) {
1734	intoHomePreFlipRevert(t)
1735	}
1736
1737	func intoHomePreFlipRevert(t *testing.T) ([]hostclaim.Claim, []string) {
1738	t.Helper()
1739	ctx := context.Background()
1740	var watch *claimWatch
1741	// The cut harness under a landing harness: its source replica is a real
1742	// one, which the recovery's cold restore restores.
1743	h := newRevertHarnessOn(t, newLandingHarness(t, intoHome, realLaunchEnv, watchClaims(&watch)).cutHarness)
1744	leases := &stagingLeaseKeys{}
1745	leases.wrap(&h.cfg)
1746	h.fromStopped(t)
1747	cleanHomeStart(t, firstMoving(t, watch.since(0)))
1748	if m := h.move(t); m.FlippedAt != nil || m.To != "" {
1749	t.Fatalf("the fixture flipped: %+v", m)
1750	}
1751	c := h.claim(t)
1752	if c.Op.Kind != hostclaim.KindReverting || c.Op.Exec != "ca-box" || c.Op.Intent["to"] != park.HomeRegion {
1753	t.Fatalf("the revert's claim: %+v", c.Op)
1754	}
1755	lane := homeLaneChore(c)
1756	if lane == nil || lane.Region != park.HomeRegion || lane.Owner != "region:"+park.HomeRegion {
1757	t.Fatalf("Revert's lane park: %+v (%+v)", lane, c.Obligations)
1758	}
1759	staged := laneKeysIn(h.phase1Harness, "data-home", p1ID)
1760	if len(staged) == 0 {
1761	t.Fatal("fixture: the home lane is empty")
1762	}
1763
1764	// The source's revert probe through the production recovery owners.
1765	h.arm(t)
1766	srcGen := c.Holder.Generation
1767	rec := revertRecovery(t, h)
1768	rec.beforeAdmit = func(o hostclaim.Observed, in ClaimLaunchInputs) {
1769	if a := in.Admission; a.Box != "ca-box" || a.Region != "ca" || a.PredecessorRegion != "ca" || a.Coverage != "" || a.Mode != "cold" || a.ObservedGeneration != srcGen ||
1770	in.Baseline == nil || in.Baseline.Generation != srcGen {
1771	t.Fatalf("the revert's selected admission: %+v baseline %+v", a, in.Baseline)
1772	}
1773	}
1774	h.pass(t)
1775	h.reopened(t)
1776	if rec.wakes != 1 || rec.materialized.Claim == nil {
1777	t.Fatalf("the revert probe's starts: %d, materialized %v", rec.wakes, rec.materialized.Claim != nil)
1778	}
1779	ah := rec.admitted.Claim.Holder
1780	last := ah.Admissions[len(ah.Admissions)-1]
1781	if ah.Generation == srcGen || ah.WriterBox != "ca-box" || last.Region != "ca" || last.PredecessorRegion != "ca" || last.Predecessor != srcGen || last.Coverage != "" {
1782	t.Fatalf("the revert's admission: holder %+v last %+v", ah, last)
1783	}
1784	// Uncovered: the source's own writes are recorded as possibly lost when
1785	// a writer session held them (the harness's serving holder never had
1786	// one, so there is nothing to name).
1787	if prior := rec.observed[0].Claim.Holder; prior.Session != 0 && (last.WritesLost == nil || last.WritesLost.Box != prior.WriterBox || last.WritesLost.Session != prior.Session) {
1788	t.Fatalf("the revert's writes lost: %+v (prior holder %+v)", last.WritesLost, prior)
1789	}
1790
1791	// home-box's moves engine archives the lane; the horizon has not passed.
1792	h.tPass(t)
1793	if left := laneKeysIn(h.phase1Harness, "data-home", p1ID); len(left) != 0 {
1794	t.Fatalf("the home lane still holds %v", left)
1795	}
1796	if homeLaneChore(h.claim(t)) == nil {
1797	t.Fatal("the lane park was acknowledged before its horizon")
1798	}
1799	h.pass(t)
1800	h.reopened(t)
1801	// Past the horizon home-box's regional runner acknowledges it, and the
1802	// revert closes.
1803	h.pastHorizon(t, lane)
1804	if err := homeRegionalOn(h.phase1Harness, h.tleases).Run(ctx, park.Work{Host: p1Host, ID: lane.ID, Op: p1ID}); err != nil {
1805	t.Fatalf("home-box's regional runner on the lane park: %v", err)
1806	}
1807	if homeLaneChore(h.claim(t)) != nil {
1808	t.Fatal("the lane park was not acknowledged")
1809	}
1810	h.pass(t)
1811	h.closed(t)
1812	if rec := h.read(t, p1Host); rec.Region != "ca" {
1813	t.Fatalf("the record after the revert: %+v", rec)
1814	}
1815	for k := range h.f.buckets["data-ca"] {
1816	if strings.HasPrefix(k, "parked/") {
1817	t.Fatalf("the revert parked in the source's bucket: %s", k)
1818	}
1819	}
1820	return watch.since(0), leases.all()
1821	}
1822
1823	// sourceRecovery binds the recover harness's source box (ca-box) to the
1824	// production mode: the recovery adapter's wake runs the claim runtime on
1825	// ca-box (newProdWake's start: Admit, Prepare, Restore, Reserve and
1826	// Materialize through the production recovery owners and the real C8).
1827	// The source is first made what a served source leaves (sourceServed).
1828	func sourceRecovery(t *testing.T, h *recoverHarness) *prodProbe {
1829	t.Helper()
1830	sourceServed(t, h.phase1Harness, h.sdir)
1831	p := newProdWake(t, h.phase1Harness, prodSide{box: h.srcBox, code: h.src, dataDir: h.sdir, bucket: h.srcBucket, aws: h.srcAWS,
1832	route: func() registry.Route { return h.srcRoute },
1833	inputs: func() *provision.MoveInputs { return h.read(t, p1Host).Move.Rollback.RecoveryInputs }})
1834	h.recoveryWake = p.recoveryWake
1835	return p
1836	}
1837
1838	// revertRecovery binds a revert harness's source box (ca-box) to the
1839	// production mode, as sourceRecovery does for a rollback: the revert's
1840	// recovery adapter wakes the claim runtime on ca-box.
1841	func revertRecovery(t *testing.T, h *revertHarness) *prodProbe {
1842	t.Helper()
1843	sourceServed(t, h.phase1Harness, h.cfg.DataDir)
1844	p := newProdWake(t, h.phase1Harness, prodSide{box: h.srcBox, code: h.src, dataDir: h.cfg.DataDir, bucket: h.srcBucket, aws: h.srcAWS,
1845	route: func() registry.Route { return h.route },
1846	inputs: func() *provision.MoveInputs { return h.read(t, p1Host).Move.RecoveryInputs }})
1847	h.cfg.recoveryProbe = func(ctx context.Context, s release.Store, r provision.Record, req recoveryProbeRequest) (recoveryProbeOutcome, error) {
1848	h.probes = append(h.probes, req)
1849	w := h.w()
1850	a := recoveryProbeAdapter{Box: w.cfg.Box, Claims: true, Route: w.cfg.Route, Current: w.recoveryInputs, Wake: p.recoveryWake}
1851	return a.probe(ctx, s, r, req)
1852	}
1853	return p
1854	}
1855
1856	// sourceServed makes the source box and its bucket what serving the
1857	// instance under claims leaves (the harness built them by hand): the
1858	// bucket versioned with a generation.json naming src_gen (sourceMarker);
1859	// the source's installation in the claim's inventory at the flip's source
1860	// stamp; src_gen's files manifest in its bucket, built from the source's
1861	// files and env; and src_gen's origin recorded, as an admission through the
1862	// real CAS (or the #250 backfill) records it.
1863	func sourceServed(t *testing.T, h *phase1Harness, dataDir string) {
1864	t.Helper()
1865	ctx := context.Background()
1866	sourceMarker(t, h)
1867	c := h.claim(t)
1868	var err error
1869	// The source's installation: the flip's journalled stamp and src_gen,
1870	// or, for a revert whose holder never left the source, the holder's.
1871	stamp, gen := c.Holder.Stamp, c.Holder.Generation
1872	if raw, ok := c.Op.Journal[hostclaim.JournalSourceStamp]; ok {
1873	if stamp, err = strconv.ParseUint(raw, 10, 64); err != nil || stamp == 0 {
1874	t.Fatalf("fixture: the claim's source stamp %q: %v", raw, err)
1875	}
1876	gen = c.Op.Journal[hostclaim.JournalSrcGen]
1877	}
1878	// The box's guard directories and its installation record at that
1879	// stamp, unless a step of the recovery already wrote it.
1880	for _, d := range []string{"hostguard", ".parked"} {
1881	if err := os.MkdirAll(filepath.Join(dataDir, d), 0o770); err != nil {
1882	t.Fatal(err)
1883	}
1884	}
1885	record := filepath.Join(dataDir, "hostguard", p1Host+".json")
1886	if _, err := os.Stat(record); errors.Is(err, os.ErrNotExist) {
1887	body := fmt.Sprintf(`{"v":1,"host":%q,"incarnation":%q,"active_stamp":%d,"retired_stamps":[]}`, p1Host, h.inc, stamp)
1888	if err := os.WriteFile(record, []byte(body), 0o660); err != nil {
1889	t.Fatal(err)
1890	}
1891	}
1892	rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
1893	if c.Inventory == nil {
1894	c.Inventory = map[string]hostclaim.InventoryEntry{}
1895	}
1896	c.Inventory[h.srcBox] = hostclaim.InventoryEntry{Layout: "flat", Paths: map[string]uint64{p1Host + ".db": stamp, p1Host + ".db.gen": stamp, p1Host: stamp, p1Host + ".env": stamp}}
1897	})
1898	if _, err = buildPreparedManifest(ctx, h.vs, h.srcBucket, cutRoot, gen, p1ID, stamp, filepath.Join(dataDir, p1Host), filepath.Join(dataDir, p1Host+".env"), true,
1899	hostfiles.WakeState{V: 1, Host: p1Host, Timers: []hostfiles.WakeTimer{}}); err != nil {
1900	t.Fatalf("src_gen's files manifest: %v", err)
1901	}
1902	recordOrigins(t, h.s, p1Host, hostclaim.OriginTuple{Incarnation: h.inc, Location: cutRoot, Region: park.PlacementRegion(h.src), Generation: gen})
1903	}
1904
1905	// firstMoving is the first claim written with the move's op: the start
1906	// CAS's.
1907	func firstMoving(t *testing.T, writes []hostclaim.Claim) *hostclaim.Claim {
1908	t.Helper()
1909	for i := range writes {
1910	if writes[i].Op.Kind == hostclaim.KindMoving {
1911	return &writes[i]
1912	}
1913	}
1914	t.Fatal("no claim was written with the move's op")
1915	return nil
1916	}
1917
1918	// sourceMarker makes the source's bucket what a source that served the
1919	// instance leaves: every object versioned (the harness stored the replica
1920	// directly), and the location's generation.json a marker naming src_gen,
1921	// cur, on ca-box (the harness's placeholder is not one).
1922	func sourceMarker(t *testing.T, h *phase1Harness) {
1923	t.Helper()
1924	body, err := json.Marshal(genselect.Marker{Op: "cur", Incarnation: h.inc, Host: p1Host, Box: h.srcBox, InstalledAt: p1Created})
1925	if err != nil {
1926	t.Fatal(err)
1927	}
1928	h.f.store(h.srcBucket, cutRoot+"/generation.json", fakeMoveObject{lease.MirrorObject{ETag: "marker", Size: int64(len(body))}, string(body)})
1929	h.f.setVersion(h.srcBucket, cutRoot+"/generation.json", h.f.nextVersion())
1930	for k := range h.f.buckets[h.srcBucket] {
1931	if h.f.versionOf(h.srcBucket, k) == "" {
1932	h.f.setVersion(h.srcBucket, k, h.f.nextVersion())
1933	}
1934	}
1935	}
1936
1937	// homeRegionalOn is home-box's regional runner over a harness's store and
1938	// lease table.
1939	func homeRegionalOn(h *phase1Harness, leases park.Leases) hostbox.Regional {
1940	return regionalRunner(h, leases, "ie", "ie", "data-home")
1941	}
1942
1943	// TestCommittedLandingIntoHomeConvertsOnTargetFence (Ask H1's witness):
1944	// after the box's commit of a move into home (uk -> home, the home box dst
1945	// in roster region ie, the fleet's home), dst is fenced as terminated, and
1946	// region HA's committed-landing conversion turns the claim into a failover
1947	// relocation: intent.region is the roster name ie, move_to_region the
1948	// move's @home, the home binding kept. The commit's chores (the lane park,
1949	// owed under @home or, retained, under ie; the source's location park in
1950	// uk) come through byte-identical, each in the spelling it was owed in.
1951	func TestCommittedLandingIntoHomeConvertsOnTargetFence(t *testing.T) {
1952	for _, lane := range []string{park.HomeRegion, "ie"} {
1953	t.Run("lane owed under "+lane, func(t *testing.T) {
1954	ctx := t.Context()
1955	w := newLandingWorldHome(t, true, nil, "ie")
1956	rec, err := provision.GetRecord(ctx, w.store, "bab", "jam", w.obs.Host)
1957	if err != nil {
1958	t.Fatal(err)
1959	}
1960	o, err := hostclaim.Read(ctx, w.store, w.obs.Host)
1961	if err != nil {
1962	t.Fatal(err)
1963	}
1964	source, staging, err := moves.CommitScopes(o.Claim, rec.Move.FlippedAt)
1965	if err != nil {
1966	t.Fatal(err)
1967	}
1968	id := o.Claim.Op.ID
1969	if lane != park.HomeRegion {
1970	// The lane park already owed in its retained spelling.
1971	if o, err = hostclaim.Update(ctx, w.store, o, func(c *hostclaim.Claim) error {
1972	s := staging
1973	c.Obligations = append(c.Obligations, hostclaim.Obligation{ID: retainedID(20), Op: id, Kind: "park-replica", Region: lane, Retirement: retainedID(21), Owner: "region:" + lane, Park: &s})
1974	return nil
1975	}); err != nil {
1976	t.Fatal(err)
1977	}
1978	}
1979	// The box's commit, as commitClaim writes it: the record's codes
1980	// through park.PlacementRegion, the target @home.
1981	if o, err = hostclaim.Update(ctx, w.store, o, hostclaim.KeepWriterSession(id, hostclaim.Commit(id, park.PlacementRegion(rec.Move.From), "src", park.PlacementRegion(rec.Move.To), source, staging, newClaimID))); err != nil {
1982	t.Fatal(err)
1983	}
1984	if o.Claim.Op.Phase != hostclaim.PhaseCommitted || o.Claim.Op.Intent["to"] != park.HomeRegion {
1985	t.Fatalf("the commit: %+v", o.Claim.Op)
1986	}
1987	committed := o.Claim.Obligations
1988	lanes := 0
1989	for _, ob := range committed {
1990	if ob.Park != nil && ob.Park.Location == moves.ScopeLocation(w.obs.Host, id) {
1991	lanes++
1992	if ob.Region != lane || ob.Owner != "region:"+lane {
1993	t.Fatalf("the lane park after the commit: %+v", ob)
1994	}
1995	}
1996	}
1997	if lanes != 1 {
1998	t.Fatalf("the commit owes %d lane parks: %+v", lanes, committed)
1999	}
2000
2001	if err := convertFencedLanding(w.homeWorker(w.enrollment["wrk"], "ie", "ie"), w.store)(ctx, w.store, o); err != nil {
2002	t.Fatalf("convert: %v", err)
2003	}
2004	after, err := hostclaim.Read(ctx, w.store, w.obs.Host)
2005	if err != nil {
2006	t.Fatal(err)
2007	}
2008	c := after.Claim
2009	if c.Op.Kind != hostclaim.KindRelocating || c.Op.Journal["conversion"] != "committed-landing" || c.Op.Journal["conversion_landing_phase"] != hostclaim.PhaseCommitted {
2010	t.Fatalf("not converted from committed: %+v", c.Op)
2011	}
2012	if c.Op.Intent["region"] != "ie" || c.Op.Intent["move_to_region"] != park.HomeRegion || c.Op.Intent["home_region"] != "ie" || c.Op.Journal["move_from_region"] != "uk" {
2013	t.Fatalf("intent %v journal from %q", c.Op.Intent, c.Op.Journal["move_from_region"])
2014	}
2015	for _, before := range committed {
2016	kept := false
2017	for _, now := range c.Obligations {
2018	if now.ID == before.ID {
2019	if obligationsJSON(t, []hostclaim.Obligation{now}) != obligationsJSON(t, []hostclaim.Obligation{before}) {
2020	t.Fatalf("the conversion rewrote %+v as %+v", before, now)
2021	}
2022	kept = true
2023	}
2024	}
2025	if !kept {
2026	t.Fatalf("the conversion dropped %+v", before)
2027	}
2028	}
2029	for _, now := range c.Obligations {
2030	if now.Region == "" {
2031	t.Fatalf("a chore in the unset region: %+v", now)
2032	}
2033	// From clean state (the lane not retained), every chore is
2034	// the source's (uk) or @home: nothing supplied the roster
2035	// name.
2036	if lane == park.HomeRegion && now.Region != "uk" && (now.Region != park.HomeRegion || now.Owner != "region:"+park.HomeRegion) {
2037	t.Fatalf("a chore %+v after a commit from clean state", now)
2038	}
2039	}
2040	})
2041	}
2042	}
2043