carlos / platform Public

Download this file

1	package host
2
3	import (
4	"context"
5	"crypto/sha256"
6	"encoding/hex"
7	"encoding/json"
8	"errors"
9	"fmt"
10	"os"
11	"path"
12	"path/filepath"
13	"reflect"
14	"sort"
15	"strings"
16	"sync"
17	"testing"
18	"time"
19
20	"github.com/carlosframework/platform/internal/hostbox"
21	"github.com/carlosframework/platform/internal/hostclaim"
22	"github.com/carlosframework/platform/internal/hostfiles"
23	"github.com/carlosframework/platform/internal/hostwrite"
24	"github.com/carlosframework/platform/internal/instancekey"
25	"github.com/carlosframework/platform/internal/lease"
26	"github.com/carlosframework/platform/internal/lease/leasetest"
27	"github.com/carlosframework/platform/internal/moves"
28	"github.com/carlosframework/platform/internal/release"
29	)
30
31	// casBucketStore is a release.Store over one leasetest bucket: current
32	// reads with the latest version id as the etag, create-only and
33	// compare-and-swap writes. It stands in for the S3 store the production
34	// provider writes the data bucket through.
35	type casBucketStore struct {
36	b *leasetest.VersionedBucket
37	bucket string
38	}
39
40	func (s casBucketStore) Get(_ context.Context, key string) ([]byte, string, error) {
41	body, ok := s.b.Current(s.bucket, key)
42	if !ok {
43	return nil, "", os.ErrNotExist
44	}
45	return body, s.b.LatestID(s.bucket, key), nil
46	}
47
48	func (s casBucketStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
49	if ifMatch == "" {
50	id, err := s.b.PutIfAbsent(ctx, s.bucket, key, body)
51	if errors.Is(err, lease.ErrExists) {
52	return "", release.ErrCASConflict
53	}
54	return id, err
55	}
56	if s.b.LatestID(s.bucket, key) != ifMatch {
57	return "", release.ErrCASConflict
58	}
59	return s.b.Put(s.bucket, key, body), nil
60	}
61
62	func (s casBucketStore) List(ctx context.Context, prefix string) ([]string, error) {
63	vs, err := s.b.ListVersions(ctx, s.bucket, prefix)
64	if err != nil {
65	return nil, err
66	}
67	var out []string
68	for _, v := range vs {
69	if v.IsLatest && !v.DeleteMarker {
70	out = append(out, v.Key)
71	}
72	}
73	return out, nil
74	}
75
76	func (casBucketStore) ListDirs(context.Context, string) ([]string, error) {
77	return nil, errors.New("test: not implemented")
78	}
79	func (casBucketStore) Delete(context.Context, string) error {
80	return errors.New("test: not implemented")
81	}
82
83	// recordingReader records every exact-version read.
84	type recordingReader struct {
85	lease.VersionedStore
86	mu sync.Mutex
87	reads []string // key@version
88	}
89
90	func (r *recordingReader) GetVersion(ctx context.Context, bucket, key, version string) ([]byte, error) {
91	r.mu.Lock()
92	r.reads = append(r.reads, key+"@"+version)
93	r.mu.Unlock()
94	return r.VersionedStore.GetVersion(ctx, bucket, key, version)
95	}
96
97	type filesFixture struct {
98	t *testing.T
99	claims release.Store
100	bucket *leasetest.VersionedBucket
101	data *recordingReader
102	files casBucketStore
103	dir string
104	f *movesFiles
105	// work is the start's captured work, modelled as the lock it holds
106	// (the hostguard's): With takes mu, runs fn and releases it. workRan
107	// counts the calls; workErr refuses them, every call when workErrAt is
108	// 0, else the call with that 1-based index only (a crash at that
109	// point).
110	mu sync.Mutex
111	workRan int
112	workErr error
113	workErrAt int
114	}
115
116	const filesHost = "shop.example.com"
117
118	// filesCreated and filesInc are the fixture host's creation and incarnation
119	// (hostclaim.Incarnation), so its location is a valid origin tuple
120	// (instances/<host>/<incarnation>) for claim Plan 15's proof.
121	var (
122	filesCreated = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
123	filesInc = func() string {
124	inc, err := hostclaim.Incarnation("bab", "jam", filesHost, filesCreated)
125	if err != nil {
126	panic(err)
127	}
128	return inc
129	}()
130	filesLoc = "instances/" + filesHost + "/" + filesInc
131	)
132
133	func (x *filesFixture) With(_ context.Context, _ hostwrite.Identity, fn func() error) error {
134	x.mu.Lock()
135	defer x.mu.Unlock()
136	x.workRan++
137	if x.workErr != nil && (x.workErrAt == 0 || x.workErrAt == x.workRan) {
138	return x.workErr
139	}
140	return fn()
141	}
142
143	func newFilesFixture(t *testing.T) *filesFixture {
144	t.Helper()
145	// Claim Plan 15: the proof reads the claim's recorded admissions from
146	// the claims store under both switches.
147	t.Setenv("CARLOS_HOST_CLAIM", "1")
148	t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1")
149	b := leasetest.NewVersionedBucket()
150	b.AddBucket("data")
151	x := &filesFixture{t: t, claims: release.NewDirStore(t.TempDir()), bucket: b, data: &recordingReader{VersionedStore: b}, files: casBucketStore{b: b, bucket: "data"}, dir: t.TempDir()}
152	x.f = &movesFiles{Files: x.files, Data: x.data, Bucket: "data", HomeRegion: "ie", Region: "uk", Box: "uk-box", DataDir: x.dir, Proof: ClaimOriginProof{Claims: x.claims, HomeRegion: "ie", Region: "uk"},
153	EnvFile: flatClaimEnvFile(x.dir)}
154	return x
155	}
156
157	func (x *filesFixture) envPath() string { return filepath.Join(x.dir, filesHost+".env") }
158
159	func (x *filesFixture) writeEnv(body string) {
160	x.t.Helper()
161	if err := os.WriteFile(x.envPath(), []byte(body), 0o600); err != nil {
162	x.t.Fatal(err)
163	}
164	}
165
166	// publish writes a manifest revision for generation gen through the
167	// ordinary publisher (a predecessor's session output) and returns it.
168	func (x *filesFixture) publish(gen string, stage func(p *hostfiles.Publisher)) hostfiles.Manifest {
169	x.t.Helper()
170	p, err := hostfiles.Open(context.Background(), x.files, movesFilesPrefix(filesLoc, gen))
171	if err != nil {
172	x.t.Fatal(err)
173	}
174	stage(p)
175	if _, err := p.Commit(context.Background()); err != nil {
176	x.t.Fatal(err)
177	}
178	m, found, err := hostfiles.Load(context.Background(), x.files, movesFilesPrefix(filesLoc, gen))
179	if err != nil || !found {
180	x.t.Fatal(err, found)
181	}
182	return m
183	}
184
185	func (x *filesFixture) manifest(gen string) (hostfiles.Manifest, bool) {
186	x.t.Helper()
187	m, found, err := hostfiles.Load(context.Background(), x.files, movesFilesPrefix(filesLoc, gen))
188	if err != nil {
189	x.t.Fatal(err)
190	}
191	return m, found
192	}
193
194	// servingObserved is a serving claim on uk-box whose holder is at gen with
195	// the cold admission chain given, newest last: each entry is
196	// "<generation>:<region>:<predecessor>:<predecessor region>". It is a
197	// hand-built claim: nothing in it is proven until observe stores it with
198	// recorded admissions (recordOrigins).
199	func servingObserved(gen string, chain ...string) hostclaim.Observed {
200	h := &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + filesHost + ".json", Account: "bab", App: "jam", CreatedAt: filesCreated, Incarnation: filesInc, Location: filesLoc, Generation: gen, Session: 3, WriterBox: "uk-box", Stamp: 7}
201	for i, c := range chain {
202	p := strings.Split(c, ":")
203	h.Admissions = append(h.Admissions, hostclaim.AdmissionRecord{Generation: p[0], Region: p[1], Predecessor: p[2], PredecessorRegion: p[3], Mode: "cold", Session: uint64(3 + i), Box: "uk-box"})
204	}
205	if n := len(h.Admissions); n > 0 {
206	h.Session = h.Admissions[n-1].Session
207	}
208	return hostclaim.Observed{Host: filesHost, ETag: "e1", Claim: &hostclaim.Claim{V: 2, Host: filesHost, Gen: 9, Holder: h, Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "uk-box"}}}
209	}
210
211	// recordOrigins records tuples as the claim's admitted origins, as the
212	// claim's CAS records an admission and the #250 files backfill records a
213	// live origin into an admitted prefix: one immutable node per tuple under
214	// control/host-origins/<host>/, chained from the claim's current head, and
215	// the head written on the stored claim. The claim must already be stored.
216	func recordOrigins(t *testing.T, claims release.Store, host string, tuples ...hostclaim.OriginTuple) {
217	t.Helper()
218	ctx := context.Background()
219	o, err := hostclaim.Read(ctx, claims, host)
220	if err != nil {
221	t.Fatal(err)
222	}
223	if o.Absent {
224	t.Fatalf("recordOrigins: %s has no stored claim", host)
225	}
226	c := o.Claim
227	head := hostclaim.OriginHead{}
228	if c.Origins != nil {
229	head = *c.Origins
230	}
231	type node struct {
232	V int `json:"v"`
233	Host string `json:"host"`
234	Count uint64 `json:"count"`
235	Previous string `json:"previous,omitempty"`
236	Origin hostclaim.OriginTuple `json:"origin"`
237	}
238	for _, tuple := range tuples {
239	body, err := json.Marshal(node{V: 1, Host: host, Count: head.Count + 1, Previous: head.Digest, Origin: tuple})
240	if err != nil {
241	t.Fatal(err)
242	}
243	sum := sha256.New()
244	sum.Write([]byte("carlos-admitted-origin-v1\x00"))
245	sum.Write(body)
246	digest := hex.EncodeToString(sum.Sum(nil))
247	key := "control/host-origins/" + host + "/" + digest + ".json"
248	if err := hostclaim.ValidateOriginObject(key, body); err != nil {
249	t.Fatalf("recordOrigins: %v", err)
250	}
251	if _, err := claims.Put(ctx, key, body, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {
252	t.Fatal(err)
253	}
254	head = hostclaim.OriginHead{Digest: digest, Count: head.Count + 1}
255	}
256	c.Origins = &head
257	raw, err := json.Marshal(c)
258	if err != nil {
259	t.Fatal(err)
260	}
261	key, err := hostclaim.Key(host)
262	if err != nil {
263	t.Fatal(err)
264	}
265	if _, err := claims.Put(ctx, key, raw, o.ETag); err != nil {
266	t.Fatal(err)
267	}
268	}
269
270	// observe stores the hand-built observation's claim in the fixture's claims
271	// store, records the named generations as admitted origins of the holder's
272	// incarnation and location in this region (the admitted generation among
273	// them, as its admission records it), and returns the stored observation,
274	// which is what Materialize is given and what the proof authenticates.
275	func (x *filesFixture) observe(o hostclaim.Observed, recorded ...string) hostclaim.Observed {
276	x.t.Helper()
277	ctx := context.Background()
278	c := o.Claim
279	if err := c.Validate(); err != nil {
280	x.t.Fatalf("observe: %v", err)
281	}
282	key, err := hostclaim.Key(o.Host)
283	if err != nil {
284	x.t.Fatal(err)
285	}
286	etag := ""
287	if stored, err := hostclaim.Read(ctx, x.claims, o.Host); err == nil && !stored.Absent {
288	// A re-observation keeps the chain the store already records: the
289	// head is the claim's, never a hand-built claim's.
290	etag, c.Origins = stored.ETag, stored.Claim.Origins
291	}
292	raw, err := json.Marshal(c)
293	if err != nil {
294	x.t.Fatal(err)
295	}
296	if _, err := x.claims.Put(ctx, key, raw, etag); err != nil {
297	x.t.Fatal(err)
298	}
299	tuples := make([]hostclaim.OriginTuple, 0, len(recorded))
300	for _, gen := range recorded {
301	tuples = append(tuples, hostclaim.OriginTuple{Incarnation: c.Holder.Incarnation, Location: c.Holder.Location, Region: "uk", Generation: gen})
302	}
303	if len(tuples) > 0 {
304	recordOrigins(x.t, x.claims, o.Host, tuples...)
305	}
306	stored, err := hostclaim.Read(ctx, x.claims, o.Host)
307	if err != nil {
308	x.t.Fatal(err)
309	}
310	return stored
311	}
312
313	func launchInputs(newGen string) ClaimLaunchInputs {
314	return ClaimLaunchInputs{Identity: hostwrite.Identity{Config: "sha256:c", Release: "sha256:r"}, Admission: hostclaim.Admission{Box: "uk-box", Mode: "cold", NewGeneration: newGen, Region: "uk"}}
315	}
316
317	func sha(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
318
319	// The prepared manifest (spec §7.6, R19-1; claim rev 6.30 R18-1): one
320	// create-only manifest at sequence 0 under files/g-<P>/ in the location,
321	// every content object create-only at sequence 0 under the same prefix,
322	// the env file and the wake state as platform items, and the binding
323	// {location, P, op, stamp} returned for the op journal. A different build
324	// of the same generation is refused and leaves the manifest as it was.
325	func TestPreparedManifestIsCreateOnlySequenceZeroAndBound(t *testing.T) {
326	x := newFilesFixture(t)
327	ctx := context.Background()
328	dir := filepath.Join(x.dir, filesHost)
329	for rel, body := range map[string]string{"a.txt": "A", "uploads/b.bin": "BB"} {
330	if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, rel)), 0o755); err != nil {
331	t.Fatal(err)
332	}
333	if err := os.WriteFile(filepath.Join(dir, rel), []byte(body), 0o640); err != nil {
334	t.Fatal(err)
335	}
336	}
337	if err := os.Symlink("a.txt", filepath.Join(dir, "link")); err != nil {
338	t.Fatal(err)
339	}
340	x.writeEnv("K=1\n")
341	wake := hostfiles.WakeState{V: 1, Host: filesHost, Timers: []hostfiles.WakeTimer{{Name: "tick", Kind: "cron", Spec: "* * * * *", NextAt: 5}}, NextDue: 5, Replication: "hot"}
342	pm, err := x.f.BuildPrepared(ctx, filesLoc, "mv1", "mv1", 7, dir, x.envPath(), true, wake)
343	if err != nil {
344	t.Fatal(err)
345	}
346	prefix := movesFilesPrefix(filesLoc, "mv1")
347	if pm.Key != hostfiles.ManifestKey(prefix) || pm.Op != "mv1" || pm.Stamp != 7 || len(pm.SHA256) != 64 {
348	t.Fatalf("binding: %+v", pm)
349	}
350	raw, ok := x.bucket.Current("data", pm.Key)
351	if !ok || sha(raw) != pm.SHA256 {
352	t.Fatalf("the manifest at %s is not the journalled bytes: %v", pm.Key, ok)
353	}
354	m, found := x.manifest("mv1")
355	if !found || m.Seq != 0 {
356	t.Fatalf("manifest found %v seq %d", found, m.Seq)
357	}
358	if err := hostfiles.ValidateManifest(m, []string{prefix}); err != nil {
359	t.Fatal(err)
360	}
361	if len(m.Entries) != 2 || m.Entries["a.txt"].Mode != 0o640 || m.Entries["uploads/b.bin"].SHA256 != sha([]byte("BB")) {
362	t.Fatalf("entries: %+v", m.Entries)
363	}
364	for _, e := range m.Entries {
365	if e.Seq != 0 || !strings.HasPrefix(e.Object, prefix+"objects/0000000000000000-") {
366	t.Fatalf("entry object: %+v", e)
367	}
368	}
369	if e := m.Platform[hostfiles.EnvItem]; e.SHA256 != sha([]byte("K=1\n")) || e.Seq != 0 {
370	t.Fatalf("env item: %+v", e)
371	}
372	ws, found, err := hostfiles.LoadWake(ctx, x.files, prefix, m, nil)
373	if err != nil || !found || len(ws.Timers) != 1 || ws.NextDue != 5 || ws.Replication != "hot" {
374	t.Fatalf("wake: %+v %v %v", ws, found, err)
375	}
376	// A different build of the same generation: refused, the manifest kept.
377	if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("changed"), 0o640); err != nil {
378	t.Fatal(err)
379	}
380	if _, err := x.f.BuildPrepared(ctx, filesLoc, "mv1", "mv1", 7, dir, x.envPath(), true, wake); !errors.Is(err, hostfiles.ErrManifestExists) {
381	t.Fatalf("second build: %v", err)
382	}
383	if again, _ := x.bucket.Current("data", pm.Key); sha(again) != pm.SHA256 {
384	t.Fatal("the prepared manifest was replaced")
385	}
386	}
387
388	// The build never turns missing inputs into an empty image (fix round 1,
389	// finding 4): the files directory the install placed must be there, and
390	// the env file must be there exactly when the install placed one (the
391	// archive's evidence, wantEnv). A tree that vanished, an expected env file
392	// that is gone, or an env file nobody placed, refuses before any write;
393	// an install whose archive carried no env file builds a manifest without
394	// the item.
395	func TestPreparedManifestRefusesMissingInputs(t *testing.T) {
396	x := newFilesFixture(t)
397	ctx := context.Background()
398	dir := filepath.Join(x.dir, filesHost)
399	wake := hostfiles.WakeState{V: 1, Host: filesHost, Timers: []hostfiles.WakeTimer{}}
400	build := func(gen string, wantEnv bool) (preparedManifest, error) {
401	return buildPreparedManifest(ctx, x.bucket, "data", filesLoc, gen, gen, 7, dir, x.envPath(), wantEnv, wake)
402	}
403	written := func() int {
404	vs, err := x.bucket.ListVersions(ctx, "data", strings.TrimSuffix(filesLoc, "/")+"/files/")
405	if err != nil {
406	t.Fatal(err)
407	}
408	return len(vs)
409	}
410	// No tree at all.
411	x.writeEnv("K=1\n")
412	if _, err := build("g1", true); err == nil {
413	t.Fatal("built over a missing files directory")
414	}
415	if err := os.MkdirAll(dir, 0o755); err != nil {
416	t.Fatal(err)
417	}
418	if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("A"), 0o640); err != nil {
419	t.Fatal(err)
420	}
421	// The env file the install placed is gone.
422	if err := os.Remove(x.envPath()); err != nil {
423	t.Fatal(err)
424	}
425	if _, err := build("g1", true); err == nil {
426	t.Fatal("built without the env file the install placed")
427	}
428	// An env file nobody placed.
429	x.writeEnv("K=1\n")
430	if _, err := build("g1", false); err == nil {
431	t.Fatal("built over an env file the install did not place")
432	}
433	if n := written(); n != 0 {
434	t.Fatalf("%d objects written by refused builds", n)
435	}
436	// An archive without an env file: a manifest without the item.
437	if err := os.Remove(x.envPath()); err != nil {
438	t.Fatal(err)
439	}
440	pm, err := build("g1", false)
441	if err != nil {
442	t.Fatal(err)
443	}
444	m, found := x.manifest("g1")
445	if !found || len(m.Entries) != 1 {
446	t.Fatalf("manifest: %+v %v", m, found)
447	}
448	if _, has := m.Platform[hostfiles.EnvItem]; has || pm.Key == "" {
449	t.Fatalf("an env item for an archive without one: %+v", m.Platform)
450	}
451	}
452
453	// The build never reads a file past the limit (fix round 1, finding 5):
454	// a file whose size is over hostfiles.MaxFileBytes is recorded as skipped
455	// from its stat alone, and every read is bounded at the limit plus one
456	// byte, so a file that grows while it is read is recorded as skipped
457	// rather than read whole. A sparse file stands in for the oversized one.
458	func TestPreparedBuildBoundsEveryRead(t *testing.T) {
459	x := newFilesFixture(t)
460	ctx := context.Background()
461	dir := filepath.Join(x.dir, filesHost)
462	if err := os.MkdirAll(dir, 0o755); err != nil {
463	t.Fatal(err)
464	}
465	if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("A"), 0o640); err != nil {
466	t.Fatal(err)
467	}
468	big := filepath.Join(dir, "big.bin")
469	if err := os.WriteFile(big, nil, 0o640); err != nil {
470	t.Fatal(err)
471	}
472	if err := os.Truncate(big, hostfiles.MaxFileBytes+1); err != nil {
473	t.Fatal(err)
474	}
475	x.writeEnv("K=1\n")
476	reads := 0
477	orig := readFileBounded
478	readFileBounded = func(root *os.Root, rel string, limit int64) ([]byte, bool, error) {
479	if path.Base(rel) == "big.bin" {
480	reads++
481	}
482	return orig(root, rel, limit)
483	}
484	defer func() { readFileBounded = orig }()
485	if _, err := x.f.BuildPrepared(ctx, filesLoc, "g1", "g1", 7, dir, x.envPath(), true, hostfiles.WakeState{V: 1, Host: filesHost, Timers: []hostfiles.WakeTimer{}}); err != nil {
486	t.Fatal(err)
487	}
488	m, _ := x.manifest("g1")
489	if e := m.Entries["big.bin"]; e.Skipped == "" || e.Object != "" {
490	t.Fatalf("the oversized file was not recorded as skipped: %+v", e)
491	}
492	if reads != 0 {
493	t.Fatalf("the oversized file was read %d times", reads)
494	}
495	if e := m.Entries["a.txt"]; e.SHA256 != sha([]byte("A")) {
496	t.Fatalf("a.txt: %+v", e)
497	}
498	// The bounded read itself: a file larger than the limit is reported as
499	// too large with at most limit+1 bytes read; one at the limit is read
500	// whole.
501	small := filepath.Join(x.dir, "small")
502	if err := os.WriteFile(small, []byte("0123456789"), 0o600); err != nil {
503	t.Fatal(err)
504	}
505	xroot, err := os.OpenRoot(x.dir)
506	if err != nil {
507	t.Fatal(err)
508	}
509	defer xroot.Close()
510	body, tooLarge, err := orig(xroot, "small", 4)
511	if err != nil || !tooLarge || len(body) > 5 {
512	t.Fatalf("over the limit: %q %v %v", body, tooLarge, err)
513	}
514	body, tooLarge, err = orig(xroot, "small", 10)
515	if err != nil || tooLarge || string(body) != "0123456789" {
516	t.Fatalf("at the limit: %q %v %v", body, tooLarge, err)
517	}
518	}
519
520	// A replay of the build (a crash after the objects, before or after the
521	// manifest) reuses every object it finds by digest and the manifest by
522	// key: the same binding, one version of each object, no new object.
523	func TestPreparedManifestReplayReusesObjects(t *testing.T) {
524	x := newFilesFixture(t)
525	ctx := context.Background()
526	dir := filepath.Join(x.dir, filesHost)
527	if err := os.MkdirAll(dir, 0o755); err != nil {
528	t.Fatal(err)
529	}
530	if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("A"), 0o640); err != nil {
531	t.Fatal(err)
532	}
533	x.writeEnv("K=1\n")
534	wake := hostfiles.WakeState{V: 1, Host: filesHost, Timers: []hostfiles.WakeTimer{}}
535	first, err := x.f.BuildPrepared(ctx, filesLoc, "mv1", "mv1", 7, dir, x.envPath(), true, wake)
536	if err != nil {
537	t.Fatal(err)
538	}
539	second, err := x.f.BuildPrepared(ctx, filesLoc, "mv1", "mv1", 7, dir, x.envPath(), true, wake)
540	if err != nil {
541	t.Fatalf("replay: %v", err)
542	}
543	if first != second {
544	t.Fatalf("replay binding differs: %+v %+v", first, second)
545	}
546	vs, err := x.bucket.ListVersions(ctx, "data", movesFilesPrefix(filesLoc, "mv1"))
547	if err != nil {
548	t.Fatal(err)
549	}
550	if len(vs) != 4 { // three objects (a.txt, env, wake) and the manifest, one version each
551	t.Fatalf("%d versions under the prepared prefix after a replay: %+v", len(vs), vs)
552	}
553	}
554
555	// A cold admission establishes its generation's files manifest from the
556	// newest valid manifest of its admitted predecessor chain, read by the
557	// exact version the listing names: here g2 (the predecessor) never
558	// published a manifest, so g1's current revision is the baseline. The copy
559	// keeps every object origin-qualified under g1, continues the sequence,
560	// and is create-only under g3.
561	func TestEstablishCopiesTheNewestValidManifestByExactVersion(t *testing.T) {
562	x := newFilesFixture(t)
563	ctx := context.Background()
564	x.writeEnv("K=1\n")
565	x.publish("g1", func(p *hostfiles.Publisher) {
566	if err := p.Stage(ctx, "old.txt", []byte("old"), 0o600, false); err != nil {
567	t.Fatal(err)
568	}
569	})
570	g1 := x.publish("g1", func(p *hostfiles.Publisher) {
571	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
572	t.Fatal(err)
573	}
574	if err := p.StagePlatform(ctx, hostfiles.EnvItem, []byte("K=1\n")); err != nil {
575	t.Fatal(err)
576	}
577	})
578	if g1.Seq != 3 {
579	t.Fatalf("g1 at seq %d", g1.Seq)
580	}
581	o := x.observe(servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk"), "g1", "g3")
582	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); err != nil {
583	t.Fatal(err)
584	}
585	// One fence for the manifest's write (the unchanged env file takes no
586	// new object) and the local effect.
587	if x.workRan != 2 {
588	t.Fatalf("work ran %d times", x.workRan)
589	}
590	m, found := x.manifest("g3")
591	if !found {
592	t.Fatal("no manifest under g3")
593	}
594	if m.Seq != 3 || len(m.Entries) != 2 || m.Entries["a.txt"].Object != g1.Entries["a.txt"].Object || m.Entries["old.txt"].Object != g1.Entries["old.txt"].Object {
595	t.Fatalf("copied manifest: %+v", m)
596	}
597	if e := m.Platform[hostfiles.EnvItem]; e.Object != g1.Platform[hostfiles.EnvItem].Object {
598	t.Fatalf("the unchanged env file took a new object: %+v", e)
599	}
600	want := hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g1")) + "@" + x.bucket.LatestID("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g1")))
601	found = false
602	for _, r := range x.data.reads {
603	found = found || r == want
604	}
605	if !found {
606	t.Fatalf("g1's manifest was not read at its exact latest version %s: %v", want, x.data.reads)
607	}
608	if _, has := x.manifest("g2"); has {
609	t.Fatal("g2 gained a manifest")
610	}
611	// A repeat (a retry before the app executed) reads its own bytes back.
612	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); err != nil {
613	t.Fatalf("repeat: %v", err)
614	}
615	if again, _ := x.manifest("g3"); again.Seq != 3 {
616	t.Fatalf("the repeat replaced the manifest: seq %d", again.Seq)
617	}
618	}
619
620	// Origins outlive admission retention (fix round 1, finding 6; platform-1e
621	// P2). AdmitWriter keeps eight admissions, while a copied manifest carries
622	// its oldest objects for as long as they are live: after g1..g10 the
623	// chain ends at g3 and the newest manifest still names g1's object. The
624	// rule a reference is held to is the claim's (P6-D3): within the
625	// incarnation's location in this region's bucket, under a generation's
626	// files prefix. The admitted chain selects the newest manifest; it does
627	// not bound what that manifest may name. An object outside the location,
628	// or under the location but not under a generation's files prefix, still
629	// refuses.
630	func TestEstablishKeepsOriginsPastAdmissionPruning(t *testing.T) {
631	x := newFilesFixture(t)
632	ctx := context.Background()
633	x.writeEnv("K=1\n")
634	prev := x.publish("g1", func(p *hostfiles.Publisher) {
635	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
636	t.Fatal(err)
637	}
638	if err := p.StagePlatform(ctx, hostfiles.EnvItem, []byte("K=1\n")); err != nil {
639	t.Fatal(err)
640	}
641	})
642	gens := []string{"g1"}
643	for i := 2; i <= 10; i++ {
644	gen := fmt.Sprintf("g%d", i)
645	p, err := hostfiles.NewFromCopy(x.files, movesFilesPrefix(filesLoc, gen), prev, []string{movesFilesPrefix(filesLoc, "g1")})
646	if err != nil {
647	t.Fatal(err)
648	}
649	// Each copy carries its set as an establishment would: the previous
650	// set, the chain then (g1 and the previous generation) and itself.
651	p.SetOrigins(append(append([]string(nil), prev.Origins...), movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, gens[len(gens)-1]), movesFilesPrefix(filesLoc, gen)))
652	if _, err := p.Commit(ctx); err != nil {
653	t.Fatal(err)
654	}
655	prev, _ = x.manifest(gen)
656	gens = append(gens, gen)
657	}
658	// The observation Materialize is given: g11 admitted, the holder at
659	// g11, the eight newest admissions only: g4..g11.
660	var chain []string
661	for i := 4; i <= 11; i++ {
662	chain = append(chain, fmt.Sprintf("g%d:uk:g%d:uk", i, i-1))
663	}
664	c := servingObserved("g11", chain...)
665	if len(c.Claim.Holder.Admissions) != 8 {
666	t.Fatalf("%d admissions", len(c.Claim.Holder.Admissions))
667	}
668	// Provenance comes from the claim (platform-1e's ruling, fix round 3;
669	// claim Plan 15): the carried set is a hint, retained admissions are
670	// never evidence, and every origin must be proven by the claim's
671	// recorded admissions. g1 is retained-only (pruned from the chain and
672	// never recorded), so the establishment refuses; once the claim records
673	// g1 (its admission, or the #250 files backfill), it passes.
674	o := x.observe(c, "g11")
675	if err := x.f.Establish(ctx, o, launchInputs("g11"), x); !errors.Is(err, hostfiles.ErrOrigin) {
676	t.Fatalf("past pruning, g1 retained only: %v", err)
677	}
678	if _, found := x.manifest("g11"); found {
679	t.Fatal("a manifest was written over an unproven ancestor")
680	}
681	o = x.observe(c, "g1")
682	if err := x.f.Establish(ctx, o, launchInputs("g11"), x); err != nil {
683	t.Fatalf("establish past pruning with the claim's record: %v", err)
684	}
685	m, found := x.manifest("g11")
686	if !found || m.Entries["a.txt"].Object != movesFilesPrefix(filesLoc, "g1")+"objects/0000000000000001-"+sha([]byte("A")) {
687	t.Fatalf("g11: %+v %v", m, found)
688	}
689	// The carried set (fix round 2, finding 3; Task 6b): only what the
690	// entries reference, g1, plus the new generation. The hints for the
691	// generations between, which nothing references, are dropped.
692	want := []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "g11")}
693	if !reflect.DeepEqual(m.Origins, want) {
694	t.Fatalf("g11 origins %v, want %v", m.Origins, want)
695	}
696	_ = gens
697	// Under the location but not a generation's files prefix: refused.
698	for name, object := range map[string]string{
699	"another location": "instances/other/files/g-g1/objects/0000000000000001-" + sha([]byte("A")),
700	"not a files prefix": strings.TrimSuffix(filesLoc, "/") + "/g-g1/objects/0000000000000001-" + sha([]byte("A")),
701	"not a generation name": strings.TrimSuffix(filesLoc, "/") + "/files/g-/objects/0000000000000001-" + sha([]byte("A")),
702	} {
703	bad := prev
704	bad.Entries = map[string]hostfiles.Entry{"a.txt": {Object: object, SHA256: sha([]byte("A")), Size: 1, Seq: 1}}
705	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g10")), mustJSON(t, bad))
706	if err := x.f.Establish(ctx, o, launchInputs("g12"), x); !errors.Is(err, hostfiles.ErrOrigin) {
707	t.Fatalf("%s: %v", name, err)
708	}
709	}
710	if _, found := x.manifest("g12"); found {
711	t.Fatal("a manifest was written over a foreign reference")
712	}
713	}
714
715	// replacingStore replaces one key's object right after the first Get of
716	// it returns: another writer landing between a reader's check and its
717	// use.
718	type replacingStore struct {
719	release.Store
720	key string
721	replace func()
722	once sync.Once
723	}
724
725	func (s *replacingStore) Get(ctx context.Context, key string) ([]byte, string, error) {
726	body, etag, err := s.Store.Get(ctx, key)
727	if key == s.key && err == nil {
728	s.once.Do(s.replace)
729	}
730	return body, etag, err
731	}
732
733	// preparedFixture builds a prepared manifest for mv1 with its binding
734	// journalled, the env file carrying the minted key, and the probe's
735	// observation and inputs.
736	func preparedFixture(t *testing.T) (*filesFixture, preparedManifest, hostclaim.Observed, ClaimLaunchInputs) {
737	t.Helper()
738	x := newFilesFixture(t)
739	ctx := context.Background()
740	dir := filepath.Join(x.dir, filesHost)
741	if err := os.MkdirAll(dir, 0o755); err != nil {
742	t.Fatal(err)
743	}
744	if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("A"), 0o640); err != nil {
745	t.Fatal(err)
746	}
747	x.writeEnv("K=1\n")
748	pm, err := x.f.BuildPrepared(ctx, filesLoc, "mv1", "mv1", 7, dir, x.envPath(), true, hostfiles.WakeState{V: 1, Host: filesHost, Timers: []hostfiles.WakeTimer{}})
749	if err != nil {
750	t.Fatal(err)
751	}
752	x.journal("mv1", pm)
753	x.writeEnv("K=1\nCARLOS_INSTANCE_KEY=abc\n")
754	o := servingObserved("mv1", "cur:ie:@root:ie", "mv1:uk:cur:ie")
755	o.Claim.Op = hostclaim.Op{Kind: hostclaim.KindLanding, ID: "mv1", Phase: "probing", Exec: "uk-box", Participants: []string{"ie-box", "uk-box"}, Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}}
756	o = x.observe(o, "mv1")
757	in := launchInputs("mv1")
758	in.Prepared = true
759	return x, pm, o, in
760	}
761
762	// The probe's publisher is built from the bytes the establishment checked
763	// and the token captured with them, never from a second read (fix round
764	// 2, finding 2). Another writer replacing P's manifest between the check
765	// and the publication makes the compare-and-swap fail: nothing is
766	// published over bytes that were never compared with the journalled
767	// version, and the replacement stands as it was.
768	func TestEstablishPublishesOnlyOverTheBytesItChecked(t *testing.T) {
769	x, pm, o, in := preparedFixture(t)
770	ctx := context.Background()
771	other := hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: 5, Entries: map[string]hostfiles.Entry{"b.txt": {Object: movesFilesPrefix(filesLoc, "mv1") + "objects/0000000000000005-" + sha([]byte("B")), SHA256: sha([]byte("B")), Size: 1, Seq: 5}}, Platform: map[string]hostfiles.Entry{}}
772	replacement := mustJSON(t, other)
773	x.f.Files = &replacingStore{Store: x.files, key: pm.Key, replace: func() { x.bucket.Put("data", pm.Key, replacement) }}
774	if err := x.f.Establish(ctx, o, in, x); err == nil {
775	t.Fatal("published over a manifest replaced after the check")
776	}
777	cur, _ := x.bucket.Current("data", pm.Key)
778	if string(cur) != string(replacement) {
779	t.Fatalf("the replacement was published over: %s", cur)
780	}
781	if x.workRan != 0 && x.workRan > 2 {
782	t.Fatalf("the local effect ran after a refused publication (%d)", x.workRan)
783	}
784	}
785
786	// A prepared replay is accepted only as exactly what this establishment
787	// would have written (fix round 2 finding 6, fix round 3): from the
788	// prepared baseline and the establishment's own operations, the env
789	// object at the sequence after the prepared one, the manifest at that
790	// sequence, the carried set the admitted one, the entries and other
791	// platform items unchanged, the adoption item gone; and the env object
792	// present at its key with the env's digest. Nothing is taken from the
793	// replay's own numbers. With correct Origins, an env entry under another
794	// location or generation, of the wrong size, torn, or at a fabricated
795	// sequence (99, or 2 where the establishment writes 1) whose object does
796	// not exist, is refused and left as it is. The genuine replay is accepted
797	// and writes nothing.
798	func TestEstablishRefusesAForeignEnvEntryOnPreparedReplay(t *testing.T) {
799	x, pm, o, in := preparedFixture(t)
800	ctx := context.Background()
801	envBytes := []byte("K=1\nCARLOS_INSTANCE_KEY=abc\n")
802	envSHA := sha(envBytes)
803	raw, err := x.bucket.GetVersion(ctx, "data", pm.Key, pm.Version)
804	if err != nil {
805	t.Fatal(err)
806	}
807	var prepared hostfiles.Manifest
808	if err := json.Unmarshal(raw, &prepared); err != nil {
809	t.Fatal(err)
810	}
811	// The genuine establishment first: what this box writes, and the set it
812	// carries, are the shape every replay is held to.
813	if err := x.f.Establish(ctx, o, in, x); err != nil {
814	t.Fatal(err)
815	}
816	genuine, _ := x.manifest("mv1")
817	genuineRaw, _ := x.bucket.Current("data", pm.Key)
818	if genuine.Seq != 1 || genuine.Platform[hostfiles.EnvItem].Seq != 1 || len(genuine.Origins) == 0 {
819	t.Fatalf("the genuine establishment: %+v", genuine)
820	}
821	under := func(prefix string, seq int64) string { return fmt.Sprintf("%sobjects/%016d-%s", prefix, seq, envSHA) }
822	for name, tc := range map[string]struct {
823	seq int64
824	entry hostfiles.Entry
825	}{
826	"another location": {1, hostfiles.Entry{Object: under("instances/other/files/g-mv1/", 1), SHA256: envSHA, Size: int64(len(envBytes)), Mode: 0o600, Seq: 1}},
827	"another generation": {1, hostfiles.Entry{Object: under(movesFilesPrefix(filesLoc, "cur"), 1), SHA256: envSHA, Size: int64(len(envBytes)), Mode: 0o600, Seq: 1}},
828	"the wrong size": {1, hostfiles.Entry{Object: under(movesFilesPrefix(filesLoc, "mv1"), 1), SHA256: envSHA, Size: 3, Mode: 0o600, Seq: 1}},
829	"torn": {1, hostfiles.Entry{Object: under(movesFilesPrefix(filesLoc, "mv1"), 1), SHA256: envSHA, Size: int64(len(envBytes)), Mode: 0o600, Seq: 1, Torn: true}},
830	"a fabricated sequence 99": {99, hostfiles.Entry{Object: under(movesFilesPrefix(filesLoc, "mv1"), 99), SHA256: envSHA, Size: int64(len(envBytes)), Mode: 0o600, Seq: 99}},
831	"a fabricated sequence 2": {2, hostfiles.Entry{Object: under(movesFilesPrefix(filesLoc, "mv1"), 2), SHA256: envSHA, Size: int64(len(envBytes)), Mode: 0o600, Seq: 2}},
832	} {
833	bumped := hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: tc.seq, Entries: prepared.Entries, Platform: map[string]hostfiles.Entry{}, Origins: genuine.Origins}
834	for k, v := range prepared.Platform {
835	bumped.Platform[k] = v
836	}
837	bumped.Platform[hostfiles.EnvItem] = tc.entry
838	body := mustJSON(t, bumped)
839	x.bucket.Put("data", pm.Key, body)
840	if err := x.f.Establish(ctx, o, in, x); err == nil {
841	t.Fatalf("%s: accepted as this box's establishment", name)
842	}
843	if cur, _ := x.bucket.Current("data", pm.Key); string(cur) != string(body) {
844	t.Fatalf("%s: the manifest was replaced: %s", name, cur)
845	}
846	}
847	// The genuine bytes with their env object gone, or corrupt at its key
848	// (other bytes, the entry's digest and size no longer its own): refused
849	// too. The object is read and verified before a replay is accepted.
850	envObject := genuine.Platform[hostfiles.EnvItem].Object
851	x.bucket.Put("data", pm.Key, genuineRaw)
852	x.bucket.BareDelete("data", envObject)
853	if err := x.f.Establish(ctx, o, in, x); err == nil {
854	t.Fatal("accepted a replay whose env object is gone")
855	}
856	x.bucket.Put("data", envObject, []byte("K=corrupt\nCARLOS_INSTANCE_KEY=abc"))
857	if err := x.f.Establish(ctx, o, in, x); err == nil {
858	t.Fatal("accepted a replay whose env object is corrupt")
859	}
860	x.bucket.Put("data", envObject, envBytes)
861	// The genuine replay is accepted and writes nothing.
862	x.bucket.Put("data", pm.Key, genuineRaw)
863	seq1 := x.bucket.LatestID("data", pm.Key)
864	if err := x.f.Establish(ctx, o, in, x); err != nil || x.bucket.LatestID("data", pm.Key) != seq1 {
865	t.Fatalf("the genuine replay: %v, revision moved %v", err, x.bucket.LatestID("data", pm.Key) != seq1)
866	}
867	}
868
869	// stallingStore holds one key's write until released, signalling that it
870	// is stalled: a slow S3 request.
871	type stallingStore struct {
872	release.Store
873	key string
874	stalled chan struct{}
875	release chan struct{}
876	once sync.Once
877	}
878
879	func (s *stallingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
880	if key == s.key {
881	s.once.Do(func() { close(s.stalled) })
882	select {
883	case <-s.release:
884	case <-ctx.Done():
885	return "", ctx.Err()
886	}
887	}
888	return s.Store.Put(ctx, key, body, ifMatch)
889	}
890
891	// Remote publication runs outside the start's work (fix round 1, finding
892	// 7; platform-1e P2): each bucket write is fenced by a short With (the
893	// selection still holds) and bounded by the request limit, and the lock
894	// is released before the request is made; With holds only the local
895	// effect. While the manifest's write stalls, another holder of the lock
896	// (BeginFence's stand-in) acquires it and proceeds; the establishment
897	// completes once the write returns.
898	func TestEstablishPublishesOutsideTheStartsWork(t *testing.T) {
899	x := newFilesFixture(t)
900	ctx := context.Background()
901	x.writeEnv("K=1\n")
902	x.publish("g1", func(p *hostfiles.Publisher) {
903	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
904	t.Fatal(err)
905	}
906	})
907	st := &stallingStore{Store: x.files, key: hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g2")), stalled: make(chan struct{}), release: make(chan struct{})}
908	x.f.Files = st
909	o := x.observe(servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk"), "g1", "g2")
910	done := make(chan error, 1)
911	go func() { done <- x.f.Establish(ctx, o, launchInputs("g2"), x) }()
912	select {
913	case <-st.stalled:
914	case err := <-done:
915	t.Fatalf("established before the manifest's write: %v", err)
916	case <-time.After(10 * time.Second):
917	t.Fatal("the manifest's write never began")
918	}
919	// The lock is free while the write stalls.
920	acquired := make(chan error, 1)
921	go func() {
922	acquired <- x.With(ctx, hostwrite.Identity{Config: "c", Release: "r"}, func() error { return nil })
923	}()
924	select {
925	case err := <-acquired:
926	if err != nil {
927	t.Fatal(err)
928	}
929	case <-time.After(5 * time.Second):
930	t.Fatal("the start's work is held across the stalled write: BeginFence would wait")
931	}
932	close(st.release)
933	select {
934	case err := <-done:
935	if err != nil {
936	t.Fatal(err)
937	}
938	case <-time.After(10 * time.Second):
939	t.Fatal("the establishment never completed")
940	}
941	if m, found := x.manifest("g2"); !found || m.Seq != 2 { // g1's one entry, then the env object
942	t.Fatalf("g2: %+v %v", m, found)
943	}
944	// Two fences (the env object, the manifest), the probe above and the
945	// local effect.
946	if x.workRan != 4 {
947	t.Fatalf("work ran %d times", x.workRan)
948	}
949	}
950
951	// A never-admitted origin under the location is refused (fix round 2,
952	// finding 3; platform-1e's ruling): the location rule is syntax, not
953	// provenance. A manifest in an admitted generation naming an object
954	// under files/g-never/, which no admission and no carried set names, is
955	// refused with ErrOrigin, whether the manifest carries no set (the old
956	// format, judged by the chain alone) or a set that omits it; and a
957	// carried set is never widened by anything but the admitted chain.
958	func TestEstablishRefusesANeverAdmittedOrigin(t *testing.T) {
959	x := newFilesFixture(t)
960	ctx := context.Background()
961	x.writeEnv("K=1\n")
962	g1 := x.publish("g1", func(p *hostfiles.Publisher) {
963	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
964	t.Fatal(err)
965	}
966	})
967	o := x.observe(servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk"), "g1", "g2")
968	never := movesFilesPrefix(filesLoc, "never") + "objects/0000000000000001-" + sha([]byte("N"))
969	x.bucket.Put("data", never, []byte("N"))
970	for name, origins := range map[string][]string{"no carried set": nil, "a carried set without it": {movesFilesPrefix(filesLoc, "g1")}} {
971	bad := g1
972	bad.Entries = map[string]hostfiles.Entry{"a.txt": g1.Entries["a.txt"], "n.txt": {Object: never, SHA256: sha([]byte("N")), Size: 1, Seq: 1}}
973	bad.Origins = origins
974	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g1")), mustJSON(t, bad))
975	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); !errors.Is(err, hostfiles.ErrOrigin) {
976	t.Fatalf("%s: %v", name, err)
977	}
978	if _, found := x.manifest("g2"); found {
979	t.Fatalf("%s: a manifest was written over a never-admitted origin", name)
980	}
981	}
982	// Origins vouch for nothing by themselves (platform-1e's ruling): a
983	// manifest that adds the never-granted origin to its own carried set
984	// and names an object under it is refused. A carried hint that no entry
985	// references is neither proven nor carried (Task 6b): the
986	// establishment goes through and the hint does not survive it.
987	bad := g1
988	bad.Entries = map[string]hostfiles.Entry{"a.txt": g1.Entries["a.txt"], "n.txt": {Object: never, SHA256: sha([]byte("N")), Size: 1, Seq: 1}}
989	bad.Origins = []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "never")}
990	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g1")), mustJSON(t, bad))
991	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); !errors.Is(err, hostfiles.ErrOrigin) {
992	t.Fatalf("named and carried: %v", err)
993	}
994	if _, found := x.manifest("g2"); found {
995	t.Fatal("a manifest was written over a self-vouched origin")
996	}
997	// The genuine manifest establishes, and the new set is exactly the
998	// referenced origin and the new generation.
999	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g1")), mustJSON(t, g1))
1000	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); err != nil {
1001	t.Fatal(err)
1002	}
1003	m, _ := x.manifest("g2")
1004	if want := []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "g2")}; !reflect.DeepEqual(m.Origins, want) {
1005	t.Fatalf("g2 origins %v, want %v", m.Origins, want)
1006	}
1007	// A carried hint that no entry references (planted at g2, the newest
1008	// the chain from g3 reads): the establishment goes through and the
1009	// hint does not survive it.
1010	hint := m
1011	hint.Origins = []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "g2"), movesFilesPrefix(filesLoc, "never")}
1012	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g2")), mustJSON(t, hint))
1013	o3 := x.observe(servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk"), "g3")
1014	if err := x.f.Establish(ctx, o3, launchInputs("g3"), x); err != nil {
1015	t.Fatalf("carried only: %v", err)
1016	}
1017	m3, _ := x.manifest("g3")
1018	if want := []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "g2"), movesFilesPrefix(filesLoc, "g3")}; !reflect.DeepEqual(m3.Origins, want) {
1019	t.Fatalf("g3 origins %v, want %v (the env object under g2 is referenced; the hint is not)", m3.Origins, want)
1020	}
1021	}
1022
1023	// A legacy adopted location, with no incarnation component, gets the same
1024	// treatment and is seeded at its first establishment: its old manifest
1025	// carries no set and is judged by the admitted chain alone; the new
1026	// manifest carries the chain and the new generation; a never-admitted
1027	// origin under it is refused as anywhere else.
1028	func TestEstablishSeedsALegacyLocationsOriginSet(t *testing.T) {
1029	x := newFilesFixture(t)
1030	ctx := context.Background()
1031	x.writeEnv("K=1\n")
1032	legacy := "instances/" + filesHost
1033	loc := func(gen string) string { return movesFilesPrefix(legacy, gen) }
1034	p, err := hostfiles.Open(ctx, x.files, loc("g1"))
1035	if err != nil {
1036	t.Fatal(err)
1037	}
1038	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1039	t.Fatal(err)
1040	}
1041	if _, err := p.Commit(ctx); err != nil {
1042	t.Fatal(err)
1043	}
1044	old, _, _ := hostfiles.Load(ctx, x.files, loc("g1"))
1045	if old.Origins != nil {
1046	t.Fatalf("the legacy manifest carries a set: %v", old.Origins)
1047	}
1048	o := servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk")
1049	o.Claim.Holder.Location = legacy
1050	o = x.observe(o, "g1", "g2")
1051	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); err != nil {
1052	t.Fatal(err)
1053	}
1054	m, found, err := hostfiles.Load(ctx, x.files, loc("g2"))
1055	if err != nil || !found {
1056	t.Fatal(err, found)
1057	}
1058	if want := []string{loc("g1"), loc("g2")}; !reflect.DeepEqual(m.Origins, want) || m.Entries["a.txt"].Object != old.Entries["a.txt"].Object {
1059	t.Fatalf("seeded: %+v", m)
1060	}
1061	bad := old
1062	bad.Entries = map[string]hostfiles.Entry{"n.txt": {Object: loc("never") + "objects/0000000000000001-" + sha([]byte("N")), SHA256: sha([]byte("N")), Size: 1, Seq: 1}}
1063	// Planted at g2, the newest manifest the chain from g3 reads.
1064	x.bucket.Put("data", hostfiles.ManifestKey(loc("g2")), mustJSON(t, bad))
1065	o3 := servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk")
1066	o3.Claim.Holder.Location = legacy
1067	o3 = x.observe(o3, "g3")
1068	if err := x.f.Establish(ctx, o3, launchInputs("g3"), x); !errors.Is(err, hostfiles.ErrOrigin) {
1069	t.Fatalf("never admitted under a legacy location: %v", err)
1070	}
1071	}
1072
1073	// answeringProof is a claim-backed origin proof with an answer per
1074	// generation: proven, not proven, or an error; a generation it does not
1075	// name is not proven.
1076	type answeringProof struct {
1077	proven map[string]bool
1078	errs map[string]error
1079	}
1080
1081	func (a answeringProof) Proven(_ context.Context, _ hostclaim.Observed, _ hostclaim.Admission, generations []string) (map[string]bool, error) {
1082	out := map[string]bool{}
1083	for _, generation := range generations {
1084	if err := a.errs[generation]; err != nil {
1085	return nil, err
1086	}
1087	out[generation] = a.proven[generation]
1088	}
1089	return out, nil
1090	}
1091
1092	// movesOriginProof is the only route to origin proof (fix round 4): the
1093	// chain's prefixes and the prepared generation go through Proven like
1094	// every other origin, so a different proof (claim Plan 15's) has the final
1095	// say. A proof that answers false, or an error, for a chain-prefix origin
1096	// or for the prepared generation's origin refuses the establishment and
1097	// nothing is written.
1098	func TestEstablishProvesEveryOriginThroughTheProof(t *testing.T) {
1099	ctx := context.Background()
1100	for name, mk := range map[string]func(gen string) answeringProof{
1101	"not proven": func(gen string) answeringProof { return answeringProof{proven: map[string]bool{}} },
1102	"an error": func(gen string) answeringProof {
1103	return answeringProof{errs: map[string]error{gen: errors.New("the record is unreadable")}}
1104	},
1105	} {
1106	t.Run("a chain prefix, "+name, func(t *testing.T) {
1107	x := newFilesFixture(t)
1108	x.writeEnv("K=1\n")
1109	x.publish("g1", func(p *hostfiles.Publisher) {
1110	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1111	t.Fatal(err)
1112	}
1113	})
1114	o := servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk")
1115	proof := mk("g1")
1116	proof.proven = map[string]bool{"g2": true}
1117	x.f.Proof = proof
1118	err := x.f.Establish(ctx, o, launchInputs("g2"), x)
1119	if err == nil || (name == "not proven" && !errors.Is(err, hostfiles.ErrOrigin)) {
1120	t.Fatalf("a chain prefix the proof does not prove: %v", err)
1121	}
1122	if _, found := x.manifest("g2"); found {
1123	t.Fatal("a manifest was written over an unproven chain prefix")
1124	}
1125	// The new generation itself is proven through the proof too.
1126	x.f.Proof = answeringProof{proven: map[string]bool{"g1": true}}
1127	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); !errors.Is(err, hostfiles.ErrOrigin) {
1128	t.Fatalf("the new generation the proof does not prove: %v", err)
1129	}
1130	x.f.Proof = answeringProof{proven: map[string]bool{"g1": true, "g2": true}}
1131	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); err != nil {
1132	t.Fatalf("everything proven: %v", err)
1133	}
1134	})
1135	t.Run("the prepared generation, "+name, func(t *testing.T) {
1136	x, pm, o, in := preparedFixture(t)
1137	proof := mk("mv1")
1138	proof.proven = map[string]bool{"cur": true}
1139	x.f.Proof = proof
1140	err := x.f.Establish(ctx, o, in, x)
1141	if err == nil || (name == "not proven" && !errors.Is(err, hostfiles.ErrOrigin)) {
1142	t.Fatalf("the prepared generation the proof does not prove: %v", err)
1143	}
1144	if x.bucket.LatestID("data", pm.Key) != pm.Version {
1145	t.Fatal("the prepared manifest was published over")
1146	}
1147	x.f.Proof = answeringProof{proven: map[string]bool{"cur": true, "mv1": true}}
1148	if err := x.f.Establish(ctx, o, in, x); err != nil {
1149	t.Fatalf("everything proven: %v", err)
1150	}
1151	})
1152	}
1153	}
1154
1155	// The proof never accepts a bare Admission.NewGeneration supplied on its
1156	// own (platform-1e's condition, fix round 4; claim Plan 15): a generation
1157	// is proven only when the bound observation's claim records its
1158	// admission, as the post-admission observation Materialize is given does.
1159	// A free-standing generation, named by the admission but recorded by no
1160	// observation, is refused at the proof and by Establish, and nothing is
1161	// written; an observation of no claim is an error.
1162	func TestClaimProofRefusesAFreeStandingGeneration(t *testing.T) {
1163	ctx := context.Background()
1164	x := newFilesFixture(t)
1165	x.writeEnv("K=1\n")
1166	x.publish("g1", func(p *hostfiles.Publisher) {
1167	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1168	t.Fatal(err)
1169	}
1170	})
1171	proof := ClaimOriginProof{Claims: x.claims, HomeRegion: "ie", Region: "uk"}
1172	before := x.observe(servingObserved("g1", "g1:uk:@root:uk"), "g1") // the observation before g2's admission
1173	in := hostclaim.Admission{Box: "uk-box", Mode: "cold", NewGeneration: "g2", Region: "uk"}
1174	if got, err := proof.Proven(ctx, before, in, []string{"g1", "g2"}); err != nil || !got["g1"] || got["g2"] {
1175	t.Fatalf("a free-standing generation: %v %v", got, err)
1176	}
1177	if _, err := proof.Proven(ctx, hostclaim.Observed{Host: filesHost, Absent: true}, in, []string{"g2"}); err == nil {
1178	t.Fatal("proven with no claim")
1179	}
1180	if err := x.f.Establish(ctx, before, launchInputs("g2"), x); !errors.Is(err, hostfiles.ErrOrigin) {
1181	t.Fatalf("establish with an observation that does not record the admission: %v", err)
1182	}
1183	if _, found := x.manifest("g2"); found {
1184	t.Fatal("a manifest was written for a free-standing generation")
1185	}
1186	after := x.observe(servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk"), "g2") // the CAS result the writer's admission returns
1187	if err := x.f.Establish(ctx, after, launchInputs("g2"), x); err != nil {
1188	t.Fatalf("establish with the bound observation: %v", err)
1189	}
1190	}
1191
1192	// countingProof records every Proven call and the set it was asked about,
1193	// answering from proven.
1194	type countingProof struct {
1195	proven map[string]bool
1196	calls [][]string
1197	}
1198
1199	func (c *countingProof) Proven(_ context.Context, _ hostclaim.Observed, _ hostclaim.Admission, generations []string) (map[string]bool, error) {
1200	c.calls = append(c.calls, append([]string(nil), generations...))
1201	out := map[string]bool{}
1202	for _, generation := range generations {
1203	out[generation] = c.proven[generation]
1204	}
1205	return out, nil
1206	}
1207
1208	// Only referenced origins are proven and carried (Task 6b, the moves side
1209	// of claim Plan 15's review): the origins the live manifest's entries
1210	// name, plus the one generation the admission opens (round 4's binding,
1211	// one call). A carried hint nothing references is neither proven nor
1212	// carried forward, so old hints do not accumulate and the proof's cost
1213	// does not grow with the history; a referenced origin is still proven and
1214	// refused when the proof says no; and the count of Proven calls is the
1215	// distinct referenced origins plus the admitted generation, each asked
1216	// once.
1217	func TestEstablishProvesAndCarriesOnlyReferencedOrigins(t *testing.T) {
1218	x := newFilesFixture(t)
1219	ctx := context.Background()
1220	x.writeEnv("K=1\n")
1221	// g5's object, referenced by the live manifest at g1 through a carried
1222	// entry; g7 is a hint nothing references and nothing can prove.
1223	g5obj := movesFilesPrefix(filesLoc, "g5") + "objects/0000000000000001-" + sha([]byte("F"))
1224	x.bucket.Put("data", g5obj, []byte("F"))
1225	live := x.publish("g1", func(p *hostfiles.Publisher) {
1226	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1227	t.Fatal(err)
1228	}
1229	})
1230	live.Entries["f.txt"] = hostfiles.Entry{Object: g5obj, SHA256: sha([]byte("F")), Size: 1, Seq: 1}
1231	live.Origins = []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "g5"), movesFilesPrefix(filesLoc, "g7")}
1232	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g1")), mustJSON(t, live))
1233	o := servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk")
1234	// The hint g7 is not proven: the establishment still goes through, and
1235	// g7 is not carried; g1, g5 and the admitted g2 are, each asked once.
1236	proof := &countingProof{proven: map[string]bool{"g1": true, "g2": true, "g5": true}}
1237	x.f.Proof = proof
1238	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); err != nil {
1239	t.Fatalf("an unreferenced hint blocked the establishment: %v", err)
1240	}
1241	m, found := x.manifest("g2")
1242	want := []string{movesFilesPrefix(filesLoc, "g1"), movesFilesPrefix(filesLoc, "g2"), movesFilesPrefix(filesLoc, "g5")}
1243	if !found || !reflect.DeepEqual(m.Origins, want) {
1244	t.Fatalf("g2 origins %v, want %v", m.Origins, want)
1245	}
1246	// One Proven(set) call per Establish (Plan 15): the deduplicated
1247	// referenced origins plus the admitted generation, in one batch (the
1248	// set's order is the caller's; the prefixes are what get sorted).
1249	if len(proof.calls) != 1 {
1250	t.Fatalf("Proven calls %v, want one call over the referenced origins and the admitted generation", proof.calls)
1251	}
1252	asked := append([]string(nil), proof.calls[0]...)
1253	sort.Strings(asked)
1254	if !reflect.DeepEqual(asked, []string{"g1", "g2", "g5"}) {
1255	t.Fatalf("Proven asked %v, want the referenced origins and the admitted generation", proof.calls[0])
1256	}
1257	// A referenced origin the proof refuses still refuses the whole.
1258	x.f.Proof = &countingProof{proven: map[string]bool{"g1": true, "g2": true, "g3": true}}
1259	if err := x.f.Establish(ctx, servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk"), launchInputs("g3"), x); !errors.Is(err, hostfiles.ErrOrigin) {
1260	t.Fatalf("a referenced origin the proof refuses: %v", err)
1261	}
1262	if _, found := x.manifest("g3"); found {
1263	t.Fatal("a manifest was written over a refused referenced origin")
1264	}
1265	}
1266
1267	// recoveryObserved is the source box's observation after a recovery
1268	// admission: the holder at the fresh recovery generation R, admitted after
1269	// the target's admission T in the other region, under a restoring op (or
1270	// a reverting one) whose journal names the source's generation src_gen,
1271	// as the flip wrote it.
1272	func recoveryObserved(kind, srcGen, target, recovered string) hostclaim.Observed {
1273	o := servingObserved(recovered, "src:uk:@root:uk", target+":ie:src:uk", recovered+":uk:"+target+":ie")
1274	if srcGen == "" {
1275	o.Claim.Holder.Admissions = o.Claim.Holder.Admissions[1:] // a legacy source: no admission of its own
1276	}
1277	o.Claim.Op = hostclaim.Op{Kind: kind, ID: target, Phase: hostclaim.PhaseProbing, Exec: "uk-box", Participants: []string{"uk-box", "ie-box"},
1278	Intent: map[string]string{"to": "ie", "source": "uk-box", "target": "ie-box", "rollback": "rb1"},
1279	Journal: map[string]string{hostclaim.JournalSrcGen: srcGen, hostclaim.JournalSourceStamp: "15", hostclaim.JournalCutSession: "3"}}
1280	return o
1281	}
1282
1283	// A recovery admission's files baseline is the recorded source restore
1284	// (whole-branch fixes, item 3, as re-ruled with platform-1e): for a
1285	// restoring op whose own lineage holds no manifest, the chain walk starts
1286	// at the journalled src_gen, pinned as the database's source restore is,
1287	// because the admission's predecessor is the target's generation in the
1288	// other region and ends the ordinary walk at once. The origin proof still
1289	// goes only through movesOriginProof with the bound observation: src_gen
1290	// is proven by the claim's journal, and a proof that refuses it refuses
1291	// the whole. Clean, declared-loss and fenced returns establish from
1292	// src_gen's manifest; a revert, whose holder never left the source,
1293	// reaches it through its own lineage; a legacy source (src_gen "") has no
1294	// files generation and refuses.
1295	func TestRecoveryEstablishesFromTheRecordedSourceGeneration(t *testing.T) {
1296	ctx := context.Background()
1297	for name, shape := range map[string]func(o *hostclaim.Observed){
1298	"a clean return": func(*hostclaim.Observed) {},
1299	"a declared-loss return": func(o *hostclaim.Observed) {
1300	o.Claim.Op.Journal["loss.l1"] = `{"kind":"move-target-lost","target":"ie-box"}`
1301	o.Claim.LossRefs, o.Claim.Lost = []string{"l1"}, []string{"ie-box"}
1302	},
1303	"a fenced return": func(o *hostclaim.Observed) {
1304	o.Claim.Op.Journal["fence.ie-box"] = `{"enrollment":"enr","key":"control/box-fences/enr.json","sha256":"ab"}`
1305	},
1306	"a revert": func(o *hostclaim.Observed) {
1307	// A revert's holder never left the source: its admission's
1308	// predecessor is src in this region, so it follows its own
1309	// lineage with no bridge (platform-1e's framing).
1310	o.Claim.Op.Kind = hostclaim.KindReverting
1311	delete(o.Claim.Op.Intent, "rollback")
1312	o.Claim.Holder.Admissions = []hostclaim.AdmissionRecord{{Generation: "src", Region: "uk", Predecessor: "@root", PredecessorRegion: "uk", Mode: "cold", Session: 3, Box: "uk-box"},
1313	{Generation: "r1", Region: "uk", Predecessor: "src", PredecessorRegion: "uk", Mode: "cold", Session: 3, Box: "uk-box"}}
1314	},
1315	} {
1316	t.Run(name, func(t *testing.T) {
1317	x := newFilesFixture(t)
1318	x.writeEnv("K=1\n")
1319	src := x.publish("src", func(p *hostfiles.Publisher) {
1320	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1321	t.Fatal(err)
1322	}
1323	})
1324	o := recoveryObserved(hostclaim.KindRestoring, "src", "mv1", "r1")
1325	shape(&o)
1326	o = x.observe(o, "src", "r1")
1327	if err := x.f.Establish(ctx, o, launchInputs("r1"), x); err != nil {
1328	t.Fatalf("establish from src_gen: %v", err)
1329	}
1330	m, found := x.manifest("r1")
1331	if !found || m.Entries["a.txt"].Object != src.Entries["a.txt"].Object || !reflect.DeepEqual(m.Origins, []string{movesFilesPrefix(filesLoc, "r1"), movesFilesPrefix(filesLoc, "src")}) {
1332	t.Fatalf("r1: %+v %v", m, found)
1333	}
1334	// The proof has the final say over src_gen.
1335	y := newFilesFixture(t)
1336	y.writeEnv("K=1\n")
1337	y.publish("src", func(p *hostfiles.Publisher) {
1338	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1339	t.Fatal(err)
1340	}
1341	})
1342	y.f.Proof = answeringProof{proven: map[string]bool{"r1": true}}
1343	if err := y.f.Establish(ctx, o, launchInputs("r1"), y); !errors.Is(err, hostfiles.ErrOrigin) {
1344	t.Fatalf("src_gen the proof refuses: %v", err)
1345	}
1346	})
1347	}
1348	// A legacy source: src_gen "" names no files generation; nothing to
1349	// establish from, and never a bootstrap.
1350	x := newFilesFixture(t)
1351	x.writeEnv("K=1\n")
1352	if err := x.f.Establish(ctx, recoveryObserved(hostclaim.KindRestoring, "", "mv1", "r1"), launchInputs("r1"), x); !errors.Is(err, hostfiles.ErrNoEvidence) {
1353	t.Fatalf("a legacy source: %v", err)
1354	}
1355	}
1356
1357	// recoveryRestartObserved is the source box's observation after a later
1358	// recovery admission: the holder at the fresh generation `next`, admitted
1359	// after the recovery generation `prev` in this region (the prepared
1360	// rollback generation, or an earlier recovery admission), itself admitted
1361	// after the target's T in the other region.
1362	func recoveryRestartObserved(kind, prev, next string) hostclaim.Observed {
1363	o := servingObserved(next, "src:uk:@root:uk", "mv1:ie:src:uk", prev+":uk:mv1:ie", next+":uk:"+prev+":uk")
1364	o.Claim.Op = hostclaim.Op{Kind: kind, ID: "mv1", Phase: hostclaim.PhaseProbing, Exec: "uk-box", Participants: []string{"uk-box", "ie-box"},
1365	Intent: map[string]string{"to": "ie", "source": "uk-box", "target": "ie-box", "rollback": "rb1"},
1366	Journal: map[string]string{hostclaim.JournalSrcGen: "src", hostclaim.JournalSourceStamp: "15", hostclaim.JournalCutSession: "3"}}
1367	if kind == hostclaim.KindReverting {
1368	delete(o.Claim.Op.Intent, "rollback")
1369	}
1370	return o
1371	}
1372
1373	// src_gen only bridges the missing source baseline (the re-review's
1374	// ruling on item 3, mirroring the database selector): once a recovery
1375	// manifest exists, the prepared rollback generation's or a later recovery
1376	// publication's, the newest authorized recovery manifest wins through the
1377	// normal chain and origin proof. A restart after the dirty return's
1378	// prepared rollback manifest publishes the rollback's contents (B), not
1379	// the source's (A), even with the source manifest absent; a clean return
1380	// and a revert after a newer recovery publication take the newer one; a
1381	// first admission with no recovery manifest still bridges from src_gen.
1382	func TestRecoveryRestartTakesTheNewestRecoveryManifest(t *testing.T) {
1383	ctx := context.Background()
1384	publish := func(x *filesFixture, gen, body string) hostfiles.Manifest {
1385	return x.publish(gen, func(p *hostfiles.Publisher) {
1386	if err := p.Stage(ctx, "a.txt", []byte(body), 0o600, false); err != nil {
1387	t.Fatal(err)
1388	}
1389	})
1390	}
1391	t.Run("a restart after the prepared rollback manifest publishes B", func(t *testing.T) {
1392	for name, withSource := range map[string]bool{"source present": true, "source absent": false} {
1393	t.Run(name, func(t *testing.T) {
1394	x := newFilesFixture(t)
1395	x.writeEnv("K=1\n")
1396	if withSource {
1397	publish(x, "src", "A")
1398	}
1399	rb := publish(x, "rb1", "B")
1400	o := x.observe(recoveryRestartObserved(hostclaim.KindRestoring, "rb1", "r2"), "rb1", "r2")
1401	if err := x.f.Establish(ctx, o, launchInputs("r2"), x); err != nil {
1402	t.Fatalf("the restart: %v", err)
1403	}
1404	m, found := x.manifest("r2")
1405	if !found || m.Entries["a.txt"].Object != rb.Entries["a.txt"].Object {
1406	t.Fatalf("r2 took %+v, want the rollback's B (%s)", m.Entries["a.txt"], rb.Entries["a.txt"].Object)
1407	}
1408	})
1409	}
1410	})
1411	for name, kind := range map[string]string{"a clean return": hostclaim.KindRestoring, "a revert": hostclaim.KindReverting} {
1412	t.Run(name+" after a newer recovery publication", func(t *testing.T) {
1413	x := newFilesFixture(t)
1414	x.writeEnv("K=1\n")
1415	publish(x, "src", "A")
1416	newer := publish(x, "r1", "C") // the earlier recovery admission's publication
1417	o := x.observe(recoveryRestartObserved(kind, "r1", "r2"), "r1", "r2")
1418	if err := x.f.Establish(ctx, o, launchInputs("r2"), x); err != nil {
1419	t.Fatalf("the restart: %v", err)
1420	}
1421	m, found := x.manifest("r2")
1422	if !found || m.Entries["a.txt"].Object != newer.Entries["a.txt"].Object {
1423	t.Fatalf("r2 took %+v, want the newer recovery's C", m.Entries["a.txt"])
1424	}
1425	})
1426	}
1427	t.Run("a dirty return's later admissions follow the rollback lineage", func(t *testing.T) {
1428	// The second cold admission restores from the rollback generation,
1429	// the third from its certified descendant (the second's publication).
1430	x := newFilesFixture(t)
1431	x.writeEnv("K=1\n")
1432	publish(x, "src", "A")
1433	rb := publish(x, "rb1", "B")
1434	o := recoveryRestartObserved(hostclaim.KindRestoring, "rb1", "r2")
1435	o.Claim.Op.Journal[moves.JournalReturnSeal] = "sealed"
1436	o = x.observe(o, "rb1", "r2")
1437	if err := x.f.Establish(ctx, o, launchInputs("r2"), x); err != nil {
1438	t.Fatalf("the second admission: %v", err)
1439	}
1440	if m, _ := x.manifest("r2"); m.Entries["a.txt"].Object != rb.Entries["a.txt"].Object {
1441	t.Fatalf("r2 took %+v, want the rollback's B", m.Entries["a.txt"])
1442	}
1443	r2 := publish(x, "r2", "D") // the second admission's own publication
1444	o3 := servingObserved("r3", "src:uk:@root:uk", "mv1:ie:src:uk", "rb1:uk:mv1:ie", "r2:uk:rb1:uk", "r3:uk:r2:uk")
1445	o3.Claim.Op = o.Claim.Op
1446	o3 = x.observe(o3, "r3")
1447	if err := x.f.Establish(ctx, o3, launchInputs("r3"), x); err != nil {
1448	t.Fatalf("the third admission: %v", err)
1449	}
1450	if m, _ := x.manifest("r3"); m.Entries["a.txt"].Object != r2.Entries["a.txt"].Object {
1451	t.Fatalf("r3 took %+v, want the descendant's D", m.Entries["a.txt"])
1452	}
1453	})
1454	t.Run("a dirty return never bridges to the source", func(t *testing.T) {
1455	// A dirty return restores the rollback generation, never the stopped
1456	// source: with no manifest in its lineage it refuses rather than
1457	// publishing the source's older files.
1458	x := newFilesFixture(t)
1459	x.writeEnv("K=1\n")
1460	publish(x, "src", "A")
1461	o := recoveryRestartObserved(hostclaim.KindRestoring, "rb1", "r2")
1462	o.Claim.Op.Journal[moves.JournalReturnSeal] = "sealed"
1463	o = x.observe(o, "src", "r2")
1464	if err := x.f.Establish(ctx, o, launchInputs("r2"), x); !errors.Is(err, hostfiles.ErrNoEvidence) {
1465	t.Fatalf("a dirty return without its rollback manifest: %v", err)
1466	}
1467	if _, found := x.manifest("r2"); found {
1468	t.Fatal("a dirty return published the source's files")
1469	}
1470	})
1471	t.Run("a first admission with no recovery manifest bridges from src_gen", func(t *testing.T) {
1472	x := newFilesFixture(t)
1473	x.writeEnv("K=1\n")
1474	src := publish(x, "src", "A")
1475	o := x.observe(recoveryObserved(hostclaim.KindRestoring, "src", "mv1", "r1"), "src", "r1")
1476	if err := x.f.Establish(ctx, o, launchInputs("r1"), x); err != nil {
1477	t.Fatalf("the first admission: %v", err)
1478	}
1479	if m, found := x.manifest("r1"); !found || m.Entries["a.txt"].Object != src.Entries["a.txt"].Object {
1480	t.Fatalf("r1 took %+v, want the source's A", m.Entries["a.txt"])
1481	}
1482	})
1483	}
1484
1485	// The proven set and the carried set order the same way (Astra's finding
1486	// on the Plan 15 switch): generation names and their files prefixes do
1487	// not sort alike ("mv1" before "mv1-old", but "g-mv1-old/" before
1488	// "g-mv1/"), so the prefixes, not the names, are sorted everywhere the
1489	// sets are compared. A prepared manifest for mv1 that also references an
1490	// object under mv1-old (a same-region retained reference) is established
1491	// once; a crash before the local effect leaves the key at sequence 1
1492	// with the carried set; the genuine replay is accepted and writes nothing.
1493	func TestPreparedReplayOrdersOriginsLikeTheCarriedSet(t *testing.T) {
1494	x := newFilesFixture(t)
1495	ctx := context.Background()
1496	x.writeEnv("K=1\n")
1497	oldObj := movesFilesPrefix(filesLoc, "mv1-old") + "objects/0000000000000001-" + sha([]byte("O"))
1498	x.bucket.Put("data", oldObj, []byte("O"))
1499	newObj := movesFilesPrefix(filesLoc, "mv1") + "objects/0000000000000000-" + sha([]byte("A"))
1500	x.bucket.Put("data", newObj, []byte("A"))
1501	envObj := movesFilesPrefix(filesLoc, "mv1") + "objects/0000000000000000-" + sha([]byte("K=1\n"))
1502	x.bucket.Put("data", envObj, []byte("K=1\n"))
1503	prepared := hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: 0,
1504	Entries: map[string]hostfiles.Entry{"a.txt": {Object: newObj, SHA256: sha([]byte("A")), Size: 1, Mode: 0o600, Seq: 0}, "old.txt": {Object: oldObj, SHA256: sha([]byte("O")), Size: 1, Mode: 0o600, Seq: 1}},
1505	Platform: map[string]hostfiles.Entry{hostfiles.EnvItem: {Object: envObj, SHA256: sha([]byte("K=1\n")), Size: 4, Mode: 0o600, Seq: 0}},
1506	Origins: []string{movesFilesPrefix(filesLoc, "mv1-old"), movesFilesPrefix(filesLoc, "mv1")}}
1507	raw := mustJSON(t, prepared)
1508	key := hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "mv1"))
1509	version := x.bucket.Put("data", key, raw)
1510	x.journal("mv1", preparedManifest{Key: key, SHA256: sha(raw), Version: version, Op: "mv1", Stamp: 7})
1511	x.writeEnv("K=1\nCARLOS_INSTANCE_KEY=abc\n")
1512	o := servingObserved("mv1", "cur:ie:@root:ie", "mv1:uk:cur:ie")
1513	o.Claim.Op = hostclaim.Op{Kind: hostclaim.KindLanding, ID: "mv1", Phase: "probing", Exec: "uk-box", Participants: []string{"ie-box", "uk-box"}, Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}}
1514	o = x.observe(o, "mv1", "mv1-old")
1515	in := launchInputs("mv1")
1516	in.Prepared = true
1517	x.workErr, x.workErrAt = errors.New("crash before the local effect"), 3
1518	if err := x.f.Establish(ctx, o, in, x); err == nil {
1519	t.Fatal("the crash was not reported")
1520	}
1521	x.workErr, x.workErrAt = nil, 0
1522	m, found := x.manifest("mv1")
1523	if !found || m.Seq != 1 {
1524	t.Fatalf("after the crashed attempt: %+v %v", m, found)
1525	}
1526	seq1 := x.bucket.LatestID("data", key)
1527	if err := x.f.Establish(ctx, o, in, x); err != nil {
1528	t.Fatalf("the genuine replay: %v", err)
1529	}
1530	if x.bucket.LatestID("data", key) != seq1 {
1531	t.Fatal("the replay wrote a new revision")
1532	}
1533	}
1534
1535	// The instance key rule (O1'): the manifest Establish leaves carries the
1536	// env file with exactly the bytes on disk when it runs, staged and
1537	// committed before it returns, and no adoption item; the local adoption
1538	// record is removed under the start's work. Establish never writes the
1539	// env file. A work that refuses leaves everything as it was.
1540	func TestEstablishCarriesTheEnvBytesAndDropsTheAdoptionItem(t *testing.T) {
1541	x := newFilesFixture(t)
1542	ctx := context.Background()
1543	x.publish("g1", func(p *hostfiles.Publisher) {
1544	if err := p.StagePlatform(ctx, hostfiles.EnvItem, []byte("K=old\n")); err != nil {
1545	t.Fatal(err)
1546	}
1547	if err := p.StagePlatform(ctx, instancekey.AdoptionItem, []byte(`{"v":2}`)); err != nil {
1548	t.Fatal(err)
1549	}
1550	})
1551	x.writeEnv("K=new\nCARLOS_INSTANCE_KEY=abc\n")
1552	adoption := filepath.Join(x.dir, filesHost+instancekey.AdoptionSuffix)
1553	if err := os.WriteFile(adoption, []byte(`{"v":2}`), 0o600); err != nil {
1554	t.Fatal(err)
1555	}
1556	o := x.observe(servingObserved("g2", "g1:uk:@root:uk", "g2:uk:g1:uk"), "g1", "g2")
1557	x.workErr = hostbox.ErrChanged
1558	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); !errors.Is(err, hostbox.ErrChanged) {
1559	t.Fatalf("refused work: %v", err)
1560	}
1561	if _, found := x.manifest("g2"); found {
1562	t.Fatal("a refused work wrote the manifest")
1563	}
1564	if _, err := os.Stat(adoption); err != nil {
1565	t.Fatal("a refused work removed the adoption record")
1566	}
1567	x.workErr = nil
1568	if err := x.f.Establish(ctx, o, launchInputs("g2"), x); err != nil {
1569	t.Fatal(err)
1570	}
1571	m, found := x.manifest("g2")
1572	if !found {
1573	t.Fatal("no manifest under g2")
1574	}
1575	e := m.Platform[hostfiles.EnvItem]
1576	if e.SHA256 != sha([]byte("K=new\nCARLOS_INSTANCE_KEY=abc\n")) || !strings.HasPrefix(e.Object, movesFilesPrefix(filesLoc, "g2")+"objects/") {
1577	t.Fatalf("env item: %+v", e)
1578	}
1579	got, err := hostfiles.ReadPlatform(ctx, x.files, movesFilesPrefix(filesLoc, "g2"), m, hostfiles.EnvItem, nil)
1580	if err != nil || string(got) != "K=new\nCARLOS_INSTANCE_KEY=abc\n" {
1581	t.Fatalf("env bytes: %q %v", got, err)
1582	}
1583	if _, has := m.Platform[instancekey.AdoptionItem]; has {
1584	t.Fatal("the adoption item was carried into the new generation")
1585	}
1586	if _, err := os.Stat(adoption); !errors.Is(err, os.ErrNotExist) {
1587	t.Fatalf("the local adoption record remains: %v", err)
1588	}
1589	if disk, _ := os.ReadFile(x.envPath()); string(disk) != "K=new\nCARLOS_INSTANCE_KEY=abc\n" {
1590	t.Fatalf("Establish wrote the env file: %q", disk)
1591	}
1592	// Without an env file on disk there is nothing to carry: an error, and
1593	// no manifest.
1594	if err := os.Remove(x.envPath()); err != nil {
1595	t.Fatal(err)
1596	}
1597	if err := x.f.Establish(ctx, servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk"), launchInputs("g3"), x); err == nil {
1598	t.Fatal("established without an env file")
1599	}
1600	if _, found := x.manifest("g3"); found {
1601	t.Fatal("a manifest was written without the env file")
1602	}
1603	}
1604
1605	// Missing or malformed data is an error, never an empty-files bootstrap
1606	// (claim §6.4a): a chain with no manifest anywhere, with no pin evidence
1607	// or with evidence of published files, refuses; a malformed newest
1608	// manifest refuses rather than falling back to an older one; a chain that
1609	// leaves this region is not followed into another region's bucket. Nothing
1610	// is written in any of these.
1611	func TestEstablishNeverBootstrapsEmpty(t *testing.T) {
1612	x := newFilesFixture(t)
1613	ctx := context.Background()
1614	x.writeEnv("K=1\n")
1615	o := servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk")
1616	// No manifest anywhere, no evidence.
1617	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); !errors.Is(err, hostfiles.ErrNoEvidence) {
1618	t.Fatalf("no manifest, no pin: %v", err)
1619	}
1620	// A pin that evidences published files is not evidence of none.
1621	x.f.Evidence = func(context.Context, hostclaim.Observed) (hostfiles.PinnedFiles, bool, error) {
1622	return hostfiles.PinnedFiles{SHA256: "ab", Retained: "retained/ab"}, true, nil
1623	}
1624	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); !errors.Is(err, hostfiles.ErrNoEvidence) {
1625	t.Fatalf("no manifest, published pin: %v", err)
1626	}
1627	// A pin reader that fails is an error, not absence.
1628	x.f.Evidence = func(context.Context, hostclaim.Observed) (hostfiles.PinnedFiles, bool, error) {
1629	return hostfiles.PinnedFiles{}, false, errors.New("pin unreadable")
1630	}
1631	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); err == nil || errors.Is(err, hostfiles.ErrNoEvidence) {
1632	t.Fatalf("unreadable pin: %v", err)
1633	}
1634	x.f.Evidence = nil
1635	// The newest manifest is malformed: refused, not skipped.
1636	x.publish("g1", func(p *hostfiles.Publisher) {
1637	if err := p.Stage(ctx, "a.txt", []byte("A"), 0o600, false); err != nil {
1638	t.Fatal(err)
1639	}
1640	})
1641	x.bucket.Put("data", hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "g2")), []byte(`{"v":2,"seq":1,"entries":{"x":{"object":"elsewhere/objects/0000000000000001-`+sha(nil)+`","sha256":"`+sha(nil)+`","seq":1}},"platform":{}}`))
1642	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); !errors.Is(err, hostfiles.ErrOrigin) {
1643	t.Fatalf("malformed newest manifest: %v", err)
1644	}
1645	// The predecessor lives in another region: its manifest is not in this
1646	// bucket and the chain is not followed there.
1647	far := servingObserved("g6", "g5:uk:g1:ie", "g6:uk:g5:uk")
1648	if err := x.f.Establish(ctx, far, launchInputs("g6"), x); !errors.Is(err, hostfiles.ErrNoEvidence) {
1649	t.Fatalf("predecessor in another region: %v", err)
1650	}
1651	for _, gen := range []string{"g3", "g6"} {
1652	if _, found := x.manifest(gen); found {
1653	t.Fatalf("a manifest was written under %s", gen)
1654	}
1655	}
1656	if x.workRan != 0 {
1657	t.Fatalf("work ran %d times with nothing to establish", x.workRan)
1658	}
1659	}
1660
1661	// The one empty start besides the claim's own bootstrap (in.Empty after
1662	// EmptyAuthority): the pin evidences a host that never published files
1663	// (region HA stage B's rule, hostfiles.PinnedFiles). Then the generation
1664	// starts from no manifest, carrying only the env file. The claim's empty
1665	// bootstrap reads no chain at all.
1666	func TestEstablishHostThatNeverPublishedFiles(t *testing.T) {
1667	x := newFilesFixture(t)
1668	ctx := context.Background()
1669	x.writeEnv("K=1\n")
1670	x.f.Evidence = func(context.Context, hostclaim.Observed) (hostfiles.PinnedFiles, bool, error) {
1671	return hostfiles.PinnedFiles{}, true, nil
1672	}
1673	o := x.observe(servingObserved("g3", "g1:uk:@root:uk", "g2:uk:g1:uk", "g3:uk:g2:uk"), "g3")
1674	if err := x.f.Establish(ctx, o, launchInputs("g3"), x); err != nil {
1675	t.Fatal(err)
1676	}
1677	m, found := x.manifest("g3")
1678	if !found || m.Seq != 1 || len(m.Entries) != 0 || len(m.Platform) != 1 || m.Platform[hostfiles.EnvItem].SHA256 != sha([]byte("K=1\n")) {
1679	t.Fatalf("never-published bootstrap: %+v %v", m, found)
1680	}
1681	// The claim's own empty bootstrap: a created incarnation before its
1682	// first writer, admitted after EmptyAuthority.
1683	x.f.Evidence = nil
1684	empty := x.observe(servingObserved("g1", "g1:uk:@root:uk"), "g1") // admitted: the holder at g1 after EmptyAuthority
1685	in := launchInputs("g1")
1686	in.Empty = true
1687	if err := x.f.Establish(ctx, empty, in, x); err != nil {
1688	t.Fatal(err)
1689	}
1690	if m, found := x.manifest("g1"); !found || m.Seq != 1 || len(m.Entries) != 0 || len(x.data.reads) != 0 {
1691	t.Fatalf("empty bootstrap: %+v %v reads %v", m, found, x.data.reads)
1692	}
1693	}
1694
1695	// journal writes the prepared manifest's binding into the op journal
1696	// under the fixture's data directory, as local does.
1697	func (x *filesFixture) journal(generation string, pm preparedManifest) {
1698	x.t.Helper()
1699	x.journalRaw(generation, string(mustJSON(x.t, pm)))
1700	}
1701
1702	func (x *filesFixture) journalRaw(generation, value string) {
1703	x.t.Helper()
1704	j, err := moves.ReadJournal(x.dir, filesHost, generation)
1705	if err != nil {
1706	x.t.Fatal(err)
1707	}
1708	j.Mark(landStepFilesManifest, value)
1709	if err := j.Commit(x.dir); err != nil {
1710	x.t.Fatal(err)
1711	}
1712	}
1713
1714	// The probe's admission opens the prepared generation itself (in.Prepared,
1715	// NewGeneration = P). Its baseline is the prepared manifest at the exact
1716	// version the op journal's binding names (fix round 1, finding 2): those
1717	// bytes are read by that version, checked against the journalled digest,
1718	// validated (sequence 0, objects within the location), and must be what
1719	// the key currently holds. The env file on disk (now carrying the minted
1720	// key) is staged over it at sequence 1, a new version of the same key; the
1721	// sequence-0 version stays readable by exact version and still satisfies
1722	// stage B's rule for a prepared manifest (sequence 0, the same entries).
1723	//
1724	// A crash after the commit and before the local effect leaves the key at
1725	// sequence 1: the retry recognises its own establishment (the same
1726	// entries, the env bytes on disk, no adoption item) and writes nothing. A
1727	// binding whose digest is not the exact version's, a current manifest that
1728	// is neither the prepared one nor this box's establishment of it, a
1729	// missing binding and a missing manifest all refuse before any write.
1730	func TestEstablishOnThePreparedGenerationContinuesFromSequenceZero(t *testing.T) {
1731	x := newFilesFixture(t)
1732	ctx := context.Background()
1733	dir := filepath.Join(x.dir, filesHost)
1734	if err := os.MkdirAll(dir, 0o755); err != nil {
1735	t.Fatal(err)
1736	}
1737	if err := os.WriteFile(filepath.Join(dir, "a.txt"), []byte("A"), 0o640); err != nil {
1738	t.Fatal(err)
1739	}
1740	x.writeEnv("K=1\n")
1741	pm, err := x.f.BuildPrepared(ctx, filesLoc, "mv1", "mv1", 7, dir, x.envPath(), true, hostfiles.WakeState{V: 1, Host: filesHost, Timers: []hostfiles.WakeTimer{}})
1742	if err != nil {
1743	t.Fatal(err)
1744	}
1745	if pm.Version == "" || pm.Version != x.bucket.LatestID("data", pm.Key) {
1746	t.Fatalf("the binding's version %q is not the manifest's exact version %q", pm.Version, x.bucket.LatestID("data", pm.Key))
1747	}
1748	x.journal("mv1", pm)
1749	x.writeEnv("K=1\nCARLOS_INSTANCE_KEY=abc\n")
1750	o := servingObserved("mv1", "cur:ie:@root:ie", "mv1:uk:cur:ie")
1751	o.Claim.Op = hostclaim.Op{Kind: hostclaim.KindLanding, ID: "mv1", Phase: "probing", Exec: "uk-box", Participants: []string{"ie-box", "uk-box"}, Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}}
1752	o = x.observe(o, "mv1")
1753	in := launchInputs("mv1")
1754	in.Prepared = true
1755	// The first attempt crashes after the commit, before the local effect:
1756	// the env object and the manifest are each fenced by one With, the
1757	// local effect is the third, which refuses.
1758	x.workErr, x.workErrAt = errors.New("crash after the commit"), 3
1759	if err := x.f.Establish(ctx, o, in, x); err == nil {
1760	t.Fatal("the crash was not reported")
1761	}
1762	x.workErr, x.workErrAt = nil, 0
1763	m, found := x.manifest("mv1")
1764	if !found || m.Seq != 1 {
1765	t.Fatalf("after the crashed attempt: %+v %v", m, found)
1766	}
1767	seq1 := x.bucket.LatestID("data", pm.Key)
1768	// The retry reads its own establishment back and writes nothing.
1769	if err := x.f.Establish(ctx, o, in, x); err != nil {
1770	t.Fatalf("retry: %v", err)
1771	}
1772	m, found = x.manifest("mv1")
1773	if !found || m.Seq != 1 || m.Entries["a.txt"].Seq != 0 || m.Platform[hostfiles.EnvItem].SHA256 != sha([]byte("K=1\nCARLOS_INSTANCE_KEY=abc\n")) {
1774	t.Fatalf("after the probe's establishment: %+v %v", m, found)
1775	}
1776	if x.bucket.LatestID("data", pm.Key) != seq1 {
1777	t.Fatal("the retry wrote a new revision")
1778	}
1779	// The exact sequence-0 version is what the binding says, and stage B's
1780	// rule for a prepared manifest still holds for it: sequence 0 and the
1781	// same entries as the establishment.
1782	raw, err := x.bucket.GetVersion(ctx, "data", pm.Key, pm.Version)
1783	if err != nil || sha(raw) != pm.SHA256 {
1784	t.Fatalf("the sequence-0 version is gone or changed: %v", err)
1785	}
1786	var seq0 hostfiles.Manifest
1787	if err := json.Unmarshal(raw, &seq0); err != nil || seq0.Seq != 0 || !reflect.DeepEqual(seq0.Entries, m.Entries) {
1788	t.Fatalf("the exact prepared version: %+v %v", seq0, err)
1789	}
1790	// A binding whose digest is not the exact version's refuses.
1791	bad := pm
1792	bad.SHA256 = sha([]byte("other"))
1793	x.journal("mv1", bad)
1794	if err := x.f.Establish(ctx, o, in, x); err == nil || errors.Is(err, hostfiles.ErrNoEvidence) {
1795	t.Fatalf("digest mismatch: %v", err)
1796	}
1797	x.journal("mv1", pm)
1798	// A current manifest that is neither the prepared one nor this box's
1799	// establishment of it refuses, and is left as it is.
1800	other := m
1801	other.Entries = map[string]hostfiles.Entry{"b.txt": {Object: movesFilesPrefix(filesLoc, "mv1") + "objects/0000000000000002-" + sha([]byte("B")), SHA256: sha([]byte("B")), Size: 1, Seq: 2}}
1802	other.Seq = 2
1803	x.bucket.Put("data", pm.Key, mustJSON(t, other))
1804	if err := x.f.Establish(ctx, o, in, x); !errors.Is(err, hostbox.ErrChanged) {
1805	t.Fatalf("foreign current manifest: %v", err)
1806	}
1807	if cur, _ := x.manifest("mv1"); cur.Seq != 2 {
1808	t.Fatal("the foreign manifest was replaced")
1809	}
1810	// A prepared manifest naming an object outside the location (P6-D3)
1811	// refuses.
1812	foreign := hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{"x": {Object: "instances/other/files/g-z/objects/0000000000000000-" + sha(nil), SHA256: sha(nil), Seq: 0}}, Platform: map[string]hostfiles.Entry{}}
1813	fraw := mustJSON(t, foreign)
1814	fkey := hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "mv2"))
1815	fver := x.bucket.Put("data", fkey, fraw)
1816	x.journal("mv2", preparedManifest{Key: fkey, SHA256: sha(fraw), Version: fver, Op: "mv1", Stamp: 7})
1817	in = launchInputs("mv2")
1818	in.Prepared = true
1819	if err := x.f.Establish(ctx, o, in, x); !errors.Is(err, hostfiles.ErrOrigin) {
1820	t.Fatalf("foreign prepared manifest: %v", err)
1821	}
1822	// No binding in the op journal, and no manifest at P: an error, never a
1823	// fresh start.
1824	in = launchInputs("mv3")
1825	in.Prepared = true
1826	if err := x.f.Establish(ctx, o, in, x); err == nil || errors.Is(err, hostfiles.ErrNoEvidence) {
1827	t.Fatalf("missing binding: %v", err)
1828	}
1829	x.journal("mv3", preparedManifest{Key: hostfiles.ManifestKey(movesFilesPrefix(filesLoc, "mv3")), SHA256: sha(nil), Version: "v-none", Op: "mv1", Stamp: 7})
1830	if err := x.f.Establish(ctx, o, in, x); err == nil || errors.Is(err, hostfiles.ErrNoEvidence) {
1831	t.Fatalf("missing prepared manifest: %v", err)
1832	}
1833	if _, found := x.manifest("mv3"); found {
1834	t.Fatal("a manifest was written for a missing prepared generation")
1835	}
1836	}
1837