amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "reflect"
12 "strings"
13 "testing"
14 "time"
15
16 "github.com/carlosframework/platform/internal/activator"
17 "github.com/carlosframework/platform/internal/appconfig"
18 "github.com/carlosframework/platform/internal/genselect"
19 "github.com/carlosframework/platform/internal/hostbox"
20 "github.com/carlosframework/platform/internal/hostclaim"
21 "github.com/carlosframework/platform/internal/hostfiles"
22 "github.com/carlosframework/platform/internal/hostguard"
23 "github.com/carlosframework/platform/internal/hostwrite"
24 "github.com/carlosframework/platform/internal/instancekey"
25 "github.com/carlosframework/platform/internal/lease/leasetest"
26 "github.com/carlosframework/platform/internal/moves"
27 "github.com/carlosframework/platform/internal/moves/archive"
28 "github.com/carlosframework/platform/internal/park"
29 "github.com/carlosframework/platform/internal/provision"
30 "github.com/carlosframework/platform/internal/registry"
31 "github.com/carlosframework/platform/internal/release"
32 "github.com/superfly/ltx"
33)
34
35const (
36 poHost = "shop.bab.oncarlos.com"
37 poOp = "mv_ownersprobe"
38)
39
40// poWork is a selected work that records the identities it runs under.
41type poWork struct {
42 action string
43 f *poFixture
44 shut bool
45}
46
47func (w poWork) With(_ context.Context, id hostwrite.Identity, fn func() error) error {
48 w.f.order = append(w.f.order, w.action+":"+id.Config+"|"+id.Release)
49 if w.shut {
50 return hostwrite.ErrUnavailable
51 }
52 return fn()
53}
54
55type poFixture struct {
56 t *testing.T
57 claims release.Store
58 bucket *leasetest.VersionedBucket
59 rec *provision.Record
60 dataDir, files string
61 route registry.Route
62 manifest moves.InstallManifest
63 launch ClaimServingLaunch
64 admitted int
65 renderETags map[string]string
66 renderOverlay string
67 order []string
68 owners *movesProbeOwners
69 seq int
70 launches, reads int
71 seal moves.Seal
72 inc string
73 snapshotVersion string
74 sourceGeneration string
75 // The move's sides (poSides): the source box and its engine region, the
76 // target box, its engine region (its roster name) and its placement
77 // code, the claim's intent "to" and the fleet's home designation.
78 poSides
79}
80
81// poSides names the fixture's two boxes. The default is ie-box in ie to
82// uk-box in uk; poIntoHome is a move from ca-box in ca into the home
83// region, whose box home-box has the roster name ie, the record code "" and
84// the claim's to @home (home part 2).
85type poSides struct {
86 srcBox, srcRegion string
87 dstBox, dstEngine, dstCode, dstTo string
88 home string
89}
90
91var poNamed = poSides{srcBox: "ie-box", srcRegion: "ie", dstBox: "uk-box", dstEngine: "uk", dstCode: "uk", dstTo: "uk"}
92
93// poOption adjusts the fixture's sides before it is built.
94type poOption func(*poSides)
95
96// poIntoHome is a move from ca-box into the home region: the target is
97// home-box, its roster name ie, the record's code "" and the claim's to the
98// placement-derived @home, with the fleet's home designation ie.
99func poIntoHome(s *poSides) {
100 *s = poSides{srcBox: "ca-box", srcRegion: "ca", dstBox: "home-box", dstEngine: "ie", dstCode: "", dstTo: park.PlacementRegion(""), home: "ie"}
101}
102
103// poFlipClaim is the claim as the flip leaves it (moves §7.5): landing at
104// probing with uk-box executing; the holder is still the source's session 15
105// in generation "src" on ie-box, at the flip's stamp 18.
106func (f *poFixture) poFlipClaim() *hostclaim.Claim {
107 f.t.Helper()
108 journal := map[string]string{hostclaim.JournalSeal: f.seal.Encode(), hostclaim.JournalCutSession: "15", hostclaim.JournalEvidenceSource: "moves/bab/jam/" + poHost + "/" + poOp + "/stopped-source.json", hostclaim.JournalSourceStamp: "15", hostclaim.JournalSrcGen: "src"}
109 intent := map[string]string{"to": f.dstTo, "source": f.srcBox, "target": f.dstBox}
110 if f.home != "" {
111 intent["home_region"] = f.home
112 }
113 c := &hostclaim.Claim{V: 2, Host: poHost, Gen: 19,
114 Holder: &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + poHost + ".json", Account: "bab", App: "jam", CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), Incarnation: f.inc, Location: "instances/" + poHost, Generation: "src", Stamp: 18, Session: 15, WriterBox: f.srcBox,
115 Admissions: []hostclaim.AdmissionRecord{{Session: 15, Box: f.srcBox, Generation: "src", Predecessor: "", Region: f.srcRegion, Mode: "cold"}}},
116 Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: poOp, Phase: hostclaim.PhaseLandingProbing, Exec: f.dstBox, Participants: []string{f.srcBox, f.dstBox}, Intent: intent, Journal: journal}}
117 if err := c.Validate(); err != nil {
118 f.t.Fatal(err)
119 }
120 return c
121}
122
123func newPOFixture(t *testing.T, opts ...poOption) *poFixture {
124 t.Helper()
125 // Claim Plan 15: admissions through the real CAS record their origin
126 // under both switches; the real C8's proof reads that record.
127 t.Setenv("CARLOS_HOST_CLAIM", "1")
128 t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1")
129 at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
130 inc, err := hostclaim.Incarnation("bab", "jam", poHost, at)
131 if err != nil {
132 t.Fatal(err)
133 }
134 f := &poFixture{t: t, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket(), dataDir: t.TempDir(), files: t.TempDir(), inc: inc, poSides: poNamed}
135 for _, o := range opts {
136 o(&f.poSides)
137 }
138 f.bucket.AddBucket("data")
139 f.seal = moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: true, SealedAt: at, LeaseUntil: at.Add(time.Minute),
140 Replica: []moves.SealedObject{{Key: "moves/" + poHost + "/" + poOp + "/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: int64(ltx.HeaderSize)}},
141 Files: []moves.SealedObject{{Key: "moves/" + poHost + "/" + poOp + "/files/archive.sealed", VersionID: "v2", Size: 10}, {Key: "moves/" + poHost + "/" + poOp + "/files/archive.json", VersionID: "v3", Size: 10}}}
142 f.putClaim(f.poFlipClaim())
143 // The prepared generation the build wrote, and its manifest in the op
144 // journal (Plan 2 Task 8).
145 key := "instances/" + poHost + "/g-" + poOp + "/0009/" + ltx.FormatFilename(1, 42)
146 f.snapshotVersion = f.bucket.Put("data", key, f.ltxHeader(1, 42))
147 f.manifest = moves.InstallManifest{V: 1, Op: poOp, Host: poHost, N: 42, SnapshotKey: key, SnapshotVersion: f.snapshotVersion, SnapshotCRC64: 7, DBSHA256: strings.Repeat("d", 64)}
148 f.bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix("instances/"+poHost, poOp)), mustJSON(t, f.manifest))
149 j, err := moves.ReadJournal(f.dataDir, poHost, poOp)
150 if err != nil {
151 t.Fatal(err)
152 }
153 body, _ := json.Marshal(f.manifest)
154 j.Mark(landStepInstall, string(body))
155 // The install receipt the landing journals: the files directory and
156 // <host>.env, placed (moveslandinstall.go).
157 // The files the landing extracted (fix round 3 changes them in place).
158 if err := os.WriteFile(filepath.Join(f.files, "a.txt"), []byte("the landing's file"), 0o600); err != nil {
159 t.Fatal(err)
160 }
161 envPath := filepath.Join(f.dataDir, poHost+".env")
162 if err := os.WriteFile(envPath, []byte("CARLOS_ADMIN_TOKEN=t\n"), 0o600); err != nil {
163 t.Fatal(err)
164 }
165 filesID, err := pathIdentity(f.files)
166 if err != nil {
167 t.Fatal(err)
168 }
169 envID, err := pathIdentity(envPath)
170 if err != nil {
171 t.Fatal(err)
172 }
173 placed, _ := json.Marshal(stagedFiles{Files: filesID, Env: envID})
174 j.Mark(landStepFilesPlaced, string(placed))
175 if err := j.Commit(f.dataDir); err != nil {
176 t.Fatal(err)
177 }
178 // The prepared files manifest local wrote at the flip's stamp, before
179 // .files-gen, and its binding in the op journal (C8, Task 6).
180 f.preparedManifestJournalledAt(poOp, poOp, 18, f.files, envPath)
181 f.route = registry.Route{Host: poHost, Kind: "instance", App: "jam", Account: "bab", Status: "landing", HoldOp: poOp, Socket: filepath.Join(f.dataDir, poHost+".sock"), DBPath: filepath.Join(f.dataDir, poHost+".db")}
182 f.writeLocal(poOp, poOp, inc)
183 pins := mpPins()
184 pins.EnvReport = "sha256:env"
185 deadline := time.Date(2026, 10, 9, 12, 10, 0, 0, time.UTC)
186 f.rec = &provision.Record{Host: poHost, Account: "bab", App: "jam", Region: f.dstCode,
187 Move: &provision.MoveState{ID: poOp, Incarnation: inc, From: f.srcRegion, To: f.dstCode, State: provision.MoveLanding, LandingPhase: provision.LandProbing, Inputs: pins, CutDeadline: &deadline,
188 Landing: &provision.MoveLandingJournal{Attempts: 1, InstalledDBSHA256: "sha256:" + f.manifest.DBSHA256}}}
189 f.launch = ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:snapshot", Release: "sha256:bytes"}, Executable: "/opt/carlos/live/" + poHost, ExecutableSHA256: strings.Repeat("b", 64), Args: []string{"--socket", f.route.Socket, "--db", f.route.DBPath}, Env: []string{"A=1", "CARLOS_MOVE_PROBE=0"}, Version: pins.Release}
190 pins.EnvReport = envDigest(poEnvKey, f.launch.Env)
191 f.admitted = 3
192 f.renderETags, f.renderOverlay = pins.BundleETags, pins.OverlayETag
193 f.owners = &movesProbeOwners{Claims: f.claims, Data: f.bucket, Bucket: "data", Region: f.dstEngine, Box: f.dstBox, DataDir: f.dataDir,
194 Records: func(context.Context, string, string, string) (*provision.Record, error) { f.reads++; return f.rec, nil },
195 ReadyPath: func(context.Context, string, string) (string, error) { return "/healthz", nil },
196 FilesDir: func(string) (string, string) { return f.files, filepath.Join(f.dataDir, poHost+".env") },
197 Markers: plan9D5Markers{f.bucket},
198 Route: func(string) (*registry.Route, error) { rt := f.route; return &rt, nil },
199 // The launch and its env report, made from the same env as the
200 // production launch source makes it (claimMoveLaunchSource).
201 Launch: func(context.Context, registry.Route) (ClaimServingLaunch, moves.EnvReport, error) {
202 f.launches++
203 return f.launch, moves.ReportLaunch(poEnvKey, launchVars(f.launch.Env), nil), nil
204 },
205 EnvKey: poEnvKey,
206 Admitted: func(context.Context, *provision.Record) (int, error) { return f.admitted, nil },
207 Works: func(_ context.Context, host, action string, captured hostguard.Captured) (hostwrite.Work, error) {
208 f.order = append(f.order, "select:"+action)
209 return poWork{action: action, f: f}, nil
210 },
211 Render: func(ctx context.Context, rt registry.Route, op string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
212 f.order = append(f.order, "render")
213 w, err := work(appconfig.HeldRender{BundleETags: f.renderETags, OverlayETag: f.renderOverlay})
214 if err != nil {
215 return appconfig.HeldRender{}, err
216 }
217 if err := w.With(ctx, hostwrite.Identity{Config: "rendered-content-identity"}, func() error { return nil }); err != nil {
218 return appconfig.HeldRender{}, err
219 }
220 return appconfig.HeldRender{Path: "p", SHA256: "s", BundleETags: f.renderETags, OverlayETag: f.renderOverlay}, nil
221 },
222 Release: func(ctx context.Context, rt registry.Route, version string, work hostwrite.Work) error {
223 f.order = append(f.order, "release:"+version)
224 return work.With(ctx, hostwrite.Identity{Release: "ignored"}, func() error { return nil })
225 },
226 NewID: func() string { f.seq++; return fmt.Sprintf("fresh%d", f.seq) },
227 Now: func() time.Time { return time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) }}
228 if f.home != "" {
229 // The target is the home box: the roster grants it the alias.
230 f.owners.HomeRegion, f.owners.FleetHomeRegion = f.home, f.home
231 }
232 // The C8 provider is the real one over the fixture's bucket, as the
233 // claim assembly binds it (whole-branch fixes, item 1).
234 f.owners.Files = f.realC8()
235 return f
236}
237
238func (f *poFixture) ltxHeader(min, max uint64) []byte {
239 h := ltx.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: ltx.TXID(min), MaxTXID: ltx.TXID(max)}
240 body, err := h.MarshalBinary()
241 if err != nil {
242 f.t.Fatal(err)
243 }
244 return body
245}
246
247// writeLocal writes the two local markers the landing left: .files-gen
248// naming files/inc and <dbPath>.gen naming db ("" removes it).
249func (f *poFixture) writeLocal(files, db, inc string) {
250 f.t.Helper()
251 body, _ := json.Marshal(moves.FilesGen{Op: files, Incarnation: inc})
252 if err := os.WriteFile(filepath.Join(f.files, archive.MarkerName), body, 0o600); err != nil {
253 f.t.Fatal(err)
254 }
255 if db == "" {
256 os.Remove(f.route.DBPath + ".gen")
257 return
258 }
259 if err := hostbox.WriteGenerationMarker(f.route.DBPath+".gen", db); err != nil {
260 f.t.Fatal(err)
261 }
262}
263
264func (f *poFixture) putClaim(c *hostclaim.Claim) {
265 f.t.Helper()
266 body, err := json.Marshal(c)
267 if err != nil {
268 f.t.Fatal(err)
269 }
270 key, err := hostclaim.Key(poHost)
271 if err != nil {
272 f.t.Fatal(err)
273 }
274 etag := ""
275 if _, current, err := f.claims.Get(f.t.Context(), key); err == nil {
276 etag = current
277 }
278 if _, err := f.claims.Put(f.t.Context(), key, body, etag); err != nil {
279 f.t.Fatal(err)
280 }
281}
282
283func (f *poFixture) read() hostclaim.Observed {
284 f.t.Helper()
285 o, err := hostclaim.Read(f.t.Context(), f.claims, poHost)
286 if err != nil {
287 f.t.Fatal(err)
288 }
289 return o
290}
291
292var poEnvKey = []byte("0123456789abcdef0123456789abcdef")
293
294var poProbe = activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second, Probe: true}
295
296// admit applies the selected admission with the claim's own CAS.
297func (f *poFixture) admit(o hostclaim.Observed, in ClaimLaunchInputs) hostclaim.Observed {
298 f.t.Helper()
299 admitted, err := hostclaim.Update(f.t.Context(), f.claims, o, hostclaim.AdmitWriter(in.Admission))
300 if err != nil {
301 f.t.Fatal(err)
302 }
303 return admitted
304}
305
306// TestProbeIsAColdAdmissionIntoThePreparedGeneration (owner half; the
307// runtime half is TestMovesProbeOwnersThroughTheClaimRuntime): the first
308// probe's admission is cold, into the move id, keeping the flip's stamp,
309// sealed, from the prepared pin; the restarted attempt's is a fresh
310// generation that keeps no stamp and claims no coverage.
311func TestProbeIsAColdAdmissionIntoThePreparedGeneration(t *testing.T) {
312 f := newPOFixture(t)
313 ctx := t.Context()
314 o := f.read()
315 in, err := f.owners.Select(ctx, f.route, o, poProbe)
316 if err != nil {
317 t.Fatal(err)
318 }
319 want := hostclaim.Admission{ObservedOpID: poOp, ObservedPhase: hostclaim.PhaseLandingProbing, ObservedSession: 15, ObservedGeneration: "src", Box: "uk-box", Mode: "cold", NewGeneration: poOp, KeepStamp: true, Coverage: "sealed", Region: "uk", PredecessorRegion: "ie", RemoteMaxTXID: 42}
320 if !reflect.DeepEqual(in.Admission, want) {
321 t.Fatalf("admission:\n got %+v\nwant %+v", in.Admission, want)
322 }
323 if !in.Prepared || in.Empty || in.Baseline == nil || in.Baseline.Generation != poOp || in.Baseline.MaxTXID != 42 || in.Baseline.Pin == nil || in.Baseline.Pin.SnapshotVersion != f.snapshotVersion {
324 t.Fatalf("baseline: prepared=%v %+v", in.Prepared, in.Baseline)
325 }
326 if in.Identity != mpLaunchIdentity(f.rec.Move.Inputs) || in.ReadyPath != "/healthz" || in.DBPath != f.route.DBPath || in.Version != f.launch.Version || in.Executable != f.launch.Executable || len(in.Roles) != 1 || in.Roles[0] != "main" {
327 t.Fatalf("launch inputs: %+v", in)
328 }
329 probe := 0
330 for _, kv := range in.Env {
331 if strings.HasPrefix(kv, "CARLOS_MOVE_PROBE=") {
332 probe++
333 if kv != "CARLOS_MOVE_PROBE=1" {
334 t.Fatalf("env %q", kv)
335 }
336 }
337 }
338 if probe != 1 || !contains(in.Env, "A=1") {
339 t.Fatalf("env: %v", in.Env)
340 }
341 if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil {
342 t.Fatal(err)
343 }
344 for name, mutate := range map[string]func(*hostclaim.Admission, *genselect.Plan){
345 "no coverage": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Coverage = "" },
346 "stamp not kept": func(a *hostclaim.Admission, _ *genselect.Plan) { a.KeepStamp = false },
347 "another region": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Region = "ie" },
348 "another box": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Box = "ie-box" },
349 "a stale baseline": func(a *hostclaim.Admission, p *genselect.Plan) { p.MaxTXID, a.RemoteMaxTXID = 41, 41 },
350 "another session": func(a *hostclaim.Admission, _ *genselect.Plan) { a.ObservedSession = 14 },
351 "warm": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Mode = "warm" },
352 "another new gen": func(a *hostclaim.Admission, _ *genselect.Plan) { a.NewGeneration = "other" },
353 "a loss": func(a *hostclaim.Admission, _ *genselect.Plan) { a.LossID = "loss" },
354 "certified": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorCertified = true },
355 "predecessor where": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorRegion = "uk" },
356 } {
357 a, p := in.Admission, *in.Baseline
358 mutate(&a, &p)
359 if err := f.owners.AdmissionEvidence(ctx, o, a, &p); !errors.Is(err, hostbox.ErrChanged) {
360 t.Fatalf("%s: %v", name, err)
361 }
362 }
363
364 // The first attempt is admitted; the next attempt over it is a fresh
365 // generation with no kept stamp and no coverage, restored from the
366 // prepared pin while the move id published nothing.
367 admitted := f.admit(o, in)
368 h := admitted.Claim.Holder
369 if h.Generation != poOp || h.Stamp != 18 || h.WriterBox != "uk-box" || h.Admissions[len(h.Admissions)-1].Coverage != "sealed" {
370 t.Fatalf("first admission: %+v", h)
371 }
372 again, err := f.owners.Select(ctx, f.route, admitted, poProbe)
373 if err != nil {
374 t.Fatal(err)
375 }
376 if a := again.Admission; a.NewGeneration != "fresh1" || a.KeepStamp || a.Coverage != "" || a.ObservedGeneration != poOp || a.ObservedSession != h.Session || a.Mode != "cold" || a.PredecessorRegion != "uk" || again.Prepared {
377 t.Fatalf("restarted admission: %+v prepared=%v", a, again.Prepared)
378 }
379 if again.Baseline == nil || again.Baseline.Generation != poOp || again.Baseline.MaxTXID != 42 {
380 t.Fatalf("restarted baseline: %+v", again.Baseline)
381 }
382 if err := f.owners.AdmissionEvidence(ctx, admitted, again.Admission, again.Baseline); err != nil {
383 t.Fatal(err)
384 }
385 reused := again.Admission
386 reused.NewGeneration = poOp
387 if err := f.owners.AdmissionEvidence(ctx, admitted, reused, again.Baseline); !errors.Is(err, hostbox.ErrChanged) {
388 t.Fatalf("the move id twice: %v", err)
389 }
390}
391
392// realC8 is the production C8 provider over the fixture's bucket, data
393// directory and files paths, as the claim assembly binds it: the probe
394// owners' Files is this, not a fake (whole-branch fixes, item 1).
395func (f *poFixture) realC8() *movesFiles {
396 return &movesFiles{Files: casBucketStore{b: f.bucket, bucket: "data"}, Data: f.bucket, Bucket: "data", HomeRegion: "ie", Region: f.owners.Region, Box: f.owners.Box, DataDir: f.dataDir, EnvFile: envFileOf(f.owners.FilesDir),
397 // Plan 15's proof over the fixture's claim store: the admissions the
398 // fixture makes through the real CAS are recorded (both switches
399 // on), and recordOrigins records what a hand-built claim or the
400 // #250 backfill would.
401 Proof: ClaimOriginProof{Claims: f.claims, HomeRegion: "ie", Region: f.owners.Region}}
402}
403
404// preparedManifestJournalled builds the prepared manifest local writes for
405// generation at the installation's stamp and journals its binding, as
406// local does (movesinstall.go).
407func (f *poFixture) preparedManifestJournalled(generation, op string, stamp uint64) preparedManifest {
408 f.t.Helper()
409 _, envPath := f.owners.FilesDir(poHost)
410 return f.preparedManifestJournalledAt(generation, op, stamp, f.files, envPath)
411}
412
413func (f *poFixture) preparedManifestJournalledAt(generation, op string, stamp uint64, dir, envPath string) preparedManifest {
414 f.t.Helper()
415 pm, err := buildPreparedManifest(f.t.Context(), f.bucket, "data", "instances/"+poHost, generation, op, stamp, dir, envPath, true, hostfiles.WakeState{V: 1, Host: poHost, Timers: []hostfiles.WakeTimer{}})
416 if err != nil {
417 f.t.Fatal(err)
418 }
419 j, err := moves.ReadJournal(f.dataDir, poHost, generation)
420 if err != nil {
421 f.t.Fatal(err)
422 }
423 j.Mark(landStepFilesManifest, string(mustJSON(f.t, pm)))
424 if err := j.Commit(f.dataDir); err != nil {
425 f.t.Fatal(err)
426 }
427 return pm
428}
429
430// The landing's first probe and its committed wake establish the admitted
431// generation's files manifest through the registered C8 provider before
432// the start check (whole-branch fixes, item 1): the probe's establishment
433// takes P's prepared manifest at the journalled version, stages the env
434// file on disk and drops the adoption record; the committed wake's copies
435// it into the fresh generation. A refusal from C8 stops the start, and
436// without a provider the wake is refused by name.
437func TestLandingWakesEstablishThroughC8(t *testing.T) {
438 f := newPOFixture(t)
439 ctx := t.Context()
440 j, err := moves.ReadJournal(f.dataDir, poHost, poOp)
441 if err != nil {
442 t.Fatal(err)
443 }
444 var pm preparedManifest
445 if err := json.Unmarshal([]byte(j.Steps[landStepFilesManifest]), &pm); err != nil {
446 t.Fatal(err)
447 }
448 // The launch source minted the instance key into <host>.env before the
449 // establishment (O1'): the env file differs from the prepared item.
450 env := []byte("CARLOS_ADMIN_TOKEN=t\nCARLOS_INSTANCE_KEY=abc\n")
451 if err := os.WriteFile(filepath.Join(f.dataDir, poHost+".env"), env, 0o600); err != nil {
452 t.Fatal(err)
453 }
454 adoption := filepath.Join(f.dataDir, poHost+instancekey.AdoptionSuffix)
455 if err := os.WriteFile(adoption, []byte(`{"v":2}`), 0o600); err != nil {
456 t.Fatal(err)
457 }
458 o := f.read()
459 in, err := f.owners.Select(ctx, f.route, o, poProbe)
460 if err != nil {
461 t.Fatal(err)
462 }
463 admitted := f.admit(o, in)
464 if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); err != nil {
465 t.Fatalf("the probe's materialization: %v", err)
466 }
467 prefix := movesFilesPrefix("instances/"+poHost, poOp)
468 m, found, err := hostfiles.Load(ctx, casBucketStore{b: f.bucket, bucket: "data"}, prefix)
469 if err != nil || !found {
470 t.Fatal(err, found)
471 }
472 if m.Seq != 1 || m.Platform[hostfiles.EnvItem].SHA256 != sha(env) || m.Entries["a.txt"].Seq != 0 {
473 t.Fatalf("P's manifest after the probe: %+v", m)
474 }
475 if _, has := m.Platform[instancekey.AdoptionItem]; has {
476 t.Fatal("the adoption item was carried")
477 }
478 if _, err := os.Stat(adoption); !errors.Is(err, os.ErrNotExist) {
479 t.Fatalf("the adoption record remains: %v", err)
480 }
481 if raw, err := f.bucket.GetVersion(ctx, "data", pm.Key, pm.Version); err != nil || sha(raw) != pm.SHA256 {
482 t.Fatalf("the prepared version: %v", err)
483 }
484 if err := f.owners.Publish(ctx, admitted, in); err != nil {
485 t.Fatal(err)
486 }
487
488 // The committed wake: the claim at landing/committed, the record
489 // committed, a cold admission into a fresh generation whose files
490 // manifest is copied from P's.
491 c := *f.read().Claim
492 c.Op.Phase = hostclaim.PhaseLandingCommitted
493 f.putClaim(&c)
494 f.rec.Move.State, f.rec.Move.LandingPhase = provision.MoveCommitted, ""
495 f.route.Status, f.route.HoldOp = "ready", ""
496 o = f.read()
497 in, err = f.owners.Select(ctx, f.route, o, activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second})
498 if err != nil {
499 t.Fatalf("the committed select: %v", err)
500 }
501 if in.Prepared || in.Admission.NewGeneration == poOp {
502 t.Fatalf("the committed admission: %+v", in.Admission)
503 }
504 admitted = f.admit(o, in)
505 f.writeLocal(poOp, admitted.Claim.Holder.Generation, f.inc)
506 if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); err != nil {
507 t.Fatalf("the committed materialization: %v", err)
508 }
509 fresh, found, err := hostfiles.Load(ctx, casBucketStore{b: f.bucket, bucket: "data"}, movesFilesPrefix("instances/"+poHost, in.Admission.NewGeneration))
510 if err != nil || !found {
511 t.Fatalf("the committed wake's manifest: %v %v", err, found)
512 }
513 if fresh.Entries["a.txt"].Object != m.Entries["a.txt"].Object || fresh.Platform[hostfiles.EnvItem].SHA256 != sha(env) {
514 t.Fatalf("the committed wake's manifest: %+v", fresh)
515 }
516
517 // C8's refusal stops the start: no binding in the op journal.
518 g := newPOFixture(t)
519 gj, err := moves.ReadJournal(g.dataDir, poHost, poOp)
520 if err != nil {
521 t.Fatal(err)
522 }
523 gj.Unmark(landStepFilesManifest)
524 if err := gj.Commit(g.dataDir); err != nil {
525 t.Fatal(err)
526 }
527 o = g.read()
528 in, err = g.owners.Select(ctx, g.route, o, poProbe)
529 if err != nil {
530 t.Fatal(err)
531 }
532 admitted = g.admit(o, in)
533 if err := g.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: g}); err == nil || !errors.Is(err, errLandingChanged) {
534 t.Fatalf("a probe without a prepared manifest binding: %v", err)
535 }
536 if m, found, _ := hostfiles.Load(ctx, casBucketStore{b: g.bucket, bucket: "data"}, movesFilesPrefix("instances/"+poHost, poOp)); !found || m.Seq != 0 {
537 t.Fatalf("a refused establishment changed the prepared manifest: %+v %v", m, found)
538 }
539 // Without a provider the wake is refused by name, before anything.
540 g.owners.Files = nil
541 var dep *ClaimDependencyError
542 if err := g.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: g}); !errors.As(err, &dep) {
543 t.Fatalf("no provider: %v", err)
544 }
545}
546
547// noopFiles is a C8 provider that establishes nothing: for tests of the
548// runtime's wiring, where the real provider's inputs are not laid out.
549type noopFiles struct{}
550
551func (noopFiles) Establish(ctx context.Context, _ hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error {
552 return work.With(ctx, in.Identity, func() error { return nil })
553}
554
555func mpLaunchIdentity(p *provision.MoveInputs) hostwrite.Identity {
556 return hostwrite.Identity{Config: HeldConfigIdentity(p.BundleETags, p.OverlayETag).Config, Release: p.Release}
557}
558
559func contains(xs []string, x string) bool {
560 for _, v := range xs {
561 if v == x {
562 return true
563 }
564 }
565 return false
566}
567
568// TestProbeRefusesUnheldRowOrForeignExec: the row must be held by the
569// move, and the claim must name this box the executor of the move's
570// landing at probing; anything else is refused before any launch is read.
571func TestProbeRefusesUnheldRowOrForeignExec(t *testing.T) {
572 for name, tc := range map[string]struct {
573 route func(*registry.Route)
574 claim func(*hostclaim.Claim)
575 rec func(*provision.Record)
576 policy func(*activator.StartPolicy)
577 }{
578 "a ready row": {route: func(r *registry.Route) { r.Status, r.HoldOp = "ready", "" }},
579 "held by another op": {route: func(r *registry.Route) { r.HoldOp = "mv_other" }},
580 "another host's row": {route: func(r *registry.Route) { r.Host = "other.bab.oncarlos.com" }},
581 "a foreign executor": {claim: func(c *hostclaim.Claim) { c.Op.Exec = "ie-box" }},
582 "not the target": {claim: func(c *hostclaim.Claim) { c.Op.Intent["target"] = "ie-box" }},
583 "another region": {claim: func(c *hostclaim.Claim) { c.Op.Intent["to"] = "ie" }},
584 "not probing": {claim: func(c *hostclaim.Claim) { c.Op.Phase = provision.LandRestored }},
585 "not a landing": {claim: func(c *hostclaim.Claim) { c.Op.Kind = hostclaim.KindReverting }},
586 "declared lost": {claim: func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"} }},
587 "a marked record": {rec: func(r *provision.Record) { r.Move.Reason = provision.RollbackProbeAttempts }},
588 "a record past probing": {rec: func(r *provision.Record) { r.Move.LandingPhase = provision.LandProbed }},
589 "no pinned inputs": {rec: func(r *provision.Record) { r.Move.Inputs = nil }},
590 "not a probe": {policy: func(p *activator.StartPolicy) { p.Probe = false }},
591 "other pins": {policy: func(p *activator.StartPolicy) { p.Inputs = &provision.MoveInputs{Release: "v0"} }},
592 "another manifest": {policy: func(p *activator.StartPolicy) { p.Prepared = &moves.InstallManifest{Op: poOp, N: 41} }},
593 } {
594 t.Run(name, func(t *testing.T) {
595 f := newPOFixture(t)
596 if tc.claim != nil {
597 c := f.poFlipClaim()
598 tc.claim(c)
599 f.putClaim(c)
600 }
601 if tc.route != nil {
602 tc.route(&f.route)
603 }
604 if tc.rec != nil {
605 tc.rec(f.rec)
606 }
607 p := poProbe
608 if tc.policy != nil {
609 tc.policy(&p)
610 }
611 if _, err := f.owners.Select(t.Context(), f.route, f.read(), p); err == nil {
612 t.Fatal("selected")
613 }
614 if f.launches != 0 {
615 t.Fatal("the launch was read")
616 }
617 })
618 }
619}
620
621// The prepared inputs are validated before any admission: the local markers
622// name the move id and the holder's incarnation, and the journalled
623// manifest is the seal's.
624func TestMovesProbeOwnersValidateThePreparedInputs(t *testing.T) {
625 for name, tc := range map[string]struct {
626 mutate func(*poFixture)
627 want error
628 }{
629 "files of another op": {func(f *poFixture) { f.writeLocal("mv_other", poOp, f.inc) }, moves.ErrStartFiles},
630 "files of another incarnation": {func(f *poFixture) { f.writeLocal(poOp, poOp, "other") }, moves.ErrStartFiles},
631 "no files marker": {func(f *poFixture) {
632 os.Remove(filepath.Join(f.files, archive.MarkerName))
633 }, moves.ErrStartFiles},
634 "a database of another op": {func(f *poFixture) { f.writeLocal(poOp, "src", f.inc) }, moves.ErrStartDB},
635 "no database marker": {func(f *poFixture) { f.writeLocal(poOp, "", f.inc) }, moves.ErrStartDB},
636 "a manifest at another N": {func(f *poFixture) {
637 f.manifest.N = 41
638 j, _ := moves.ReadJournal(f.dataDir, poHost, poOp)
639 body, _ := json.Marshal(f.manifest)
640 j.Mark(landStepInstall, string(body))
641 j.Commit(f.dataDir)
642 }, hostbox.ErrChanged},
643 "no manifest": {func(f *poFixture) {
644 j, _ := moves.ReadJournal(f.dataDir, poHost, poOp)
645 j.Unmark(landStepInstall)
646 j.Commit(f.dataDir)
647 }, hostbox.ErrChanged},
648 } {
649 t.Run(name, func(t *testing.T) {
650 f := newPOFixture(t)
651 tc.mutate(f)
652 if _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe); !errors.Is(err, tc.want) {
653 t.Fatalf("got %v, want %v", err, tc.want)
654 }
655 })
656 }
657}
658
659// The launch-input pin at spawn (§7.7, round-2 finding 14): the live
660// release and the launch environment's report are the ones pinned at
661// readiness, else the probe fails with the recorded rollback sentence
662// before any admission.
663func TestMovesProbeOwnersPinTheLaunchInputs(t *testing.T) {
664 for name, mutate := range map[string]func(*poFixture){
665 "another release": func(f *poFixture) { f.launch.Version = "v9" },
666 "another environment": func(f *poFixture) { f.rec.Move.Inputs.EnvReport = "sha256:other" },
667 } {
668 t.Run(name, func(t *testing.T) {
669 f := newPOFixture(t)
670 mutate(f)
671 _, err := f.owners.Select(t.Context(), f.route, f.read(), poProbe)
672 var changed *ProbeInputsChanged
673 if !errors.As(err, &changed) || changed.Reason != provision.RollbackConfigChanged {
674 t.Fatalf("got %v", err)
675 }
676 })
677 }
678}
679
680// TestFirstProbeFromTheSourceGenerationIsAdmitted: from the claim as the
681// flip left it, the prepared inputs validate, the admission names the move
682// id, and Materialize, before every start, renders and materializes the held
683// inputs under their own selections and passes the §5.1 start check after
684// the CAS. A hand-built claim already in the move id whose files name
685// another op is refused there.
686func TestFirstProbeFromTheSourceGenerationIsAdmitted(t *testing.T) {
687 f := newPOFixture(t)
688 ctx := t.Context()
689 o := f.read()
690 in, err := f.owners.Select(ctx, f.route, o, poProbe)
691 if err != nil {
692 t.Fatal(err)
693 }
694 admitted := f.admit(o, in)
695 f.order = nil
696 spawn := poWork{action: "spawn", f: f}
697 if err := f.owners.Materialize(ctx, admitted, in, spawn); err != nil {
698 t.Fatal(err)
699 }
700 pins := f.rec.Move.Inputs
701 launch := mpLaunchIdentity(pins)
702 want := []string{
703 "spawn:" + launch.Config + "|" + launch.Release,
704 "render", "select:held-config", "held-config:" + HeldConfigIdentity(pins.BundleETags, pins.OverlayETag).Config + "|",
705 "select:held-release", "release:" + pins.Release, "held-release:|" + pins.Release,
706 // The prepared marker under the writer's fence: before the request
707 // and in the store before its attempt.
708 "spawn:" + launch.Config + "|" + launch.Release, "spawn:" + launch.Config + "|" + launch.Release,
709 // C8's establishment (item 1): the env file is still the prepared
710 // item's and there is no adoption item, so nothing is published and
711 // only the local effect runs under the work.
712 "spawn:" + launch.Config + "|" + launch.Release,
713 }
714 if !reflect.DeepEqual(f.order, want) {
715 t.Fatalf("order:\n got %q\nwant %q", f.order, want)
716 }
717
718 // A render that is not the pinned configuration fails the probe and
719 // materializes nothing.
720 f.order = nil
721 f.renderOverlay = "e-changed"
722 err = f.owners.Materialize(ctx, admitted, in, spawn)
723 var changed *ProbeInputsChanged
724 if !errors.As(err, &changed) || changed.Reason != provision.RollbackConfigChanged || contains(f.order, "select:held-release") {
725 t.Fatalf("a changed render: %v %v", err, f.order)
726 }
727 f.renderOverlay = pins.OverlayETag
728
729 // The start check against the claim after the CAS.
730 f.writeLocal("mv_other", poOp, f.inc)
731 if err := f.owners.Materialize(ctx, admitted, in, spawn); !errors.Is(err, moves.ErrStartFiles) {
732 t.Fatalf("files of another op: %v", err)
733 }
734 f.writeLocal(poOp, "src", f.inc)
735 if err := f.owners.Materialize(ctx, admitted, in, spawn); !errors.Is(err, moves.ErrStartDB) {
736 t.Fatalf("a database the restore did not write: %v", err)
737 }
738 f.writeLocal(poOp, poOp, f.inc)
739 // The spawn selection itself refused: nothing is rendered.
740 f.order = nil
741 if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f, shut: true}); !errors.Is(err, hostwrite.ErrUnavailable) || contains(f.order, "render") {
742 t.Fatalf("a refused spawn selection: %v %v", err, f.order)
743 }
744}
745
746func mustJSON(t *testing.T, v any) []byte {
747 t.Helper()
748 b, err := json.Marshal(v)
749 if err != nil {
750 t.Fatal(err)
751 }
752 return b
753}
754
755// The prepared marker (moves §7.14 marker row, R10-3): the first probe's
756// admission publishes generation.json naming the move id with the build's
757// pin, superseding an earlier stay's marker by its exact ETag, before
758// replication; the evidence is this process's receipt.
759func TestMovesProbeOwnersPublishThePreparedMarker(t *testing.T) {
760 f := newPOFixture(t)
761 ctx := t.Context()
762 key := "instances/" + poHost + "/generation.json"
763 f.bucket.Put("data", key, mustJSON(t, genselect.Marker{Op: "earlier", Incarnation: f.inc, Host: poHost, Box: "uk-box", InstalledAt: time.Unix(1, 0).UTC(), Pin: genselect.Pin{SnapshotTXID: 3}}))
764 o := f.read()
765 in, err := f.owners.Select(ctx, f.route, o, poProbe)
766 if err != nil {
767 t.Fatal(err)
768 }
769 admitted := f.admit(o, in)
770 if err := f.owners.PublicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) {
771 t.Fatalf("evidence before publication: %v", err)
772 }
773 // Materialize publishes under the writer's fence; Group D's Publish
774 // records it.
775 if err := f.owners.Materialize(ctx, admitted, in, poWork{action: "spawn", f: f}); err != nil {
776 t.Fatal(err)
777 }
778 if err := f.owners.Publish(ctx, admitted, in); err != nil {
779 t.Fatal(err)
780 }
781 body, ok := f.bucket.Current("data", key)
782 var m genselect.Marker
783 if !ok || json.Unmarshal(body, &m) != nil || m.Op != poOp || m.Box != "uk-box" || m.Incarnation != f.inc || m.Host != poHost ||
784 m.Pin != (genselect.Pin{SnapshotKey: f.manifest.SnapshotKey, SnapshotVersion: f.manifest.SnapshotVersion, SnapshotTXID: 42, SnapshotCRC64: 7}) || m.ManifestSHA256 == "" {
785 t.Fatalf("marker: %s", body)
786 }
787 if err := f.owners.PublicationEvidence(ctx, admitted); err != nil {
788 t.Fatal(err)
789 }
790 written := f.bucket.LatestID("data", key)
791 if err := f.owners.Publish(ctx, admitted, in); err != nil || f.bucket.LatestID("data", key) != written {
792 t.Fatalf("replay: %v", err)
793 }
794 // A restarted attempt's baseline is now the published move id.
795 again, err := f.owners.Select(ctx, f.route, admitted, poProbe)
796 if err != nil || again.Baseline.Generation != poOp || again.Baseline.Marker == nil || again.Baseline.Marker.Op != poOp {
797 t.Fatalf("restarted baseline after publication: %+v %v", again.Baseline, err)
798 }
799 f.bucket.Put("data", key, mustJSON(t, genselect.Marker{Op: poOp, Incarnation: f.inc, Host: poHost, Box: "ie-box", InstalledAt: time.Unix(2, 0).UTC(), Pin: m.Pin, ManifestSHA256: m.ManifestSHA256}))
800 if err := f.owners.PublicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) {
801 t.Fatalf("evidence for a marker this session did not write: %v", err)
802 }
803 if err := f.owners.EmptyAuthority(ctx, admitted, in); err == nil {
804 t.Fatal("a move admission claimed the empty bootstrap")
805 }
806}
807
808// A restarted attempt's fresh generation is published fenced, after its own
809// first root, as serving publishes.
810func TestMovesProbeOwnersPublishARestartedAttemptFenced(t *testing.T) {
811 f := newPOFixture(t)
812 ctx := t.Context()
813 o := f.read()
814 in, err := f.owners.Select(ctx, f.route, o, poProbe)
815 if err != nil {
816 t.Fatal(err)
817 }
818 first := f.admit(o, in)
819 again, err := f.owners.Select(ctx, f.route, first, poProbe)
820 if err != nil {
821 t.Fatal(err)
822 }
823 admitted := f.admit(first, again)
824 f.restore(admitted.Claim.Holder.Generation)
825 if err := f.owners.Materialize(ctx, admitted, again, newFenceWork()); err != nil {
826 t.Fatal(err)
827 }
828 synced := 0
829 pub := ClaimPublication{Sync: func(context.Context) error { synced++; return nil }, Check: func(context.Context) error { return nil }}
830 if err := f.owners.Publish(ctx, admitted, again); !errors.Is(err, hostbox.ErrChanged) {
831 t.Fatalf("an unfenced marker for a fresh generation: %v", err)
832 }
833 if _, err := f.owners.PublishFenced(ctx, admitted, again, pub); !errors.Is(err, hostbox.ErrChanged) || synced != 1 {
834 t.Fatalf("a marker before the root: %v synced=%d", err, synced)
835 }
836 root := "instances/" + poHost + "/g-" + again.Admission.NewGeneration + "/0009/" + ltx.FormatFilename(1, 44)
837 f.bucket.Put("data", root, f.ltxHeader(1, 44))
838 receipt, err := f.owners.PublishFenced(ctx, admitted, again, pub)
839 if err != nil || receipt.Root.Key != root || receipt.MarkerVersion == "" {
840 t.Fatalf("publish: %+v %v", receipt, err)
841 }
842 if err := f.owners.PublicationEvidence(ctx, admitted); err != nil {
843 t.Fatal(err)
844 }
845}
846
847// TestMovesProbeOwnersThroughTheClaimRuntime drives the claim's own runtime
848// (claim Plan 9 Group D's ClaimWakeAssembly, the recovery fixture's
849// controller) with this owner set and the moves probe attempts bound, as
850// production binds them: the admission is the prepared cold admission,
851// journalled before any start; the probe start carries the moves attempt
852// journal; and Materialize renders the held inputs before any launch is
853// prepared.
854func TestMovesProbeOwnersThroughTheClaimRuntime(t *testing.T) {
855 f := newRecoveryFixture(t)
856 ctx := t.Context()
857 journalDir, filesDir := t.TempDir(), t.TempDir()
858 seal := moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: true, SealedAt: f.now, LeaseUntil: f.now.Add(time.Minute),
859 Replica: []moves.SealedObject{{Key: "moves/" + recoveryHost + "/" + poOp + "/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: int64(ltx.HeaderSize)}},
860 Files: []moves.SealedObject{{Key: "moves/" + recoveryHost + "/" + poOp + "/files/archive.sealed", VersionID: "v2", Size: 10}}}
861 if _, err := hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error {
862 c.Op = hostclaim.Op{Kind: hostclaim.KindLanding, ID: poOp, Phase: hostclaim.PhaseLandingProbing, Exec: "box-1", Participants: []string{"box-0", "box-1"},
863 Intent: map[string]string{"to": "ie", "source": "box-0", "target": "box-1"}, Journal: map[string]string{hostclaim.JournalSeal: seal.Encode()}}
864 return nil
865 }); err != nil {
866 t.Fatal(err)
867 }
868 o := f.read()
869 h := o.Claim.Holder
870 bucket := leasetest.NewVersionedBucket()
871 bucket.AddBucket("data")
872 key := strings.TrimSuffix(h.Location, "/") + "/g-" + poOp + "/0009/" + ltx.FormatFilename(1, 42)
873 hdr, _ := (&ltx.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: 1, MaxTXID: 42}).MarshalBinary()
874 manifest := moves.InstallManifest{V: 1, Op: poOp, Host: recoveryHost, N: 42, SnapshotKey: key, SnapshotVersion: bucket.Put("data", key, hdr), DBSHA256: strings.Repeat("d", 64)}
875 bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix(h.Location, poOp)), mustJSON(t, manifest))
876 j, _ := moves.ReadJournal(journalDir, recoveryHost, poOp)
877 body, _ := json.Marshal(manifest)
878 j.Mark(landStepInstall, string(body))
879 if err := j.Commit(journalDir); err != nil {
880 t.Fatal(err)
881 }
882 route := registry.Route{Host: recoveryHost, Kind: "instance", App: "jam", Account: "bab", Status: "landing", HoldOp: poOp, Socket: filepath.Join(f.guard.DataDir, "app.sock"), DBPath: filepath.Join(f.guard.DataDir, "state", "data.db")}
883 if err := os.MkdirAll(filepath.Dir(route.DBPath), 0o700); err != nil {
884 t.Fatal(err)
885 }
886 if err := hostbox.WriteGenerationMarker(route.DBPath+".gen", poOp); err != nil {
887 t.Fatal(err)
888 }
889 if err := os.WriteFile(filepath.Join(filesDir, archive.MarkerName), mustJSON(t, moves.FilesGen{Op: poOp, Incarnation: h.Incarnation}), 0o600); err != nil {
890 t.Fatal(err)
891 }
892 pins := mpPins()
893 pins.EnvReport = envDigest(poEnvKey, []string{"A=1"})
894 deadline := f.now.Add(time.Hour)
895 rec := &provision.Record{Host: recoveryHost, Account: "bab", App: "jam", Region: "ie",
896 Move: &provision.MoveState{ID: poOp, Incarnation: h.Incarnation, From: "uk", To: "ie", State: provision.MoveLanding, LandingPhase: provision.LandProbing, Inputs: pins, CutDeadline: &deadline, CutAttempts: 3,
897 Landing: &provision.MoveLandingJournal{Attempts: 1, InstalledDBSHA256: "sha256:" + manifest.DBSHA256}}}
898 records := func(context.Context, string, string, string) (*provision.Record, error) { return rec, nil }
899 var order []string
900 launches := 0
901 owners := &movesProbeOwners{Claims: f.claims, Records: records, Data: bucket, Bucket: "data", Region: "ie", Box: "box-1", DataDir: journalDir, Markers: plan9D5Markers{bucket}, Files: noopFiles{},
902 ReadyPath: func(context.Context, string, string) (string, error) { return "/ready", nil },
903 FilesDir: func(string) (string, string) { return filesDir, "" },
904 Route: func(string) (*registry.Route, error) { rt := route; return &rt, nil },
905 Launch: func(context.Context, registry.Route) (ClaimServingLaunch, moves.EnvReport, error) {
906 return ClaimServingLaunch{Executable: "/usr/bin/true", ExecutableSHA256: strings.Repeat("a", 64), Env: []string{"A=1"}, Version: pins.Release}, moves.ReportLaunch(poEnvKey, map[string]string{"A": "1"}, nil), nil
907 },
908 EnvKey: poEnvKey,
909 Admitted: MovesRecordAdmittedAttempts,
910 Works: func(_ context.Context, _, action string, _ hostguard.Captured) (hostwrite.Work, error) {
911 order = append(order, "select:"+action)
912 return plan9D5Work{}, nil
913 },
914 Render: func(ctx context.Context, _ registry.Route, _ string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
915 order = append(order, "render")
916 if _, err := work(appconfig.HeldRender{BundleETags: pins.BundleETags, OverlayETag: pins.OverlayETag}); err != nil {
917 return appconfig.HeldRender{}, err
918 }
919 return appconfig.HeldRender{BundleETags: pins.BundleETags, OverlayETag: pins.OverlayETag}, nil
920 },
921 Release: func(context.Context, registry.Route, string, hostwrite.Work) error {
922 order = append(order, "release")
923 return nil
924 },
925 NewID: f.newID, Now: func() time.Time { return f.now }}
926 attempts := &movesProbeAttempts{Claims: f.claims, Records: records, DataDir: journalDir, NewID: f.newID, Now: func() time.Time { return f.now },
927 Admitted: MovesRecordAdmittedAttempts}
928 c := f.controller("boot-1", "controller-1", 4101, "start-1")
929 bound := owners.MovesOwners(attempts)
930 c.wake.Inputs = ClaimOwnerDispatch{Moves: &bound.ClaimKindOwners}
931 c.wake.Continue = hostclaim.SessionContinues
932 c.wake.Operations = bound.Operations
933 c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }
934 c.wake.Launch = func(context.Context, hostclaim.Observed, hostwrite.Work, ClaimLaunchInputs, *activator.RoleLaunch) (*activator.RoleLaunch, error) {
935 launches++
936 return nil, hostwrite.ErrUnavailable
937 }
938 live, cancel := context.WithCancel(context.Background())
939 t.Cleanup(cancel)
940 done := make(chan struct{})
941 go func() { <-live.Done(); close(done) }()
942 c.wake.mu.Lock()
943 c.wake.pending[recoveryHost] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done}
944 c.wake.mu.Unlock()
945
946 admitted, err := c.runtime.Admit(ctx, route, o, poProbe)
947 if err != nil {
948 t.Fatal(err)
949 }
950 ah := admitted.Claim.Holder
951 last := ah.Admissions[len(ah.Admissions)-1]
952 if ah.Generation != poOp || ah.Stamp != h.Stamp || ah.WriterBox != "box-1" || last.Mode != "cold" || last.Coverage != "sealed" || last.Generation != poOp || last.Region != "ie" {
953 t.Fatalf("the admission: %+v %+v", ah, last)
954 }
955 s, err := c.runtime.Prepare(ctx, route, admitted, poProbe)
956 if err != nil {
957 t.Fatal(err)
958 }
959 if s.OpenProbe == nil || s.Identity != mpLaunchIdentity(pins) {
960 t.Fatalf("the probe start: %+v", s.Identity)
961 }
962 if opened, err := s.OpenProbe(ctx, f.now); err != nil || !opened.Equal(f.now) {
963 t.Fatalf("open: %v %v", opened, err)
964 }
965 if err := s.Reserve(ctx, 0); err != nil {
966 t.Fatal(err)
967 }
968 if a := s.CurrentAttempt(); a.ID == "" || a.Session != ah.Session || a.ControllerID != "controller-1" {
969 t.Fatalf("the reservation: %+v", a)
970 }
971 if err := s.Materialize(ctx); err != nil {
972 t.Fatal(err)
973 }
974 if want := []string{"render", "select:held-config", "select:held-release", "release"}; !reflect.DeepEqual(order, want) || launches != 0 {
975 t.Fatalf("before the launch: %v launches=%d", order, launches)
976 }
977 if _, err := s.PrepareLaunch(ctx); err == nil || launches != 1 {
978 t.Fatalf("the launch after Materialize: %v %d", err, launches)
979 }
980}
981