amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "context"
7 "crypto/ecdh"
8 "crypto/sha256"
9 "encoding/hex"
10 "encoding/json"
11 "errors"
12 "io/fs"
13 "os"
14 "path/filepath"
15 "strings"
16 "syscall"
17 "testing"
18 "time"
19
20 "github.com/carlosframework/platform/internal/appconfig"
21 "github.com/carlosframework/platform/internal/hostbox"
22 "github.com/carlosframework/platform/internal/hostclaim"
23 execlease "github.com/carlosframework/platform/internal/hostexec"
24 "github.com/carlosframework/platform/internal/hostfiles"
25 "github.com/carlosframework/platform/internal/hostguard"
26 "github.com/carlosframework/platform/internal/hostrelocate"
27 "github.com/carlosframework/platform/internal/hostwrite"
28 "github.com/carlosframework/platform/internal/instancekey"
29 "github.com/carlosframework/platform/internal/moves"
30 "github.com/carlosframework/platform/internal/park"
31 "github.com/carlosframework/platform/internal/provision"
32 "github.com/carlosframework/platform/internal/registry"
33 "github.com/carlosframework/platform/internal/release"
34)
35
36const installEnv = "# the host's own env\nCARLOS_ADMIN_TOKEN=t0k3n\n CARLOS_INSTANCE_KEY = q+/Zk3vJ0aW1Yx9sR7tUe2oPl4mN8bC6dF5gH0iJ1kA=\nAPP_MODE=prod"
37
38// installFixture is a fresh spare at installed, after P's build: the
39// production box layout (bootstrap), the real EmptyInstallationCensus with
40// claimGlobalSharedPaths, the real ReconcileAdapter with park.Local, main's
41// ClaimRoleAssembly.PrepareState, and the claim, record and P's files
42// manifest as the build left them. The held config render and the held
43// stage are seams that select their work exactly as production does.
44type installFixture struct {
45 *buildFixture
46 r ClaimRuntimeResources
47 reg *registry.Registry
48 inv *hostbox.StopInventoryDB
49 actor hostbox.Actor
50 stageErr error // the held stage's answer; nil points live/<host> at the version
51 renders int
52 configs map[string]string // the rendered config's variables
53 // verifyErr is the held reading's answer once live/<host> names the
54 // version (release.VerifyHeldLiveUnder's bytes check); lockProbe runs
55 // inside the publication's guard and the reading's lock.
56 verifyErr error
57 lockProbe func(where string)
58 published int
59}
60
61// installWake is the fixture's published wake state: two cron timers.
62func installWake() string {
63 ws, _ := json.Marshal(hostfiles.WakeState{V: 1, Host: buildHost, NextDue: 1700000000, Timers: []hostfiles.WakeTimer{
64 {Name: "nightly", Kind: "cron", Spec: "0 3 * * *", Path: "/jobs/nightly", NextAt: 1700000000, CreatedAt: 1690000000, Registration: "r1"},
65 {Name: "hourly", Kind: "cron", Spec: "0 * * * *", Path: "/jobs/hourly", NextAt: 1700000100, CreatedAt: 1690000000, Registration: "r2"},
66 }})
67 return string(ws)
68}
69
70func newInstallFixture(t *testing.T, platform map[string]string) *installFixture {
71 t.Helper()
72 if platform == nil {
73 platform = map[string]string{hostfiles.EnvItem: installEnv, hostfiles.WakeItem: installWake()}
74 }
75 return newInstallFixtureWith(t, buildOptions{Platform: platform, Files: map[string]string{"public/index.html": "<h1>shop</h1>", "uploads/a/b.png": "png-bytes"}})
76}
77
78// newInstallFixtureWith is newInstallFixture over the build options opts
79// (opts.Data is the bootstrapped box's).
80func newInstallFixtureWith(t *testing.T, opts buildOptions) *installFixture {
81 t.Helper()
82 ctx := t.Context()
83 for _, k := range []string{"CARLOS_HOST_CLAIM", "CARLOS_MOVES_PHASE1", "CARLOS_REGION_HA"} {
84 t.Setenv(k, "1")
85 }
86 r := claimCensusBox(t)
87 opts.Data = r.DataDir
88 b := newBuildFixtureWith(t, opts)
89 f := &installFixture{buildFixture: b, r: r, actor: hostbox.Actor{Box: "dst", Enrollment: strings.Repeat("a", 32), Region: "ie", Revision: 4, Binding: "4"}}
90 // The record the claim's holder names: a hibernating instance that
91 // derives its paths.
92 if err := provision.CreateRecord(ctx, b.claims, provision.Record{Schema: 1, Host: buildHost, Slug: "shop", App: "app", Account: "acct", Region: "ie", Channel: "stable", CreatedAt: time.Date(2026, 10, 7, 1, 0, 0, 0, time.UTC)}); err != nil && !errors.Is(err, release.ErrCASConflict) {
93 t.Fatal(err)
94 }
95 if err := b.build(); err != nil {
96 t.Fatal(err)
97 }
98 // installed as ToInstalled leaves it: the verified build's prepared
99 // authority, bound to the install stamp the build ran at.
100 auth, done, err := b.verifier.Verify(ctx, b.obs)
101 if err != nil || !done {
102 t.Fatalf("P is not verified: done=%v %v", done, err)
103 }
104 o, err := hostclaim.Update(ctx, b.claims, b.obs, func(c *hostclaim.Claim) error {
105 c.Op.Phase = hostclaim.PhaseInstalled
106 auth.InstallStamp = c.Holder.Stamp
107 c.Op.Journal["prepared_authority"] = string(mustJSON(t, auth))
108 c.Op.Intent["target_enrollment"] = f.actor.Enrollment
109 c.Op.Journal["exec_ack.src"], c.Op.Journal["exec_ack.dst"] = `{"op":"rel-1"}`, `{"op":"rel-1"}`
110 return nil
111 })
112 if err != nil {
113 t.Fatal(err)
114 }
115 b.obs = o
116 f.onBox(r, f.actor)
117 b.cfg.render = f.render
118 b.cfg.materialize = f.materialize
119 b.cfg.verifyLive = f.verifyLive
120 return f
121}
122
123// onBox points the fixture's install at box actor over the census box r:
124// its registry, typed inventory, launch journal and census, its guard over
125// r's data directory, main's reconciler and role assembly, and the
126// install's paths and work source. A replacement target is a second call
127// with a fresh box (newReplacedProbeFilesFixture).
128func (f *installFixture) onBox(r ClaimRuntimeResources, actor hostbox.Actor) {
129 t, b := f.t, f.buildFixture
130 var err error
131 f.r, f.actor = r, actor
132 if err := os.MkdirAll(filepath.Join(r.DataDir, "hostguard"), 0o770); err != nil {
133 t.Fatal(err)
134 }
135 if f.reg, err = registry.Open(filepath.Join(r.DataDir, "registry.db")); err != nil {
136 t.Fatal(err)
137 }
138 reg := f.reg
139 t.Cleanup(func() { reg.Close() })
140 if f.inv, err = openLegacyInventory(t, filepath.Join(r.DataDir, ".hostclaims", "inventory.db")); err != nil {
141 t.Fatal(err)
142 }
143 inv := f.inv
144 t.Cleanup(func() { inv.Close() })
145 launches, err := hostbox.OpenLaunchJournal(filepath.Join(r.DataDir, ".hostclaims", "launches.db"))
146 if err != nil {
147 t.Fatal(err)
148 }
149 t.Cleanup(func() { launches.Close() })
150 census := &hostbox.EmptyInstallationCensus{Inventory: f.inv, Registry: f.reg, Launches: launches, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "4",
151 Roots: map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir}, SharedPaths: claimGlobalSharedPaths(r)}
152 if b.a.Guard.DataDir != r.DataDir {
153 b.a.Guard = hostguard.Guard{DataDir: r.DataDir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: b.claims}}
154 }
155 guard := b.a.Guard
156 b.a.Reconcile = &hostbox.ReconcileAdapter{Registry: f.reg, Inventory: f.inv, Census: census, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "4",
157 Reconciler: hostbox.Reconciler{Claims: b.claims, Guard: guard, Box: actor.Box, Region: "ie", Local: park.Local{Guard: guard, Claims: b.claims, DataDir: r.DataDir, Box: actor.Box}}}
158 b.a.Wake = &ClaimWakeAssembly{Roles: &ClaimRoleAssembly{Inventory: f.inv, Identity: handoverRecoveryIdentity(t, actor), ManagedRoots: map[string]string{"data": r.DataDir}}}
159 b.a.stopLocal = func(context.Context) (claimStopLocal, error) {
160 return claimStopLocal{Actor: actor, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil
161 }
162 b.cfg.Registry, b.cfg.OptDir, b.cfg.DataDir, b.cfg.WorkDir = f.reg, r.OptDir, r.DataDir, filepath.Join(r.DataDir, "relocate", "build")
163 b.cfg.Source.Box, b.cfg.Source.Guard, b.cfg.Source.Now = actor.Box, guard, time.Now
164}
165
166// render stands for appconfig.RenderHeldUnder: it selects its work with
167// the merged versions, as production does, and writes the held config in
168// the host's instance directory through that work.
169func (f *installFixture) render(ctx context.Context, _ release.Store, _ *ecdh.PrivateKey, dataDir string, rt registry.Route, op, box string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
170 f.renders++
171 pre := appconfig.HeldRender{BundleETags: map[string]string{"default": "d1", "environment": "e1"}, OverlayETag: "o1"}
172 w, err := work(pre)
173 if err != nil {
174 return appconfig.HeldRender{}, err
175 }
176 if rt.Status != "relocating" || rt.HoldOp != op || box != f.actor.Box {
177 return appconfig.HeldRender{}, appconfig.ErrNotHeld
178 }
179 var body strings.Builder
180 for k, v := range f.configs {
181 body.WriteString(k + "=" + v + "\n")
182 }
183 path := filepath.Join(provision.InstanceDir(dataDir, rt.Host), "config.env")
184 if err := w.With(ctx, hostwrite.Identity{}, func() error { return os.WriteFile(path, []byte(body.String()), 0o600) }); err != nil {
185 return appconfig.HeldRender{}, err
186 }
187 sum := sha256.Sum256([]byte(body.String()))
188 pre.Path, pre.SHA256 = path, hex.EncodeToString(sum[:])
189 return pre, nil
190}
191
192// materialize stands for release.MaterializeHeldUnder: blocked (the edge
193// unit cannot write the version store) or, through its work and inside
194// guard (the caller's held-stage lock), live/<host> pointed at the pinned
195// version.
196func (f *installFixture) materialize(ctx context.Context, _ release.Store, optDir string, rt registry.Route, version string, _ *release.ProofGate, work hostwrite.Work, guard func(func() error) error) (*release.Manifest, error) {
197 if f.stageErr != nil {
198 return nil, f.stageErr
199 }
200 if guard == nil {
201 guard = func(publish func() error) error { return publish() }
202 }
203 return &release.Manifest{}, work.With(ctx, hostwrite.Identity{Release: version}, func() error {
204 return guard(func() error {
205 if f.lockProbe != nil {
206 f.lockProbe("publish")
207 }
208 f.published++
209 return f.publishLive(rt.Account, rt.App, version)
210 })
211 })
212}
213
214// verifyLive stands for release.VerifyHeldLiveUnder: a peek at the link,
215// then, under lock, the link names the version and the bytes check answers
216// verifyErr.
217func (f *installFixture) verifyLive(_ context.Context, _ release.Store, optDir string, rt registry.Route, version string, proof *release.ProofGate, lock func(func() error) error) error {
218 if lock == nil || proof == nil {
219 return errors.New("a held reading without its lock or gate")
220 }
221 if live, err := release.LiveVersion(optDir, rt.Host); err != nil || live != version {
222 return errors.Join(release.ErrHeldNotLive, err)
223 }
224 return lock(func() error {
225 if f.lockProbe != nil {
226 f.lockProbe("verify")
227 }
228 if live, err := release.LiveVersion(optDir, rt.Host); err != nil || live != version {
229 return errors.Join(release.ErrHeldNotLive, err)
230 }
231 return f.verifyErr
232 })
233}
234
235func (f *installFixture) publishLive(acct, app, version string) error {
236 // The box's version store already holds the release (bootstrap staged
237 // it); a fresh store gets a directory for it.
238 target := release.VersionPath(f.r.OptDir, acct, version, app)
239 if _, err := os.Lstat(target); errors.Is(err, os.ErrNotExist) {
240 if err := os.MkdirAll(target, 0o755); err != nil {
241 return err
242 }
243 }
244 link := release.LiveLink(f.r.OptDir, buildHost)
245 if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil {
246 return err
247 }
248 _ = os.Remove(link)
249 return os.Symlink(target, link)
250}
251
252func (f *installFixture) install() (hostclaim.LocalInstall, error) {
253 return f.a.relocationInstall(f.cfg)(f.t.Context(), f.read())
254}
255
256func (f *installFixture) read() hostclaim.Observed {
257 f.t.Helper()
258 o, err := hostclaim.Read(f.t.Context(), f.claims, buildHost)
259 if err != nil {
260 f.t.Fatal(err)
261 }
262 return o
263}
264
265func (f *installFixture) paths() relocationPaths {
266 return relocationPathsFor(f.r.DataDir, registry.Route{Host: buildHost, DBPath: provision.NestedDBPath(f.r.DataDir, buildHost), Socket: provision.NestedSocket(buildHost)})
267}
268
269func (f *installFixture) effects() *RelocationEffects {
270 return &RelocationEffects{Claims: f.claims, Install: f.a.relocationInstall(f.cfg), VerifyLocal: f.a.relocationVerifyLocal(f.cfg), Receipt: f.a.relocationReceipt(f.cfg)}
271}
272
273// Findings 3 and 5, Review Focus 2: from an empty target, the install mints
274// the held row first (the census initialises), makes the instance
275// directory through main's PrepareState, restores files and env into the
276// nested paths, imports the wake state (timers owned by the row), renders
277// the held config and stages the pinned release, with the flags on.
278func TestRelocationInstallFromAnEmptyTargetThroughTheProductionCensus(t *testing.T) {
279 f := newInstallFixture(t, nil)
280 ctx := t.Context()
281 l, err := f.install()
282 if err != nil {
283 t.Fatal(err)
284 }
285 stamp := f.obs.Claim.Holder.Stamp
286 row, err := f.reg.Get(buildHost)
287 if err != nil || row == nil {
288 t.Fatalf("row: %v", err)
289 }
290 if row.Status != "relocating" || row.HoldOp != "rel-1" || row.DBPath != provision.NestedDBPath(f.r.DataDir, buildHost) || row.ClaimOwnership.ClaimStamp != stamp {
291 t.Fatalf("row %+v", row)
292 }
293 timers, err := f.reg.TimersFor(buildHost)
294 if err != nil || len(timers) != 2 {
295 t.Fatalf("timers %+v %v", timers, err)
296 }
297 for _, tm := range timers {
298 if tm.ClaimOwner != row.ClaimOwnership.ClaimOwner {
299 t.Fatalf("timer %s is not owned by the row: %+v", tm.Name, tm)
300 }
301 }
302 p := f.paths()
303 for rel, want := range map[string]string{"public/index.html": "<h1>shop</h1>", "uploads/a/b.png": "png-bytes"} {
304 got, err := os.ReadFile(filepath.Join(p.FilesDir, filepath.FromSlash(rel)))
305 if err != nil || string(got) != want {
306 t.Fatalf("%s: %q %v", rel, got, err)
307 }
308 }
309 if env, err := os.ReadFile(p.EnvPath); err != nil || string(env) != installEnv {
310 t.Fatalf("env %q %v", env, err)
311 }
312 if live, err := release.LiveVersion(f.r.OptDir, buildHost); err != nil || live != "v1" {
313 t.Fatalf("live %q %v", live, err)
314 }
315 path, _ := relocationInstallPath(f.r.DataDir, "rel-1")
316 raw, err := os.ReadFile(path)
317 if err != nil {
318 t.Fatal(err)
319 }
320 sum := sha256.Sum256(raw)
321 if l.Box != "dst" || l.Enrollment != f.actor.Enrollment || l.InstallStamp != stamp || l.JournalSHA256 != hex.EncodeToString(sum[:]) || l.FilesManifestVersion != 0 {
322 t.Fatalf("receipt %+v", l)
323 }
324 again, err := f.install()
325 if err != nil || again != l {
326 t.Fatalf("a second install: %+v %v", again, err)
327 }
328 if ok, err := f.a.relocationVerifyLocal(f.cfg)(ctx, f.read(), l); err != nil || !ok {
329 t.Fatalf("verify: %v %v", ok, err)
330 }
331}
332
333// Rev 3 finding 3, ruling 57: the instance directory is main's typed path
334// entry, intended then materialized with its device and inode, so the
335// probe's own PrepareState accepts it as its own.
336func TestRelocationInstallMaterializesTheInstanceDirectoryTyped(t *testing.T) {
337 f := newInstallFixture(t, nil)
338 if _, err := f.install(); err != nil {
339 t.Fatal(err)
340 }
341 h := f.read().Claim.Holder
342 in, err := f.inv.ReadInstallation(t.Context(), buildHost, h.Incarnation, h.Stamp)
343 if err != nil {
344 t.Fatal(err)
345 }
346 dir := provision.InstanceDir(f.r.DataDir, buildHost)
347 rel, _ := filepath.Rel(f.r.DataDir, dir)
348 info, err := os.Stat(dir)
349 if err != nil {
350 t.Fatal(err)
351 }
352 found := false
353 for _, e := range in.Entries {
354 if e.Kind == "path" && e.ManagedRoot == "data" && e.RelativePath == rel {
355 found = e.Materialization == "materialized" && e.Inode == inodeOf(t, info)
356 }
357 }
358 if !found {
359 t.Fatalf("no materialized path entry for %s: %+v", rel, in.Entries)
360 }
361 // The probe's own PrepareState finds it as its own.
362 work, err := relocationInstallSource(f.cfg.Source, f.reg).SelectAt(t.Context(), buildHost, relocationActionInstall, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp})
363 if err != nil {
364 t.Fatal(err)
365 }
366 if err := f.a.Wake.Roles.PrepareState(t.Context(), f.read(), work, ClaimLaunchInputs{DBPath: provision.NestedDBPath(f.r.DataDir, buildHost)}); err != nil {
367 t.Fatalf("PrepareState refused the install's directory: %v", err)
368 }
369}
370
371// O3: the env file is P's EnvItem byte for byte, its comments, spacing and
372// unterminated last line included; nothing is minted.
373func TestRelocationInstallKeepsThePublishedEnvByteForByte(t *testing.T) {
374 f := newInstallFixture(t, nil)
375 if _, err := f.install(); err != nil {
376 t.Fatal(err)
377 }
378 p := f.paths()
379 if env, err := os.ReadFile(p.EnvPath); err != nil || string(env) != installEnv {
380 t.Fatalf("env %q %v", env, err)
381 }
382 if _, err := os.Stat(filepath.Join(p.FilesDir, hostrelocate.InstanceKeyPath)); !errors.Is(err, os.ErrNotExist) {
383 t.Fatalf("an instance key was minted: %v", err)
384 }
385}
386
387// Astra round 3 finding 5 and ruling 68, amended in Task 8c (absence is
388// never evidence): no code writes a platform tombstone, so a source whose
389// env file was deleted publishes a manifest without the item, exactly as
390// one whose env file was never published does. Such a manifest, under a
391// pin of published files, refuses the install before anything is written:
392// no row, no files, no env file, no journal, and a stray env file at the
393// path is left as it was.
394func TestRelocationInstallRefusesAManifestWithoutTheEnvItem(t *testing.T) {
395 f := newInstallFixture(t, map[string]string{hostfiles.WakeItem: installWake()})
396 before := f.read().Claim.Gen
397 p := f.paths()
398 if err := os.MkdirAll(filepath.Dir(p.EnvPath), 0o700); err != nil {
399 t.Fatal(err)
400 }
401 if err := os.WriteFile(p.EnvPath, []byte("STALE=1\n"), 0o600); err != nil {
402 t.Fatal(err)
403 }
404 if _, err := f.install(); !errors.Is(err, hostrelocate.ErrRefused) {
405 t.Fatalf("a manifest without the env item: %v", err)
406 }
407 if row, err := f.reg.Get(buildHost); err != nil || row != nil {
408 t.Fatalf("a row was minted: %+v %v", row, err)
409 }
410 if _, err := os.Stat(filepath.Join(p.FilesDir, "public", "index.html")); !errors.Is(err, os.ErrNotExist) {
411 t.Fatalf("files were restored: %v", err)
412 }
413 if env, err := os.ReadFile(p.EnvPath); err != nil || string(env) != "STALE=1\n" {
414 t.Fatalf("the env path was touched: %q %v", env, err)
415 }
416 if after := f.read().Claim.Gen; after != before {
417 t.Fatalf("the claim was written: gen %d -> %d", before, after)
418 }
419 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
420 t.Fatal("a refused install left a receipt")
421 }
422}
423
424// The wake half of the same rule: a manifest without wake state (a
425// failover's, which no release flush wrote) is not a host without timers,
426// and refuses the install.
427func TestRelocationInstallRefusesAManifestWithoutWakeState(t *testing.T) {
428 f := newInstallFixture(t, map[string]string{hostfiles.EnvItem: installEnv})
429 if _, err := f.install(); !errors.Is(err, hostrelocate.ErrRefused) {
430 t.Fatalf("a manifest without wake state: %v", err)
431 }
432 if row, err := f.reg.Get(buildHost); err != nil || row != nil {
433 t.Fatalf("a row was minted: %+v %v", row, err)
434 }
435}
436
437// Ruling 68's positive evidence: a pin of a host that never published
438// files (no digest, no retained copy) is no env file and no timers. The
439// install writes no env file and imports no wake state, and a repeated
440// install (a crash before its journal) removes a stray env file at the
441// path. VerifyLocal reads the same evidence: whole with no env file, not
442// whole once one appears.
443func TestRelocationInstallRemovesAStaleEnvWhenThePinEvidencesNoFiles(t *testing.T) {
444 f := newInstallFixtureWith(t, buildOptions{NoFiles: true})
445 ctx := t.Context()
446 l, err := f.install()
447 if err != nil {
448 t.Fatal(err)
449 }
450 p := f.paths()
451 if _, err := os.Stat(p.EnvPath); !errors.Is(err, os.ErrNotExist) {
452 t.Fatalf("an env file was written for a host with none: %v", err)
453 }
454 if timers, err := f.reg.TimersFor(buildHost); err != nil || len(timers) != 0 {
455 t.Fatalf("timers %+v %v", timers, err)
456 }
457 verify := f.a.relocationVerifyLocal(f.cfg)
458 if ok, err := verify(ctx, f.read(), l); err != nil || !ok {
459 t.Fatalf("verify with no env file: %v %v", ok, err)
460 }
461 if err := os.WriteFile(p.EnvPath, []byte("STALE=1\n"), 0o600); err != nil {
462 t.Fatal(err)
463 }
464 if ok, err := verify(ctx, f.read(), l); err != nil || ok {
465 t.Fatalf("verify with a stray env file: %v %v", ok, err)
466 }
467 // A crash before the journal, and the env file at the path meanwhile.
468 journal, _ := relocationInstallPath(f.r.DataDir, "rel-1")
469 if err := os.Remove(journal); err != nil {
470 t.Fatal(err)
471 }
472 if _, err := f.install(); err != nil {
473 t.Fatal(err)
474 }
475 if _, err := os.Stat(p.EnvPath); !errors.Is(err, os.ErrNotExist) {
476 t.Fatalf("a stale env survived: %v", err)
477 }
478}
479
480// The O3' check reads the env file the install wrote: one gone by the time
481// the key is checked is not an env with no key of its own.
482func TestRelocationInstallKeyCheckNeedsTheEnvItWrote(t *testing.T) {
483 const configured = "configured-key"
484 f := newInstallFixture(t, map[string]string{hostfiles.EnvItem: "CARLOS_ADMIN_TOKEN=t\n", hostfiles.WakeItem: installWake()})
485 f.configs = map[string]string{"CARLOS_INSTANCE_KEY": configured}
486 repinWithKey(t, f.buildFixture, &hostrelocate.KeyPin{Source: instancekey.SourceConfig, SHA256: instancekey.Digest(configured)})
487 f.cfg.render = func(ctx context.Context, s release.Store, k *ecdh.PrivateKey, d string, rt registry.Route, op, box string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
488 if err := os.Remove(f.paths().EnvPath); err != nil {
489 return appconfig.HeldRender{}, err
490 }
491 return f.render(ctx, s, k, d, rt, op, box, work)
492 }
493 if _, err := f.install(); !errors.Is(err, hostrelocate.ErrRefused) || !errors.Is(err, os.ErrNotExist) {
494 t.Fatalf("the key check without the env it wrote: %v", err)
495 }
496 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
497 t.Fatal("a receipt without the env file")
498 }
499}
500
501// Ruling 44 and K25: nothing region HA leaves under the data root names the
502// host outside the host's own installation, except main's own shared paths
503// and moves' held-stage relay (census-opaque, N11). The build's scratch
504// left by a crash is removed by the install.
505func TestNoRelocationFileNamesTheHostOutsideItsInstallation(t *testing.T) {
506 f := newInstallFixture(t, nil)
507 scratch := filepath.Join(f.cfg.WorkDir, "rel-1", buildHost+".db")
508 if err := os.MkdirAll(filepath.Dir(scratch), 0o700); err != nil {
509 t.Fatal(err)
510 }
511 if err := os.WriteFile(scratch, []byte("left by a crash"), 0o600); err != nil {
512 t.Fatal(err)
513 }
514 f.stageErr = release.ErrStageBlocked
515 if _, err := f.install(); !errors.Is(err, hostrelocate.ErrNotYet) {
516 t.Fatalf("blocked stage: %v", err)
517 }
518 f.stageErr = nil
519 if _, err := f.install(); err != nil {
520 t.Fatal(err)
521 }
522 inst := provision.InstanceDir(f.r.DataDir, buildHost)
523 allowed := []string{inst, filepath.Join(f.r.DataDir, "hostguard"), filepath.Join(f.r.DataDir, ".hostexec"), filepath.Join(f.r.DataDir, ".hostclaims"),
524 release.HeldStageDir(f.r.DataDir), release.HeldStagedDir(f.r.DataDir), filepath.Join(f.r.DataDir, ".moves", heldStageLockFile(buildHost))}
525 err := filepath.WalkDir(f.r.DataDir, func(path string, d fs.DirEntry, err error) error {
526 if err != nil || !strings.Contains(path, buildHost) {
527 return err
528 }
529 for _, a := range allowed {
530 if path == a || strings.HasPrefix(path, a+string(filepath.Separator)) {
531 return nil
532 }
533 }
534 t.Errorf("%s names the host outside its installation", path)
535 return nil
536 })
537 if err != nil {
538 t.Fatal(err)
539 }
540 ops, err := os.ReadDir(filepath.Join(f.r.DataDir, "relocate", "ops", "rel-1"))
541 if err != nil || len(ops) != 1 || ops[0].Name() != "install.json" {
542 t.Fatalf("relocate/ops/rel-1: %v %v", ops, err)
543 }
544 if _, err := os.Stat(scratch); !errors.Is(err, os.ErrNotExist) {
545 t.Fatalf("the build's scratch survived the install: %v", err)
546 }
547}
548
549// Finding 5, M3: MaterializeHeld is never called; a blocked stage leaves
550// root's relay request at install_release (never the route's version) and
551// the receipt waits for live/<host>.
552func TestRelocationInstallStagesThroughTheClaimPathWithClaimsOn(t *testing.T) {
553 f := newInstallFixture(t, nil)
554 f.stageErr = release.ErrStageBlocked
555 if _, err := f.install(); !errors.Is(err, hostrelocate.ErrNotYet) {
556 t.Fatalf("got %v", err)
557 }
558 raw, err := os.ReadFile(filepath.Join(release.HeldStageDir(f.r.DataDir), buildHost+".json"))
559 if err != nil {
560 t.Fatal(err)
561 }
562 var req release.HeldStageRequest
563 if err := json.Unmarshal(raw, &req); err != nil || req.Version != "v1" || req.Op != "rel-1" || req.Box != "dst" || req.Host != buildHost || req.App != "app" {
564 t.Fatalf("request %+v %v", req, err)
565 }
566 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
567 t.Fatal("a receipt before the release is live")
568 }
569 // Root's pass serves the request.
570 if err := f.publishLive("acct", "app", "v1"); err != nil {
571 t.Fatal(err)
572 }
573 if l, err := f.install(); err != nil || l.InstallStamp == 0 {
574 t.Fatalf("after the stage: %+v %v", l, err)
575 }
576}
577
578// Finding 10: every install write runs main's Local first. With the
579// execution lease lost after the install work and before the render, the
580// held-config selection refuses: no config, no request, no journal, and
581// the claim is not written.
582func TestRelocationInstallStopsWhenTheLeaseIsLost(t *testing.T) {
583 f := newInstallFixture(t, nil)
584 f.cfg.render = func(ctx context.Context, s release.Store, k *ecdh.PrivateKey, d string, rt registry.Route, op, box string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
585 f.localErr = execlease.ErrLost
586 return f.render(ctx, s, k, d, rt, op, box, work)
587 }
588 before := f.read()
589 if _, err := f.install(); !errors.Is(err, execlease.ErrLost) {
590 t.Fatalf("got %v", err)
591 }
592 if _, err := os.Stat(filepath.Join(provision.InstanceDir(f.r.DataDir, buildHost), "config.env")); !errors.Is(err, os.ErrNotExist) {
593 t.Fatalf("a config was written: %v", err)
594 }
595 if _, err := os.Stat(filepath.Join(release.HeldStageDir(f.r.DataDir), buildHost+".json")); !errors.Is(err, os.ErrNotExist) {
596 t.Fatalf("a stage request was written: %v", err)
597 }
598 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
599 t.Fatal("a journal was written")
600 }
601 // park's RecordInventory wrote the claim's inventory once, before; the
602 // lost lease wrote nothing more.
603 after := f.read()
604 if after.Claim.Gen != before.Claim.Gen+1 || after.Claim.Inventory["dst"].Layout != "nested" {
605 t.Fatalf("the claim moved: gen %d -> %d", before.Claim.Gen, after.Claim.Gen)
606 }
607}
608
609// O3' (instance-key plan rev 3): when the pin carries the key by an
610// adoption record, a configuration that no longer delivers that key
611// refuses the install before anything launches.
612func TestRelocationInstallRefusesAKeyChangedSinceThePin(t *testing.T) {
613 const configured = "configured-key"
614 for _, tc := range []struct {
615 name string
616 config string
617 want error
618 }{
619 {"the configured key the pin names", configured, nil},
620 {"a key changed since the pin", "another-key", hostrelocate.ErrRefused},
621 } {
622 t.Run(tc.name, func(t *testing.T) {
623 f := newInstallFixture(t, map[string]string{hostfiles.EnvItem: "CARLOS_ADMIN_TOKEN=t\n", hostfiles.WakeItem: installWake()})
624 f.configs = map[string]string{"CARLOS_INSTANCE_KEY": tc.config}
625 repinWithKey(t, f.buildFixture, &hostrelocate.KeyPin{Source: instancekey.SourceConfig, SHA256: instancekey.Digest(configured)})
626 _, err := f.install()
627 if !errors.Is(err, tc.want) {
628 t.Fatalf("got %v, want %v", err, tc.want)
629 }
630 if tc.want != nil {
631 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
632 t.Fatal("a refused key left a receipt")
633 }
634 }
635 })
636 }
637}
638
639// repinWithKey rewrites the fixture's pin object with an instance key pin
640// and journals its new digest, as the pinner would have.
641func repinWithKey(t *testing.T, f *buildFixture, key *hostrelocate.KeyPin) {
642 t.Helper()
643 ctx := t.Context()
644 o, err := hostclaim.Read(ctx, f.claims, buildHost)
645 if err != nil {
646 t.Fatal(err)
647 }
648 var ref hostclaim.RecoveryPin
649 if err := json.Unmarshal([]byte(o.Claim.Op.Journal["pin"]), &ref); err != nil {
650 t.Fatal(err)
651 }
652 pin, err := hostrelocate.ReadPin(ctx, f.claims, ref)
653 if err != nil {
654 t.Fatal(err)
655 }
656 pin.InstanceKey = key
657 raw, _ := json.Marshal(pin)
658 ref.Key += ".keyed"
659 if _, err := f.claims.Put(ctx, ref.Key, raw, ""); err != nil {
660 t.Fatal(err)
661 }
662 sum := sha256.Sum256(raw)
663 ref.SHA256 = hex.EncodeToString(sum[:])
664 refRaw, _ := json.Marshal(ref)
665 if f.obs, err = hostclaim.Update(ctx, f.claims, o, func(c *hostclaim.Claim) error { c.Op.Journal["pin"] = string(refRaw); return nil }); err != nil {
666 t.Fatal(err)
667 }
668}
669
670// Ruling 52 and finding 13: a damaged install is repaired under its active
671// stamp, in installed and in probing before P; a retired stamp refuses and
672// is never reactivated.
673func TestRelocationEnsureLocalRefusesARetiredStamp(t *testing.T) {
674 f := newInstallFixture(t, nil)
675 ctx := t.Context()
676 l, err := f.install()
677 if err != nil {
678 t.Fatal(err)
679 }
680 receipt, _ := json.Marshal(l)
681 if f.obs, err = hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Journal["local_install"] = string(receipt); return nil }); err != nil {
682 t.Fatal(err)
683 }
684 e := f.effects()
685 file := filepath.Join(f.paths().FilesDir, "public", "index.html")
686 for _, phase := range []string{hostclaim.PhaseInstalled, hostclaim.PhaseRelocatingProbing} {
687 o, err := hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Phase = phase; return nil })
688 if err != nil {
689 t.Fatal(err)
690 }
691 if err := os.Remove(file); err != nil {
692 t.Fatal(err)
693 }
694 if err := e.EnsureLocal(ctx, o); err != nil {
695 t.Fatalf("%s: %v", phase, err)
696 }
697 if got, err := os.ReadFile(file); err != nil || string(got) != "<h1>shop</h1>" {
698 t.Fatalf("%s: not restored: %q %v", phase, got, err)
699 }
700 }
701 if err := f.a.Guard.RetireInstallation(ctx, buildHost, func(*hostclaim.Claim) error { return nil }); err != nil {
702 t.Fatal(err)
703 }
704 if err := os.Remove(file); err != nil {
705 t.Fatal(err)
706 }
707 if err := e.EnsureLocal(ctx, f.read()); !errors.Is(err, hostrelocate.ErrRefused) {
708 t.Fatalf("a retired stamp was repaired: %v", err)
709 }
710 if _, err := os.Stat(file); !errors.Is(err, os.ErrNotExist) {
711 t.Fatalf("a retired stamp's install was written: %v", err)
712 }
713 if c, err := hostguard.Capture(f.a.Guard.DataDir, buildHost); err == nil && c.Stamp == l.InstallStamp {
714 t.Fatalf("the retired stamp was reactivated: %+v", c)
715 }
716}
717
718// Stage B final review A3 (Astra 3): a repair after the held configuration
719// changed re-renders it, so the repaired journal's digest is not the
720// journaled receipt's. EnsureLocal replaces the receipt in one CAS on a
721// fresh read before any admission, so the pre-admission verification, the
722// receipt check after P opens (VerifyReceipt) and the post-admission check
723// (relocationAfterAdmission) all compare the digest the box holds with the
724// one the claim journals. A CAS that fails once leaves the old receipt, and
725// the next pass repairs to the same journal and records it.
726func TestRelocationEnsureLocalRepairsAcrossAConfigurationChange(t *testing.T) {
727 f := newInstallFixture(t, nil)
728 ctx := t.Context()
729 l, err := f.install()
730 if err != nil {
731 t.Fatal(err)
732 }
733 receipt, _ := json.Marshal(l)
734 if f.obs, err = hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error {
735 c.Op.Phase = hostclaim.PhaseRelocatingProbing
736 c.Op.Journal["local_install"] = string(receipt)
737 return nil
738 }); err != nil {
739 t.Fatal(err)
740 }
741 e := f.effects()
742 file := filepath.Join(f.paths().FilesDir, "public", "index.html")
743 journaled := func() hostclaim.LocalInstall {
744 t.Helper()
745 var got hostclaim.LocalInstall
746 if err := json.Unmarshal([]byte(f.read().Claim.Op.Journal["local_install"]), &got); err != nil {
747 t.Fatal(err)
748 }
749 return got
750 }
751 onDisk := func() hostclaim.LocalInstall {
752 t.Helper()
753 got, ok, err := f.a.relocationReceipt(f.cfg)(ctx, f.read())
754 if err != nil || !ok {
755 t.Fatalf("the durable receipt: %v %v", ok, err)
756 }
757 return got
758 }
759 agree := func(want string) {
760 t.Helper()
761 if j, d := journaled(), onDisk(); j != d {
762 t.Fatalf("%s: the claim journals %+v, the box holds %+v", want, j, d)
763 }
764 if ok, err := e.VerifyReceipt(ctx, f.read()); err != nil || !ok {
765 t.Fatalf("%s: VerifyReceipt %v %v", want, ok, err)
766 }
767 prior := f.read()
768 admitted := f.read()
769 admitted.Claim.Holder.WriterBox, admitted.Claim.Holder.Generation = "dst", admitted.Claim.Op.Journal["prepared"]
770 if err := relocationAfterAdmission(f.r.DataDir, "dst")(ctx, prior, admitted); err != nil {
771 t.Fatalf("%s: after admission: %v", want, err)
772 }
773 }
774 agree("after the install")
775
776 // The configuration changes, then a file is lost: the repair renders
777 // the new configuration and records the new receipt.
778 f.configs = map[string]string{"CARLOS_FEATURE": "2"}
779 if err := os.Remove(file); err != nil {
780 t.Fatal(err)
781 }
782 before := f.read().Claim.Gen
783 if err := e.EnsureLocal(ctx, f.read()); err != nil {
784 t.Fatalf("the repair across a configuration change: %v", err)
785 }
786 if got, err := os.ReadFile(file); err != nil || string(got) != "<h1>shop</h1>" {
787 t.Fatalf("not restored: %q %v", got, err)
788 }
789 repaired := journaled()
790 if repaired == l || repaired.Box != l.Box || repaired.Enrollment != l.Enrollment || repaired.InstallStamp != l.InstallStamp || f.read().Claim.Gen == before {
791 t.Fatalf("the receipt was not replaced for the same installation: %+v (was %+v)", repaired, l)
792 }
793 agree("after the repair")
794 // Whole now: nothing is reinstalled or rewritten.
795 gen := f.read().Claim.Gen
796 if err := e.EnsureLocal(ctx, f.read()); err != nil || f.read().Claim.Gen != gen {
797 t.Fatalf("a whole install was repaired again: %v gen %d -> %d", err, gen, f.read().Claim.Gen)
798 }
799
800 // The configuration changes again and the CAS fails once: the old
801 // receipt stays in the claim, the box holds the new journal, and the
802 // next pass repairs to the same journal and records it.
803 f.configs = map[string]string{"CARLOS_FEATURE": "3"}
804 if err := os.Remove(file); err != nil {
805 t.Fatal(err)
806 }
807 e.Claims = errPutStore{Store: f.claims}
808 if err := e.EnsureLocal(ctx, f.read()); err == nil {
809 t.Fatal("a failed receipt CAS was reported as a repair")
810 }
811 if j, d := journaled(), onDisk(); j != repaired || d == repaired {
812 t.Fatalf("after the failed CAS: the claim journals %+v, the box holds %+v", j, d)
813 }
814 if ok, err := e.VerifyReceipt(ctx, f.read()); err != nil || ok {
815 t.Fatalf("the old receipt verified against the new journal: %v %v", ok, err)
816 }
817 e.Claims = f.claims
818 if err := e.EnsureLocal(ctx, f.read()); err != nil {
819 t.Fatalf("the pass after the failed CAS: %v", err)
820 }
821 if journaled() == repaired {
822 t.Fatal("the receipt was not replaced on the next pass")
823 }
824 agree("after the retried repair")
825}
826
827func TestRelocationVerifyLocalReadsTheInstallBack(t *testing.T) {
828 f := newInstallFixture(t, nil)
829 ctx := t.Context()
830 l, err := f.install()
831 if err != nil {
832 t.Fatal(err)
833 }
834 verify := f.a.relocationVerifyLocal(f.cfg)
835 if ok, err := verify(ctx, f.read(), l); err != nil || !ok {
836 t.Fatalf("after the install: %v %v", ok, err)
837 }
838 other := l
839 other.InstallStamp++
840 if ok, err := verify(ctx, f.read(), other); err != nil || ok {
841 t.Fatalf("another receipt verified: %v %v", ok, err)
842 }
843 if err := os.Remove(filepath.Join(f.paths().FilesDir, "uploads", "a", "b.png")); err != nil {
844 t.Fatal(err)
845 }
846 if ok, err := verify(ctx, f.read(), l); err != nil || ok {
847 t.Fatalf("a missing file verified: %v %v", ok, err)
848 }
849 // Final review minor 10: P's manifest is read as the install reads it
850 // (hostfiles.Load); one that is gone is a refusal, not an install to
851 // repair (the install refuses without it too).
852 o := f.read()
853 if err := f.cfg.Data.Delete(ctx, hostfiles.ManifestKey(moves.GenerationFilesPrefix(o.Claim.Holder.Location, o.Claim.Op.Journal["prepared"]))); err != nil {
854 t.Fatal(err)
855 }
856 if ok, err := verify(ctx, o, l); ok || !errors.Is(err, hostrelocate.ErrRefused) {
857 t.Fatalf("a missing manifest: %v %v", ok, err)
858 }
859}
860
861func TestRelocationReceiptIsTheDurableJournal(t *testing.T) {
862 f := newInstallFixture(t, nil)
863 ctx := t.Context()
864 receipt := f.a.relocationReceipt(f.cfg)
865 if _, ok, err := receipt(ctx, f.read()); ok || err != nil {
866 t.Fatalf("a receipt before the install: %v %v", ok, err)
867 }
868 l, err := f.install()
869 if err != nil {
870 t.Fatal(err)
871 }
872 if got, ok, err := receipt(ctx, f.read()); err != nil || !ok || got != l {
873 t.Fatalf("receipt %+v %v %v", got, ok, err)
874 }
875 // Final review minor 1: a journal that is present but cannot be read,
876 // or is not this operation's completed install, is an error, never
877 // "no receipt" that EnsureLocal would reinstall over.
878 journaled, _ := json.Marshal(l)
879 if _, err := hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op.Journal["local_install"] = string(journaled); return nil }); err != nil {
880 t.Fatal(err)
881 }
882 path, _ := relocationInstallPath(f.r.DataDir, "rel-1")
883 for name, body := range map[string]string{
884 "a damaged journal": `{"v":1,"host":"`,
885 "another operation's": `{"v":1,"host":"` + buildHost + `","op":"rel-2","steps":{"materialize":"v1"}}`,
886 "another host's": `{"v":1,"host":"other.example","op":"rel-1","steps":{"materialize":"v1"}}`,
887 "no materialized release": `{"v":1,"host":"` + buildHost + `","op":"rel-1","steps":{}}`,
888 } {
889 if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
890 t.Fatal(err)
891 }
892 if _, ok, err := receipt(ctx, f.read()); ok || !errors.Is(err, hostrelocate.ErrRefused) {
893 t.Fatalf("%s: %v %v", name, ok, err)
894 }
895 if err := f.effects().EnsureLocal(ctx, f.read()); !errors.Is(err, hostrelocate.ErrRefused) {
896 t.Fatalf("%s: EnsureLocal reinstalled over it: %v", name, err)
897 }
898 }
899 if err := os.Remove(path); err != nil {
900 t.Fatal(err)
901 }
902 if _, ok, err := receipt(ctx, f.read()); ok || err != nil {
903 t.Fatalf("no journal: %v %v", ok, err)
904 }
905}
906
907// Rev 3 finding 2, ruling 62: Participant.Step over the real effects
908// records local_install on the first pass (the install wrote the claim's
909// inventory, so the receipt is recorded on a fresh read), then enters
910// probing on the next, with no conflict loop.
911func TestRelocationInstallRecordsItsReceiptThroughParticipantStep(t *testing.T) {
912 f := newInstallFixture(t, nil)
913 e := f.effects()
914 e.Participation = func(context.Context, hostclaim.Observed) (execlease.State, error) {
915 return execlease.State{Binding: execlease.Binding{Host: buildHost, Incarnation: f.obs.Claim.Holder.Incarnation, Op: "rel-1"}}, nil
916 }
917 p := hostrelocate.Participant{Claims: f.claims, Box: "dst", Enrollment: f.actor.Enrollment, Effects: e, Now: time.Now}
918 o, err := p.Step(t.Context(), buildHost)
919 if err != nil || o.Claim.Op.Journal["local_install"] == "" || o.Claim.Op.Phase != hostclaim.PhaseInstalled {
920 t.Fatalf("first pass: %v %+v", err, o.Claim.Op.Journal)
921 }
922 o, err = p.Step(t.Context(), buildHost)
923 if err != nil {
924 t.Fatalf("second pass: %v", err)
925 }
926 if o.Claim.Op.Phase != hostclaim.PhaseRelocatingProbing {
927 t.Fatalf("second pass: %s", o.Claim.Op.Phase)
928 }
929}
930
931func inodeOf(t *testing.T, info os.FileInfo) uint64 {
932 t.Helper()
933 st, ok := info.Sys().(*syscall.Stat_t)
934 if !ok {
935 t.Fatal("no stat")
936 }
937 return st.Ino
938}
939
940// Fix round 1, M1: every precondition is checked before anything is
941// written. An assembly without main's role assembly (the instance
942// directory's typed protocol) refuses with no row and no claim write.
943func TestRelocationInstallRefusesAMisconfiguredAssemblyBeforeAnyWrite(t *testing.T) {
944 f := newInstallFixture(t, nil)
945 before := f.read().Claim.Gen
946 f.a.Wake = nil
947 if _, err := f.install(); !errors.Is(err, ErrClaimStopUnavailable) {
948 t.Fatalf("no role assembly: %v", err)
949 }
950 if row, err := f.reg.Get(buildHost); err != nil || row != nil {
951 t.Fatalf("a row was minted: %+v %v", row, err)
952 }
953 if after := f.read().Claim.Gen; after != before {
954 t.Fatalf("the claim was written: gen %d -> %d", before, after)
955 }
956}
957
958// Fix round 1, I1: the held release is published under main's held-stage
959// lock, and read back under the same lock before the receipt, as moves'
960// T7 does (heldReleaseReady). Another holder of the lock is refused while
961// each runs.
962func TestRelocationInstallPublishesAndReadsUnderTheHeldStageLock(t *testing.T) {
963 prev := heldLockWait
964 t.Cleanup(func() { heldLockWait = prev })
965 heldLockWait = 50 * time.Millisecond
966 f := newInstallFixture(t, nil)
967 held := map[string]bool{}
968 f.lockProbe = func(where string) {
969 err := withHeldStageLock(f.r.DataDir, buildHost, func() error { return nil })
970 held[where] = errors.Is(err, errHeldLockBusy)
971 }
972 if _, err := f.install(); err != nil {
973 t.Fatal(err)
974 }
975 if f.published != 1 || !held["publish"] || !held["verify"] {
976 t.Fatalf("published %d, under the lock %v", f.published, held)
977 }
978}
979
980// Fix round 1, I1: a link alone is never evidence. A link the reading does
981// not prove (the bytes it names, or the policy, refuse) leaves no receipt,
982// whether this install published it or root's relay did; once proven, the
983// install completes without publishing again.
984func TestRelocationInstallWritesNoReceiptOnAnUnprovenLink(t *testing.T) {
985 for _, byRoot := range []bool{false, true} {
986 f := newInstallFixture(t, nil)
987 if byRoot {
988 // The edge cannot write the version store: the install leaves
989 // root's request, and root's relay publishes the link.
990 f.stageErr = release.ErrStageBlocked
991 if _, err := f.install(); !errors.Is(err, hostrelocate.ErrNotYet) {
992 t.Fatalf("blocked: %v", err)
993 }
994 f.stageErr = nil
995 if err := f.publishLive("acct", "app", "v1"); err != nil {
996 t.Fatal(err)
997 }
998 }
999 f.verifyErr = release.ErrHeldNotProven
1000 if _, err := f.install(); !errors.Is(err, release.ErrHeldNotProven) {
1001 t.Fatalf("root %v: an unproven link: %v", byRoot, err)
1002 }
1003 if live, err := release.LiveVersion(f.r.OptDir, buildHost); err != nil || live != "v1" {
1004 t.Fatalf("root %v: live %q %v", byRoot, live, err)
1005 }
1006 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
1007 t.Fatalf("root %v: a receipt on an unproven link", byRoot)
1008 }
1009 f.verifyErr = nil
1010 if _, err := f.install(); err != nil {
1011 t.Fatalf("root %v: proven: %v", byRoot, err)
1012 }
1013 if want := map[bool]int{false: 1, true: 0}[byRoot]; f.published != want {
1014 t.Fatalf("root %v: published %d times, want %d", byRoot, f.published, want)
1015 }
1016 }
1017}
1018
1019// Fix round 1, I3: each of the install's write groups runs inside a With
1020// of its selection, so main's Local (the execution lease's deadline and
1021// loss latch) stops it. The lease is lost at one group at a time; that
1022// group and everything after it write nothing. (The held config's group
1023// is TestRelocationInstallStopsWhenTheLeaseIsLost.)
1024func TestRelocationInstallStopsAtEachWriteGroupWhenTheLeaseIsLost(t *testing.T) {
1025 materializedDir := func(f *installFixture) bool {
1026 h := f.read().Claim.Holder
1027 in, err := f.inv.ReadInstallation(context.Background(), buildHost, h.Incarnation, h.Stamp)
1028 if err != nil {
1029 return false
1030 }
1031 rel, _ := filepath.Rel(f.r.DataDir, provision.InstanceDir(f.r.DataDir, buildHost))
1032 for _, e := range in.Entries {
1033 if e.Kind == "path" && e.ManagedRoot == "data" && e.RelativePath == rel && e.Materialization == "materialized" {
1034 return true
1035 }
1036 }
1037 return false
1038 }
1039 live := func(f *installFixture) bool {
1040 v, err := release.LiveVersion(f.r.OptDir, buildHost)
1041 return err == nil && v == "v1"
1042 }
1043 for _, tc := range []struct {
1044 name string
1045 lose func(*installFixture, hostbox.Selection) bool
1046 row bool // the row was minted before the loss
1047 file bool // the files were restored before the loss
1048 link bool // the release was published before the loss
1049 }{
1050 {"the row", func(*installFixture, hostbox.Selection) bool { return true }, false, false, false},
1051 {"the files, env and wake state", func(f *installFixture, s hostbox.Selection) bool {
1052 return s.Action == relocationActionInstall && materializedDir(f)
1053 }, true, false, false},
1054 {"the held release", func(_ *installFixture, s hostbox.Selection) bool { return s.Action == relocationActionHeldRelease }, true, true, false},
1055 {"the journal", func(f *installFixture, s hostbox.Selection) bool {
1056 return s.Action == relocationActionInstall && live(f)
1057 }, true, true, true},
1058 } {
1059 t.Run(tc.name, func(t *testing.T) {
1060 f := newInstallFixture(t, nil)
1061 f.localHook = func(s hostbox.Selection) error {
1062 if tc.lose(f, s) {
1063 return execlease.ErrLost
1064 }
1065 return nil
1066 }
1067 if _, err := f.install(); !errors.Is(err, execlease.ErrLost) {
1068 t.Fatalf("got %v", err)
1069 }
1070 row, err := f.reg.Get(buildHost)
1071 if err != nil || (row != nil) != tc.row {
1072 t.Fatalf("row %+v %v, want minted %v", row, err, tc.row)
1073 }
1074 p := f.paths()
1075 _, ferr := os.Stat(filepath.Join(p.FilesDir, "public", "index.html"))
1076 _, eerr := os.Stat(p.EnvPath)
1077 timers, _ := f.reg.TimersFor(buildHost)
1078 if (ferr == nil) != tc.file || (eerr == nil) != tc.file || (len(timers) == 2) != tc.file {
1079 t.Fatalf("files %v, env %v, timers %d; want restored %v", ferr, eerr, len(timers), tc.file)
1080 }
1081 if live(f) != tc.link {
1082 t.Fatalf("live/<host> published %v, want %v", live(f), tc.link)
1083 }
1084 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
1085 t.Fatal("a journal was written")
1086 }
1087 })
1088 }
1089}
1090
1091// Fix round 1, M2: VerifyLocal reads the guard record and the row back. A
1092// stamp activated since the install, and a row whose hold was released or
1093// taken by another op, are not the receipt's install.
1094func TestRelocationVerifyLocalNeedsTheStampAndTheHold(t *testing.T) {
1095 ctx := t.Context()
1096 t.Run("the hold", func(t *testing.T) {
1097 f := newInstallFixture(t, nil)
1098 l, err := f.install()
1099 if err != nil {
1100 t.Fatal(err)
1101 }
1102 verify := f.a.relocationVerifyLocal(f.cfg)
1103 if changed, err := f.reg.ReleaseHold(buildHost, "rel-1"); err != nil || !changed {
1104 t.Fatalf("release: %v %v", changed, err)
1105 }
1106 if ok, err := verify(ctx, f.read(), l); err != nil || ok {
1107 t.Fatalf("a released hold verified: %v %v", ok, err)
1108 }
1109 if err := f.reg.SetHold(buildHost, "relocating", "other-op"); err != nil {
1110 t.Fatal(err)
1111 }
1112 if ok, err := verify(ctx, f.read(), l); err != nil || ok {
1113 t.Fatalf("another op's hold verified: %v %v", ok, err)
1114 }
1115 })
1116 t.Run("the stamp", func(t *testing.T) {
1117 f := newInstallFixture(t, nil)
1118 l, err := f.install()
1119 if err != nil {
1120 t.Fatal(err)
1121 }
1122 o, err := hostclaim.Update(ctx, f.claims, f.read(), func(c *hostclaim.Claim) error { c.Holder.Stamp++; return nil })
1123 if err != nil {
1124 t.Fatal(err)
1125 }
1126 if err := f.a.Guard.Activate(ctx, buildHost, "dst", nil); err != nil {
1127 t.Fatal(err)
1128 }
1129 if c, err := hostguard.Capture(f.a.Guard.DataDir, buildHost); err != nil || c.Stamp == l.InstallStamp {
1130 t.Fatalf("not re-activated: %+v %v", c, err)
1131 }
1132 if ok, err := f.a.relocationVerifyLocal(f.cfg)(ctx, o, l); err != nil || ok {
1133 t.Fatalf("another stamp verified: %v %v", ok, err)
1134 }
1135 })
1136}
1137
1138// O3' reads the held config and the env file it wrote from the instance
1139// directory, the tenant's to write, through main's safe read: a symlink
1140// (at the name or the directory), a FIFO or an oversized file at either
1141// refuses the install, leaves no receipt, and never blocks it. Each plant
1142// keeps the bytes a following or unbounded read would accept.
1143func TestRelocationInstallKeyCheckRefusesWhatTheTenantPlants(t *testing.T) {
1144 const configured = "configured-key"
1145 for _, target := range []string{"config", "env"} {
1146 for _, p := range tenantPlants {
1147 if target == "env" && p.name == "a symlinked parent directory" {
1148 continue // the config's directory too: the config case covers it
1149 }
1150 t.Run(target+"/"+p.name, func(t *testing.T) {
1151 f := newInstallFixture(t, map[string]string{hostfiles.EnvItem: "CARLOS_ADMIN_TOKEN=t\n", hostfiles.WakeItem: installWake()})
1152 f.configs = map[string]string{"CARLOS_INSTANCE_KEY": configured}
1153 repinWithKey(t, f.buildFixture, &hostrelocate.KeyPin{Source: instancekey.SourceConfig, SHA256: instancekey.Digest(configured)})
1154 f.cfg.render = func(ctx context.Context, s release.Store, k *ecdh.PrivateKey, d string, rt registry.Route, op, box string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
1155 r, err := f.render(ctx, s, k, d, rt, op, box, work)
1156 if err != nil {
1157 return r, err
1158 }
1159 path := r.Path
1160 if target == "env" {
1161 path = f.paths().EnvPath
1162 }
1163 body, err := os.ReadFile(path)
1164 if err != nil {
1165 return r, err
1166 }
1167 plantAt(t, p, path, body, instancekey.EnvReadLimit, '\n')
1168 return r, nil
1169 }
1170 var err error
1171 if e := bounded(t, func() error { _, err = f.install(); return nil }); e != nil {
1172 t.Fatal(e)
1173 }
1174 if !errors.Is(err, p.want) {
1175 t.Fatalf("install: %v; want %v", err, p.want)
1176 }
1177 if target == "env" && !errors.Is(err, hostrelocate.ErrRefused) {
1178 t.Fatalf("an unreadable env file is not a refusal: %v", err)
1179 }
1180 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
1181 t.Fatal("a refused install left a receipt")
1182 }
1183 })
1184 }
1185 }
1186}
1187
1188// The held render always writes its file, so a config missing at the key
1189// check is an error, never a configuration without a key.
1190func TestRelocationInstallKeyCheckNeedsTheConfigItRendered(t *testing.T) {
1191 const configured = "configured-key"
1192 f := newInstallFixture(t, map[string]string{hostfiles.EnvItem: "CARLOS_ADMIN_TOKEN=t\nCARLOS_INSTANCE_KEY=" + configured + "\n", hostfiles.WakeItem: installWake()})
1193 repinWithKey(t, f.buildFixture, &hostrelocate.KeyPin{Source: instancekey.SourcePlatform, SHA256: instancekey.Digest(configured)})
1194 f.cfg.render = func(ctx context.Context, s release.Store, k *ecdh.PrivateKey, d string, rt registry.Route, op, box string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) {
1195 r, err := f.render(ctx, s, k, d, rt, op, box, work)
1196 if err != nil {
1197 return r, err
1198 }
1199 return r, os.Remove(r.Path)
1200 }
1201 if _, err := f.install(); !errors.Is(err, os.ErrNotExist) {
1202 t.Fatalf("the key check without the config it rendered: %v", err)
1203 }
1204 if _, ok, _ := f.a.relocationReceipt(f.cfg)(t.Context(), f.read()); ok {
1205 t.Fatal("a receipt without the config")
1206 }
1207}
1208