amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "io/fs"
8 "os"
9 "path/filepath"
10 "sort"
11 "strings"
12 "testing"
13
14 "github.com/carlosframework/platform/internal/appconfig"
15 "github.com/carlosframework/platform/internal/hostbox"
16 "github.com/carlosframework/platform/internal/hostclaim"
17 "github.com/carlosframework/platform/internal/hostguard"
18 "github.com/carlosframework/platform/internal/lease"
19 "github.com/carlosframework/platform/internal/moves"
20 "github.com/carlosframework/platform/internal/moves/archive"
21 "github.com/carlosframework/platform/internal/park"
22 "github.com/carlosframework/platform/internal/provision"
23 "github.com/carlosframework/platform/internal/registry"
24 "github.com/carlosframework/platform/internal/release"
25)
26
27// The shared installer on the source (Plan 4 Task 3, rulings J13, J14, J20,
28// J24): a dirty return installs the generation the target sent back, from
29// the return lane in the source's own bucket, through the same six phases
30// the landing runs. The fixture is the landing harness driven to the flip,
31// then taken by provision's and hostclaim's own transitions to the point
32// Task 5's source recovery hands the install over: the record rolling_back,
33// dirty, the source at parking; the claim restoring/claimed with the source
34// executing at its pre-move stamp S; the source's installation record
35// active at S; and the source's own pre-move files still at the flat paths.
36// The return lane holds the staged seal's objects "as given" (J22), copied
37// by version into moves/<host>/<rb id>/ of the source's bucket.
38
39const p1RB = "rb_abcdefghijklmnopqrstuvwxyz"
40
41type returnHarness struct {
42 *landingHarness
43 sdir string // the source's data directory
44 rcfg moveCfg
45 rw *movesWorker
46 seal moves.Seal // the return seal
47 baseline provision.MoveSource
48 srcRoute registry.Route
49 asked []string // the releases ReleaseReady was asked about
50 render appconfig.HeldRender
51 sealOf func(*hostclaim.Claim) (moves.Seal, error)
52 installed bool // stop the drive at restored
53}
54
55// newReturnHarness builds the fixture. registered lists the source's four
56// flat paths in the claim's inventory at S and queues its park-local, as
57// Task 5's claimed does when the inventory lists them (J13).
58func newReturnHarness(t *testing.T, registered bool) *returnHarness {
59 t.Helper()
60 return newReturnHarnessWith(t, registered, true)
61}
62
63// returnJournal is the return's own op journal on the source.
64func (h *returnHarness) returnJournal(t *testing.T) *moves.Journal {
65 t.Helper()
66 j, err := moves.ReadJournal(h.sdir, p1Host, p1RB)
67 if err != nil {
68 t.Fatal(err)
69 }
70 return j
71}
72
73// newReturnHarnessWith is newReturnHarness with the target's admission
74// recorded or not (fix round 3): admitted, the claim holds the target's
75// probe admission of the move's prepared generation, the evidence a dirty
76// return needs before it parks the pre-move generations.
77func newReturnHarnessWith(t *testing.T, registered, admitted bool) *returnHarness {
78 t.Helper()
79 ctx := context.Background()
80 h := &returnHarness{landingHarness: newLandingHarness(t)}
81 h.flip(t)
82 if admitted {
83 admitTarget(t, h.landingHarness)
84 }
85 h.sdir = h.cfg.DataDir
86 rec := h.read(t, p1Host)
87 h.baseline = *rec.Move.Source
88 g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
89
90 // The return lane, as given: every sealed object copied by version into
91 // moves/<host>/<rb id>/ of the source's bucket.
92 staged := h.landingHarness.seal(t)
93 lane, from := moves.StagingPrefix(p1Host, p1RB), moves.StagingPrefix(p1Host, p1ID)
94 copyClass := func(objs []moves.SealedObject) []moves.SealedObject {
95 var out []moves.SealedObject
96 for _, o := range objs {
97 key := lane + strings.TrimPrefix(o.Key, from)
98 ver, _, err := h.vs.CopyVersion(ctx, "data-uk", o.Key, o.VersionID, "data-ie", key, true)
99 if err != nil {
100 t.Fatalf("copy %s into the return lane: %v", o.Key, err)
101 }
102 out = append(out, moves.SealedObject{Key: key, VersionID: ver, Size: o.Size})
103 }
104 return out
105 }
106 h.seal = staged
107 h.seal.Replica, h.seal.Files, h.seal.Timers = copyClass(staged.Replica), copyClass(staged.Files), copyClass(staged.Timers)
108 if err := h.seal.Validate(); err != nil {
109 t.Fatal(err)
110 }
111 h.sealOf = func(*hostclaim.Claim) (moves.Seal, error) { return h.seal, nil }
112
113 // The record: the rollback, the target released dirty and returned, the
114 // source claimed and at parking.
115 now := h.clock.Now()
116 for _, step := range []func(*provision.Record) error{
117 provision.MoveRollBack(g, p1RB, provision.RollbackLandingDeadline, now),
118 provision.MoveRollbackTargetPhase(g, provision.RbTargetReleasing, provision.RbTargetReleased, func(rb *provision.MoveRollback) {
119 rb.Dirty, rb.Evidence = true, "moves/bab/app/"+p1Host+"/"+p1ID+"/stopped-target.json"
120 }),
121 provision.MoveRollbackTargetPhase(g, provision.RbTargetReleased, provision.RbTargetReturning, nil),
122 provision.MoveRollbackTargetPhase(g, provision.RbTargetReturning, provision.RbTargetReturned, nil),
123 provision.MoveRollbackSourcePhase(g, provision.RbSourceWaiting, provision.RbSourceClaimed, nil),
124 provision.MoveRollbackSourcePhase(g, provision.RbSourceClaimed, provision.RbSourceRestoring, nil),
125 provision.MoveRollbackSourcePhase(g, provision.RbSourceRestoring, provision.RbSourceParking, nil),
126 } {
127 if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, step); err != nil {
128 t.Fatal(err)
129 }
130 }
131 // The claim: the rollback, the target's release, the source's return.
132 for _, step := range []func(*hostclaim.Claim) error{
133 hostclaim.RollBack(p1ID, p1RB),
134 hostclaim.TargetReleased(p1ID, "uk-box", "uk", "stopped-target", hostclaim.ParkScope{Location: cutRoot, Incarnation: h.inc, EvidenceAt: now, RequestLimit: landingRequestLimit}, obligationID),
135 hostclaim.ReturnToSource(p1ID, "ie-box", false, hostclaim.ReturnFence{}),
136 } {
137 obs, err := hostclaim.Read(ctx, h.s, p1Host)
138 if err != nil {
139 t.Fatal(err)
140 }
141 if _, err := hostclaim.Update(ctx, h.s, obs, step); err != nil {
142 t.Fatal(err)
143 }
144 }
145 if c := h.claim(t); c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 1 {
146 t.Fatalf("fixture claim: %+v holder %+v", c.Op, c.Holder)
147 }
148 // The source's installation record, active at S (Task 5's claimed).
149 for _, d := range []string{"hostguard", ".parked"} {
150 if err := os.MkdirAll(filepath.Join(h.sdir, d), 0o770); err != nil {
151 t.Fatal(err)
152 }
153 }
154 guard := hostguard.Guard{DataDir: h.sdir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: h.s}}
155 if err := guard.Activate(ctx, p1Host, "ie-box", nil); err != nil {
156 t.Fatal(err)
157 }
158 // The source's pre-move installation beside the files and env the cut
159 // left: the database, its marker naming the source's generation, a WAL
160 // and the litestream shadow.
161 db := provision.DefaultDBPath(h.sdir, p1Host)
162 for path, body := range map[string]string{db + ".gen": "cur\n", db + "-wal": "a stale wal", filepath.Join(h.sdir, ".activator-"+p1Host+".db-litestream", "generation"): "shadow"} {
163 if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
164 t.Fatal(err)
165 }
166 if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
167 t.Fatal(err)
168 }
169 }
170 if registered {
171 rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
172 c.Inventory = map[string]hostclaim.InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{p1Host + ".db": 1, p1Host + ".db.gen": 1, p1Host: 1, p1Host + ".env": 1}}}
173 })
174 obs, err := hostclaim.Read(ctx, h.s, p1Host)
175 if err != nil {
176 t.Fatal(err)
177 }
178 if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.QueueLocalPark(p1ID, "ie-box", obligationID)); err != nil {
179 t.Fatal(err)
180 }
181 }
182 h.srcRoute = registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Hibernate: true, Status: "restoring", HoldOp: p1RB,
183 Socket: provision.DefaultSocket(p1Host), DBPath: db, Provisioned: true}
184 // The render answers the current bundle, which is not the pinned one: a
185 // recovery records what it rendered and never compares it.
186 h.render = appconfig.HeldRender{Path: filepath.Join(h.sdir, p1Host+".json"), SHA256: "y", BundleETags: map[string]string{"default": "e9", "environment": ""}, OverlayETag: "o9"}
187 h.rcfg = h.sourceCfg(t)
188 return h
189}
190
191// sourceCfg is the source box's configuration with the installer armed:
192// the landing's hooks over the source's data directory, bucket and region.
193func (h *returnHarness) sourceCfg(t *testing.T) moveCfg {
194 t.Helper()
195 c := h.landingCfg(t)
196 c.Region, c.Box, c.OwnBucket, c.OwnBucketRegion = "ie", "ie-box", "data-ie", "eu-west-1"
197 c.DataDir = h.sdir
198 c.OptDir = h.optDir
199 c.OwnsRoute = func(host string) bool { return host == p1Host }
200 c.Route = func(host string) (*registry.Route, error) {
201 if host != p1Host {
202 return nil, nil
203 }
204 rt := h.srcRoute
205 return &rt, nil
206 }
207 c.FilesDir = func(host string) (string, string) {
208 return filepath.Join(h.sdir, host), filepath.Join(h.sdir, host+".env")
209 }
210 c.Guard = hostguard.Guard{DataDir: h.sdir, LockDir: t.TempDir()}
211 c.Local = park.Local{DataDir: h.sdir, Box: "ie-box"}
212 c.ParkView = recordView(h.s, "ie", roleSource)
213 c.RenderHeld = func(_ context.Context, _ release.Store, rt registry.Route, op string, _ *provision.MoveInputs, before func(string) error) (appconfig.HeldRender, error) {
214 h.renders++
215 if h.renderHook != nil {
216 return h.renderHook(before)
217 }
218 if err := before(h.render.SHA256); err != nil {
219 return appconfig.HeldRender{}, err
220 }
221 if op != p1RB || rt.HoldOp != p1RB {
222 t.Errorf("rendered for %q over a row held by %q, want the rollback %s", op, rt.HoldOp, p1RB)
223 }
224 return h.render, nil
225 }
226 c.ReleaseReady = func(_ context.Context, _ release.Store, _ provision.Record, v string) moves.Check {
227 h.asked = append(h.asked, v)
228 return h.release
229 }
230 c.RemoveVerifyEntry = func(context.Context, release.Store, string) error { return nil }
231 c.WriteGen = hostbox.WriteGenerationMarker
232 return c
233}
234
235// install drives the installer the way Task 5's pass will: one phase per
236// call, the record re-read between them, until it stops advancing. It
237// answers the last call's recorded refusal and error.
238func (h *returnHarness) install(t *testing.T) (string, error) {
239 t.Helper()
240 ctx := context.Background()
241 if h.rw == nil {
242 h.rw = &movesWorker{mirrors: map[string]lease.MirrorStore{}}
243 }
244 h.rw.cfg = h.rcfg
245 defer releaseOwnLease(h.rw, p1Host)
246 for i := 0; i < 8; i++ {
247 rec := h.read(t, p1Host)
248 src, err := recoveryInstallSource(h.rw, h.s, rec, h.sealOf, &h.baseline)
249 if err != nil {
250 return "", err
251 }
252 advanced, refused, err := installGeneration(ctx, h.s, h.rw, rec, src)
253 if err != nil || refused != "" || !advanced {
254 return refused, err
255 }
256 }
257 return "", nil
258}
259
260func (h *returnHarness) rollback(t *testing.T) (*provision.MoveState, *provision.MoveRollback) {
261 t.Helper()
262 m := h.read(t, p1Host).Move
263 if m == nil || m.Rollback == nil {
264 t.Fatalf("record: %+v", m)
265 }
266 return m, m.Rollback
267}
268
269func (h *returnHarness) journal(t *testing.T) *moves.Journal {
270 t.Helper()
271 j, err := moves.ReadJournal(h.sdir, p1Host, p1RB)
272 if err != nil {
273 t.Fatal(err)
274 }
275 return j
276}
277
278// parkedFiles maps every regular file under the source's .parked/ to its body.
279func (h *returnHarness) parkedFiles(t *testing.T) map[string]string {
280 t.Helper()
281 out := map[string]string{}
282 root := filepath.Join(h.sdir, ".parked")
283 err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
284 if err != nil || d.IsDir() {
285 return err
286 }
287 body, err := os.ReadFile(p)
288 if err != nil {
289 return err
290 }
291 rel, _ := filepath.Rel(root, p)
292 out[rel] = string(body)
293 return nil
294 })
295 if err != nil {
296 t.Fatal(err)
297 }
298 return out
299}
300
301func parkedHas(files map[string]string, suffix, body string) bool {
302 for p, b := range files {
303 if strings.HasSuffix(p, suffix) && b == body {
304 return true
305 }
306 }
307 return false
308}
309
310// installedReturn asserts the end state of a dirty return's install through
311// restored, on the source.
312func (h *returnHarness) installedReturn(t *testing.T, pinned *provision.MoveInputs) moves.InstallManifest {
313 t.Helper()
314 m, rb := h.rollback(t)
315 if m.State != provision.MoveRollingBack || rb.SourcePhase != provision.RbSourceRestored || m.StuckReason != "" || !rb.Dirty {
316 t.Fatalf("record: %+v %+v", m, rb)
317 }
318 // configured rendered the current inputs and recorded them, never
319 // comparing them with the pin, which is untouched.
320 want := provision.MoveInputs{BundleETags: map[string]string{"default": "e9", "environment": ""}, OverlayETag: "o9", Release: "v7"}
321 if in := rb.RecoveryInputs; in == nil || in.Release != want.Release || in.OverlayETag != want.OverlayETag || !sameBundle(in.BundleETags, want.BundleETags) {
322 t.Fatalf("recovery_inputs: %+v, want %+v", rb.RecoveryInputs, want)
323 }
324 if got := m.Inputs; got == nil || pinned == nil || got.Release != pinned.Release || !sameBundle(got.BundleETags, pinned.BundleETags) || got.OverlayETag != pinned.OverlayETag {
325 t.Fatalf("move.inputs changed: %+v, was %+v", got, pinned)
326 }
327 if len(h.asked) == 0 {
328 t.Fatal("the release gate was never asked")
329 }
330 for _, v := range h.asked {
331 if v != "v7" {
332 t.Fatalf("release gate asked about %v, want the live v7", h.asked)
333 }
334 }
335 // The claim does not follow the source's install phases: it stays at
336 // restoring/claimed with the source executing at S.
337 c := h.claim(t)
338 if c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 1 {
339 t.Fatalf("claim: %+v holder %+v", c.Op, c.Holder)
340 }
341 // The return generation, g-<rb id>, in the source's own bucket: the
342 // build's four objects.
343 prefix := moves.GenerationPrefix(cutRoot, p1RB)
344 var keys []string
345 for k := range h.f.buckets["data-ie"] {
346 if strings.HasPrefix(k, prefix) {
347 keys = append(keys, strings.TrimPrefix(k, prefix))
348 }
349 }
350 if len(keys) != 4 {
351 t.Fatalf("return generation objects: %v", keys)
352 }
353 var im moves.InstallManifest
354 if err := json.Unmarshal([]byte(h.journal(t).Steps[landStepInstall]), &im); err != nil || im.Op != p1RB || im.N != h.seal.N {
355 t.Fatalf("install manifest in the return's journal: %v %+v", err, im)
356 }
357 dbPath := provision.DefaultDBPath(h.sdir, p1Host)
358 // The pre-move WAL and litestream shadow are gone from the live paths
359 // (checked before anything here opens the database, which makes its own).
360 for _, p := range []string{dbPath + "-wal", filepath.Join(h.sdir, ".activator-"+p1Host+".db-litestream")} {
361 if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
362 t.Fatalf("%s is still at the live path: %v", p, err)
363 }
364 }
365 if got, err := fileSHA256(dbPath); err != nil || got != "sha256:"+im.DBSHA256 {
366 t.Fatalf("the installed database: %v %s want %s", err, got, im.DBSHA256)
367 }
368 if got := countRowsForTest(t, dbPath); got != h.rows {
369 t.Fatalf("the installed database holds %d rows, want %d", got, h.rows)
370 }
371 if gen, err := hostbox.ReadGenerationMarker(dbPath + ".gen"); err != nil || gen != p1RB {
372 t.Fatalf("%s.gen: %q %v, want the return generation %s", dbPath, gen, err, p1RB)
373 }
374 body, err := os.ReadFile(filepath.Join(h.sdir, p1Host, archive.MarkerName))
375 var fg filesGenMarker
376 if err != nil || json.Unmarshal(body, &fg) != nil || fg.Op != p1RB || fg.Incarnation != h.inc {
377 t.Fatalf(".files-gen: %s %v", body, err)
378 }
379 if body, err := os.ReadFile(filepath.Join(h.sdir, p1Host, "upload.txt")); err != nil || string(body) != "hello" {
380 t.Fatalf("the extracted files: %q %v", body, err)
381 }
382 // The new installation is registered at S, the source's own stamp.
383 inv := c.Inventory["ie-box"]
384 if inv.Layout != "flat" || len(inv.Paths) != 4 {
385 t.Fatalf("inventory: %+v", inv)
386 }
387 for _, p := range []string{p1Host + ".db", p1Host + ".db.gen", p1Host, p1Host + ".env"} {
388 if inv.Paths[p] != 1 {
389 t.Fatalf("inventory path %s at %d, want S 1", p, inv.Paths[p])
390 }
391 }
392 // The pre-move generations at the location are residue parked in the
393 // source's region, under the move's op.
394 var gens []string
395 for _, o := range c.Obligations {
396 if o.Op == p1ID && o.Kind == "park-generation" && o.Region == "ie" && o.Park != nil && o.Park.Location == cutRoot {
397 gens = append(gens, o.Generation)
398 }
399 if o.Kind == "park-local" {
400 t.Fatalf("a park-local is still owed: %+v", o)
401 }
402 }
403 sort.Strings(gens)
404 if strings.Join(gens, " ") != "cur old" {
405 t.Fatalf("residue chores: %v (%+v)", gens, c.Obligations)
406 }
407 for k := range h.f.buckets["data-ie"] {
408 if strings.HasPrefix(k, cutRoot+"/g-cur/") || strings.HasPrefix(k, cutRoot+"/g-old/") {
409 t.Fatalf("%s is still at the source's location", k)
410 }
411 }
412 // Timers and the columns come from the return's baseline.
413 if len(h.replaced) != 1 || moves.TimersDigest(h.replaced[0]) != h.baseline.TimersSHA256 {
414 t.Fatalf("ReplaceTimers: %+v", h.replaced)
415 }
416 if len(h.repl) != 1 || h.repl[0] != h.baseline.Replication || len(h.nextDue) != 1 || h.nextDue[0] != h.baseline.NextDue {
417 t.Fatalf("columns: %v %v", h.repl, h.nextDue)
418 }
419 if held := h.leases.holding(); len(held) != 0 {
420 t.Fatalf("the install still holds %v", held)
421 }
422 return im
423}
424
425// A registered source (J13): RetireArtifacts(S) parks the inventoried
426// pre-move paths, the moves-owned rename parks what the inventory does not
427// list (the WAL, the litestream shadow), and only then is the return's
428// installation extracted and registered at S.
429func TestDirtyReturnInstallsUnderTheSourceStamp(t *testing.T) {
430 t.Parallel()
431 h := newReturnHarness(t, true)
432 pinned := h.read(t, p1Host).Move.Inputs
433 if refused, err := h.install(t); err != nil || refused != "" {
434 t.Fatalf("install: %q %v", refused, err)
435 }
436 h.installedReturn(t, pinned)
437 parked := h.parkedFiles(t)
438 for suffix, body := range map[string]string{
439 "/artifacts/" + p1Host + ".db": "the stopped database",
440 "/artifacts/" + p1Host + ".db.gen": "cur\n",
441 "/artifacts/" + p1Host + "/upload.txt": "hello",
442 "/artifacts/" + p1Host + ".db-wal": "a stale wal",
443 "/artifacts/.activator-" + p1Host + ".db-litestream/generation": "shadow",
444 } {
445 if !parkedHas(parked, suffix, body) {
446 t.Fatalf("%s was not parked (%v)", suffix, parked)
447 }
448 }
449 // The registered half was park.Local's: its journal is in the
450 // retirement. The rest was the moves-owned rename, recorded by the op
451 // that ran it, the return.
452 var receipts int
453 for p, b := range parked {
454 if strings.HasSuffix(p, "/"+localParkReceiptName) {
455 receipts++
456 var r localParkReceipt
457 if err := json.Unmarshal([]byte(b), &r); err != nil || r.Op != p1RB {
458 t.Fatalf("receipt %s: %s %v", p, b, err)
459 }
460 }
461 }
462 if receipts != 1 {
463 t.Fatalf("%d moves-owned receipts", receipts)
464 }
465 if !h.journal(t).Done(installStepPreMove) || !h.journal(t).Done(landStepRetired) {
466 t.Fatalf("journal: %v", h.journal(t).Steps)
467 }
468 // restored is the probe's (Task 9): another call changes nothing.
469 before := h.read(t, p1Host).Move.UpdatedAt
470 if refused, err := h.install(t); err != nil || refused != "" {
471 t.Fatalf("install at restored: %q %v", refused, err)
472 }
473 if after := h.read(t, p1Host).Move; !after.UpdatedAt.Equal(before) {
474 t.Fatalf("a call at restored wrote the record: %+v", after)
475 }
476}
477
478// An unregistered source (Paul's H5(b)): no park-local is owed, and the
479// moves-owned rename parks every pre-move artifact before extraction.
480func TestDirtyReturnParksAnUnregisteredSource(t *testing.T) {
481 t.Parallel()
482 h := newReturnHarness(t, false)
483 pinned := h.read(t, p1Host).Move.Inputs
484 if refused, err := h.install(t); err != nil || refused != "" {
485 t.Fatalf("install: %q %v", refused, err)
486 }
487 h.installedReturn(t, pinned)
488 parked := h.parkedFiles(t)
489 for suffix, body := range map[string]string{
490 "/artifacts/" + p1Host + ".db": "the stopped database",
491 "/artifacts/" + p1Host + ".db.gen": "cur\n",
492 "/artifacts/" + p1Host + ".env": "APP_MODE=prod\n",
493 "/artifacts/" + p1Host + "/upload.txt": "hello",
494 "/artifacts/" + p1Host + ".db-wal": "a stale wal",
495 } {
496 if !parkedHas(parked, suffix, body) {
497 t.Fatalf("%s was not parked (%v)", suffix, parked)
498 }
499 }
500}
501
502// A crash at each journalled step of the pre-move park and of the install
503// is finished by the next call, from the journal, with nothing parked twice.
504func TestDirtyReturnReplaysFromTheJournal(t *testing.T) {
505 t.Parallel()
506 for _, point := range []string{"after-retired", "after-" + installStepPreMove + "-plan", "after-" + installStepPreMove, "after-staged", "after-files-rename", "after-record-local", "after-db-rename", "after-record-restored"} {
507 t.Run(point, func(t *testing.T) {
508 h := newReturnHarness(t, false)
509 pinned := h.read(t, p1Host).Move.Inputs
510 h.landAt = point
511 if _, err := h.install(t); !errors.Is(err, errCrash) {
512 t.Fatalf("the crash at %s: %v (points %v)", point, err, h.landHits)
513 }
514 h.rw = nil
515 h.leases.expire()
516 if refused, err := h.install(t); err != nil || refused != "" {
517 t.Fatalf("the replay: %q %v", refused, err)
518 }
519 h.installedReturn(t, pinned)
520 if n := len(h.parkedFiles(t)); n == 0 {
521 t.Fatal("nothing parked")
522 }
523 })
524 }
525}
526
527// The source's own pre-move files are installed over only once parked: a
528// files path that holds anything after the park is not the return's, and
529// the install stops for an operator (stuck, never a landing's rollback)
530// with nothing moved; a marker that names another generation is never
531// replaced.
532func TestDirtyReturnNeverInstallsOverWhatItDidNotPark(t *testing.T) {
533 t.Parallel()
534 t.Run("files", func(t *testing.T) {
535 h := newReturnHarness(t, false)
536 sentinel := filepath.Join(h.sdir, p1Host, "sentinel.txt")
537 h.landFault = func(p string) error {
538 if p == "after-"+landStepRetired {
539 if err := os.MkdirAll(filepath.Dir(sentinel), 0o755); err != nil {
540 t.Fatal(err)
541 }
542 return os.WriteFile(sentinel, []byte("not the return's"), 0o644)
543 }
544 return nil
545 }
546 refused, err := h.install(t)
547 if err != nil || refused != provision.StuckReturnFilesOccupied {
548 t.Fatalf("install: %q %v", refused, err)
549 }
550 m, rb := h.rollback(t)
551 if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || rb.StuckAt != provision.RbSourceLocal || m.Reason != "" {
552 t.Fatalf("record: %+v %+v", m, rb)
553 }
554 if body, err := os.ReadFile(sentinel); err != nil || string(body) != "not the return's" {
555 t.Fatalf("the sentinel: %q %v", body, err)
556 }
557 if _, err := os.Stat(filepath.Join(h.sdir, p1Host, "upload.txt")); !errors.Is(err, os.ErrNotExist) {
558 t.Fatalf("the return's files were placed beside the sentinel: %v", err)
559 }
560 // A stuck install is not driven on.
561 if _, err := h.install(t); err == nil {
562 t.Fatal("the installer ran a stuck record")
563 }
564 })
565 t.Run("marker", func(t *testing.T) {
566 h := newReturnHarness(t, true)
567 gen := provision.DefaultDBPath(h.sdir, p1Host) + ".gen"
568 h.landFault = func(p string) error {
569 if p == "after-"+landStepRetired {
570 return os.WriteFile(gen, []byte("cur\n"), 0o600)
571 }
572 return nil
573 }
574 if _, err := h.install(t); !errors.Is(err, errLandingChanged) {
575 t.Fatalf("install over the pre-move marker: %v", err)
576 }
577 if m, rb := h.rollback(t); m.State != provision.MoveRollingBack || rb.SourcePhase != provision.RbSourceConfigured {
578 t.Fatalf("record: %+v %+v", m, rb)
579 }
580 if body, err := os.ReadFile(gen); err != nil || string(body) != "cur\n" {
581 t.Fatalf("the marker was replaced: %q %v", body, err)
582 }
583 if _, err := os.Stat(provision.DefaultDBPath(h.sdir, p1Host)); !errors.Is(err, os.ErrNotExist) {
584 t.Fatalf("a database was installed beside the marker: %v", err)
585 }
586 })
587}
588
589// A return build that does not verify stops the install at configured,
590// stuck, with nothing installed.
591func TestDirtyReturnCorruptBuildIsStuck(t *testing.T) {
592 t.Parallel()
593 h := newReturnHarness(t, true)
594 h.landFault = func(point string) error {
595 if point == "before-db-verify" {
596 f, err := os.OpenFile(filepath.Join(h.sdir, ".moves", p1Host, p1RB, "build", p1Host+".db"), os.O_APPEND|os.O_WRONLY, 0o600)
597 if err != nil {
598 t.Fatal(err)
599 }
600 defer f.Close()
601 if _, err := f.Write([]byte("corrupt")); err != nil {
602 t.Fatal(err)
603 }
604 }
605 return nil
606 }
607 refused, err := h.install(t)
608 if err != nil || refused != provision.StuckReturnInstallMismatch {
609 t.Fatalf("install: %q %v", refused, err)
610 }
611 m, rb := h.rollback(t)
612 if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnInstallMismatch || rb.StuckAt != provision.RbSourceConfigured || rb.RecoveryInputs != nil {
613 t.Fatalf("record: %+v %+v", m, rb)
614 }
615 dbPath := provision.DefaultDBPath(h.sdir, p1Host)
616 for _, p := range []string{dbPath, dbPath + ".gen"} {
617 if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) {
618 t.Fatalf("%s exists after a refused install: %v", p, err)
619 }
620 }
621}
622
623// The claim check and the record check are the recovery's own (J13): a
624// claim with another executor, or at another kind or phase, is refused
625// before anything is written, and a clean rollback has nothing to install.
626func TestDirtyReturnRequiresItsClaimAndRecord(t *testing.T) {
627 t.Parallel()
628 for name, edit := range map[string]func(*hostclaim.Claim){
629 "another executor": func(c *hostclaim.Claim) { c.Op.Exec = "uk-box" },
630 "another phase": func(c *hostclaim.Claim) { c.Op.Phase = hostclaim.PhaseProbing },
631 "another rollback": func(c *hostclaim.Claim) { c.Op.Intent["rollback"] = "rb_bcdefghijklmnopqrstuvwxya" },
632 "another holder": func(c *hostclaim.Claim) { c.Holder.Stamp = 7 },
633 "a landing op kind": func(c *hostclaim.Claim) { c.Op.Kind = hostclaim.KindLanding },
634 } {
635 t.Run(name, func(t *testing.T) {
636 h := newReturnHarness(t, true)
637 rewriteP1Claim(t, h.s, edit)
638 before := h.claim(t)
639 if _, err := h.install(t); err == nil {
640 t.Fatal("installed under the wrong claim")
641 }
642 if _, rb := h.rollback(t); rb.SourcePhase != provision.RbSourceParking {
643 t.Fatalf("the record advanced: %+v", rb)
644 }
645 if after := h.claim(t); len(after.Obligations) != len(before.Obligations) {
646 t.Fatalf("chores queued: %+v", after.Obligations)
647 }
648 })
649 }
650 t.Run("a clean rollback", func(t *testing.T) {
651 h := newReturnHarness(t, true)
652 rec := h.read(t, p1Host)
653 rec.Move.Rollback.Dirty = false
654 if _, err := recoveryInstallSource(&movesWorker{cfg: h.rcfg}, h.s, rec, h.sealOf, &h.baseline); err == nil {
655 t.Fatal("a clean rollback has an install source")
656 }
657 })
658 t.Run("a seal outside the lane", func(t *testing.T) {
659 // A usable seal over the move's own staging lane in the source's
660 // bucket: every object readable at its version, but not the return's.
661 h := newReturnHarness(t, true)
662 ctx := context.Background()
663 other := h.landingHarness.seal(t)
664 for _, class := range [][]moves.SealedObject{other.Replica, other.Files, other.Timers} {
665 for i, o := range class {
666 ver, _, err := h.vs.CopyVersion(ctx, "data-uk", o.Key, o.VersionID, "data-ie", o.Key, true)
667 if err != nil {
668 t.Fatal(err)
669 }
670 class[i].VersionID = ver
671 }
672 }
673 h.sealOf = func(*hostclaim.Claim) (moves.Seal, error) { return other, nil }
674 if _, err := h.install(t); !errors.Is(err, moves.ErrBadSeal) {
675 t.Fatalf("built from the move's staging lane, not the return lane: %v", err)
676 }
677 if n := len(h.vs.callsOn(moves.GenerationPrefix(cutRoot, p1RB))); n != 0 {
678 t.Fatalf("the build ran: %d calls", n)
679 }
680 })
681}
682
683// J24: the stand-in view's authoritative region is the other side's. A
684// landing works before the target's probe is admitted, so the source's
685// region (From) is authoritative; a dirty return works before the
686// recovery's admission, so the target's (To) is.
687func TestStandInViewTakesTheRole(t *testing.T) {
688 m := &provision.MoveState{From: "ie", To: "uk"}
689 obs := hostclaim.Observed{ETag: "e", Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Location: cutRoot + "/"}}}
690 for role, want := range map[string][2]string{roleTarget: {"uk", "ie"}, roleSource: {"ie", "uk"}} {
691 v, err := standInView(want[0], role, m)(context.Background(), obs)
692 if err != nil || v.Region != want[0] || v.AuthoritativeRegion != want[1] {
693 t.Fatalf("%s: %+v %v", role, v, err)
694 }
695 if g, ok := v.GenerationOf(park.Ref{Key: cutRoot + "/g-" + p1RB + "/0000/x.ltx"}); !ok || g != p1RB {
696 t.Fatalf("%s: generation %q %v", role, g, ok)
697 }
698 }
699 if _, err := standInView("ie", "", m)(context.Background(), obs); !errors.Is(err, park.ErrUnsafe) {
700 t.Fatalf("no role: %v", err)
701 }
702}
703
704// Fix round 1, ruling 1: the moves-owned rename of the source's pre-move
705// artifacts, its plan and its renames, runs inside the host guard, with the
706// claim and the record read fresh there. Authority that changes before it
707// (another executor, or the record no longer this return at local) leaves
708// every live path where it is and journals no plan.
709func TestDirtyReturnRetirementIsGuarded(t *testing.T) {
710 t.Parallel()
711 g := provision.MoveGuard{ID: p1ID}
712 for name, change := range map[string]func(t *testing.T, h *returnHarness){
713 "the claim's executor": func(t *testing.T, h *returnHarness) {
714 rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Op.Exec = "uk-box" })
715 },
716 "the record": func(t *testing.T, h *returnHarness) {
717 g.Incarnation = h.inc
718 if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveMarkStuck(g, provision.StuckReturnFilesOccupied)); err != nil {
719 t.Fatal(err)
720 }
721 },
722 } {
723 t.Run(name, func(t *testing.T) {
724 h := newReturnHarness(t, false)
725 changed := false
726 h.landFault = func(p string) error {
727 if p == "before-"+installStepPreMove && !changed {
728 changed = true
729 change(t, h)
730 }
731 return nil
732 }
733 if _, err := h.install(t); err == nil {
734 t.Fatal("the rename ran after its authority changed")
735 }
736 if !changed {
737 t.Fatalf("the rename's guard point was never reached: %v", h.landHits)
738 }
739 db := provision.DefaultDBPath(h.sdir, p1Host)
740 for path, body := range map[string]string{db: "the stopped database", db + ".gen": "cur\n", db + "-wal": "a stale wal",
741 filepath.Join(h.sdir, p1Host, "upload.txt"): "hello", filepath.Join(h.sdir, p1Host+".env"): "APP_MODE=prod\n"} {
742 if got, err := os.ReadFile(path); err != nil || string(got) != body {
743 t.Fatalf("%s moved: %q %v", path, got, err)
744 }
745 }
746 if j := h.journal(t); j.Done(installStepPreMove+"-plan") || j.Done(installStepPreMove) {
747 t.Fatalf("a plan was journalled: %v", j.Steps)
748 }
749 if parked := h.parkedFiles(t); len(parked) != 0 {
750 t.Fatalf("parked: %v", parked)
751 }
752 })
753 }
754}
755
756// Fix round 1, ruling 2 (§7.8, rev 7.11): a leftover tagged with a stamp the
757// claim shows as superseded on this box (a cold admission here that issued
758// it, now neither the holder's stamp nor its live session) is retired
759// through park.Local, so its inventory entries clear before the return's
760// installation is registered; the moves-owned rename then finds nothing of
761// it to move.
762func TestDirtyReturnRetiresRetiredStampLeftovers(t *testing.T) {
763 t.Parallel()
764 h := newReturnHarness(t, true)
765 wal, shadow := p1Host+".db-wal", ".activator-"+p1Host+".db-litestream"
766 rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
767 inv := c.Inventory["ie-box"]
768 inv.Paths[wal], inv.Paths[shadow] = 5, 5
769 c.Inventory["ie-box"] = inv
770 c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "ie", Session: 5, Box: "ie-box", Generation: "g5", Mode: "cold", Covered: true})
771 })
772 pinned := h.read(t, p1Host).Move.Inputs
773 if refused, err := h.install(t); err != nil || refused != "" {
774 t.Fatalf("install: %q %v", refused, err)
775 }
776 h.installedReturn(t, pinned)
777 parked := h.parkedFiles(t)
778 for suffix, body := range map[string]string{"/artifacts/" + wal: "a stale wal", "/artifacts/" + shadow + "/generation": "shadow"} {
779 if !parkedHas(parked, suffix, body) {
780 t.Fatalf("%s was not parked (%v)", suffix, parked)
781 }
782 }
783 // Retired by park.Local, not by the rename: the rename's plan found
784 // them gone.
785 plan, err := journalledLocalPlan(h.journal(t), installStepPreMove)
786 if err != nil {
787 t.Fatal(err)
788 }
789 for _, it := range plan.Items {
790 if (it.Path == wal || it.Path == shadow || it.Path == p1Host+".db") && it.ID != "" {
791 t.Fatalf("the rename planned %s, which the inventory claimed", it.Path)
792 }
793 }
794}
795
796// Fix round 1, ruling 2: a leftover tagged with any other stamp (live,
797// unknown, or another holder's) stops the return for an operator before
798// anything moves, with a recorded reason; once the operator has dealt with
799// it, move --retry (MoveRetry) goes back to local and the install proceeds.
800func TestDirtyReturnUnknownStampIsStuckUntilRetried(t *testing.T) {
801 t.Parallel()
802 for name, stamp := range map[string]uint64{"unknown": 9, "the live session": 0} {
803 t.Run(name, func(t *testing.T) {
804 h := newReturnHarness(t, true)
805 wal := p1Host + ".db-wal"
806 rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
807 s := stamp
808 if s == 0 {
809 // The holder's live writer session on this box: never a
810 // leftover, even with a cold admission record here.
811 s = c.Holder.Session
812 if s == 0 || s == c.Holder.Stamp {
813 s = 11
814 c.Holder.Session = s
815 }
816 c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "ie", Session: s, Box: "ie-box", Generation: "g-live", Mode: "cold", Covered: true})
817 }
818 inv := c.Inventory["ie-box"]
819 inv.Paths[wal] = s
820 c.Inventory["ie-box"] = inv
821 })
822 pinned := h.read(t, p1Host).Move.Inputs
823 refused, err := h.install(t)
824 if err != nil || refused != provision.StuckReturnUnknownStamp {
825 t.Fatalf("install: %q %v", refused, err)
826 }
827 m, rb := h.rollback(t)
828 if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnUnknownStamp || rb.StuckAt != provision.RbSourceLocal {
829 t.Fatalf("record: %+v %+v", m, rb)
830 }
831 db := provision.DefaultDBPath(h.sdir, p1Host)
832 for path, body := range map[string]string{db: "the stopped database", db + ".gen": "cur\n", db + "-wal": "a stale wal"} {
833 if got, err := os.ReadFile(path); err != nil || string(got) != body {
834 t.Fatalf("%s moved: %q %v", path, got, err)
835 }
836 }
837 if parked := h.parkedFiles(t); len(parked) != 0 {
838 t.Fatalf("parked before the refusal: %v", parked)
839 }
840 if !hostclaim.InventoryHasStamp(h.claim(t), "ie-box", 1) {
841 t.Fatal("S was retired before the refusal")
842 }
843 // The operator accounts for the file (here: its stray entry is
844 // dropped), then retries.
845 rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
846 inv := c.Inventory["ie-box"]
847 delete(inv.Paths, wal)
848 c.Inventory["ie-box"] = inv
849 })
850 g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
851 if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetry(g)); err != nil {
852 t.Fatal(err)
853 }
854 if refused, err := h.install(t); err != nil || refused != "" {
855 t.Fatalf("install after the retry: %q %v", refused, err)
856 }
857 h.installedReturn(t, pinned)
858 if !parkedHas(h.parkedFiles(t), "/artifacts/"+wal, "a stale wal") {
859 t.Fatal("the WAL was not parked after the retry")
860 }
861 })
862 }
863}
864