amadan

carlos / platform Public

host: the return's pre-move retirement accounts for every stamp and runs its rename in the guard (Task 3 fix round 1)

Ruling 1: the moves-owned rename of the source's pre-move artifacts, its
plan and its renames, runs inside the host guard (guardedWith), with the
claim and the record read fresh there and no inventory entry claiming a
path it renames; RetireArtifacts keeps its own guard.

Ruling 2 (spec §7.8, rev 7.11): before extraction the source's inventory
is walked stamp by stamp from a fresh claim. S is retired as before; a
stamp the claim shows superseded on this box (a cold admission here
issued it, and it is neither S nor the live session) is retired through
park.Local against a park-local queued for it, so its entries clear; any
other stamp stops the return with StuckReturnUnknownStamp before anything
moves, and move --retry after the operator's fix proceeds. Registration
never meets a differently tagged leftover.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 44a3a502b5f1566b77226de3aee9040501913019 parent af8f1f2
3 files changed, +325 −40
  • internal/host/movesinstall.go +156 −40
  • internal/host/movesinstall_test.go +158 −0
  • internal/host/moveslandinstall.go +11 −0
diff --git a/internal/host/movesinstall.go b/internal/host/movesinstall.go
index 6c03b26b5..5d1f4e8f8 100644
--- a/internal/host/movesinstall.go
+++ b/internal/host/movesinstall.go
@@ -47,10 +47,12 @@ import (
// never compared with move.inputs (J2-R2-5);
// - local: a landing parks an earlier stay's artifacts tagged S'; a return
// parks the source's own pre-move installation before anything is
-// extracted (J13): RetireArtifacts(S) when the claim's inventory lists
-// paths of S, then the moves-owned rename (H5(b), parkLocalArtifacts) of
-// every pre-move artifact still at the flat paths. placeStaged and
-// restore then find nothing there, and refuse anything that appears;
+// extracted (J13, fix round 1): RetireArtifacts for S and for every
+// stamp the claim shows superseded on this box, a stuck mark for any
+// other stamp, then the moves-owned rename (H5(b), parkLocalArtifacts)
+// of every pre-move artifact still at the flat paths, inside the host
+// guard. placeStaged and restore then find nothing there, and refuse
+// anything that appears;
// - the stand-in park view's authoritative region (J24): the source's for
// a landing, the target's for a return, each the other side of the move
// before this side's probe is admitted.
@@ -82,6 +84,7 @@ const (
refuseInstallMismatch // configured: the build does not verify
refuseConfigChanged // configured: a landing's render is not its pin
refuseDeadline // configured: a landing ran past its deadline
+ refuseUnknownStamp // local: a return found paths under a stamp it cannot account for
)
// installSource is one kind of install: what the six phases need from the
@@ -609,8 +612,14 @@ func (ir *installRun) local_(ctx context.Context) (bool, error) {
}
if !j.Done(landStepRetired) {
if ir.src.Role == roleSource {
- if err := ir.retirePreMove(ctx, obs, captured, j, mark); err != nil {
+ outcome, err := ir.retirePreMove(ctx, captured, j, mark)
+ switch {
+ case err != nil:
return false, err
+ case outcome == restoreMarked:
+ return true, nil // the record changed: the refusal
+ case outcome == restoreWaiting:
+ return false, nil
}
} else if raw, ok := j.Steps[landStepPrevious]; ok {
earlier, err := strconv.ParseUint(raw, 10, 64)
@@ -744,49 +753,154 @@ func installArtifactPaths(host string) []string {
return []string{db, db + ".gen", db + "-wal", db + "-shm", db + ".corrupt", ".activator-" + db + "-litestream", host, host + ".env"}
}
-// retirePreMove is a dirty return's local sub-step (2) (ruling J13): the
-// source's own pre-move installation leaves the flat paths before anything
-// of the return's is placed there, so placeStaged and restore find them
-// empty and never install over them. When the claim's inventory lists paths
-// of S, the source's installation stamp (the holder's after ReturnToSource),
-// they are retired through park.Local under the host guard against the
-// park-local the source's claimed queued (RetireArtifacts completes it);
-// with paths of S listed and no park-local owed it stops, since only the
-// claimed CAS queues it. A park-local owed with nothing of S listed (a replay
-// after the retirement) is completed. Then every pre-move artifact still at
-// the flat paths, registered or not, is parked by the moves-owned journalled
-// rename (H5(b), parkLocalArtifacts) into the engine's layout. Retired
-// stamps other than S are not the source's to find here: RecordInventory
-// refuses a path still listed under one, and the install stops there.
-func (ir *installRun) retirePreMove(ctx context.Context, obs hostclaim.Observed, captured hostguard.Captured, j *moves.Journal, mark func(step, value string) error) error {
+// retirePreMove is a dirty return's local sub-step (2) (ruling J13; fix
+// round 1): the source's own pre-move installation, and anything an earlier
+// installation of this holder left on this box, leaves the flat paths before
+// anything of the return's is placed there, so placeStaged and restore find
+// them empty and never install over them, and the return's registration
+// never meets a path tagged with another stamp.
+//
+// First the claim's inventory for this box, read fresh each round, is
+// accounted for stamp by stamp, before anything moves:
+// - S, the source's installation stamp (the holder's after
+// ReturnToSource): retired through park.Local under the host guard
+// against the park-local the source's claimed queued
+// (RetireArtifacts completes it); paths of S listed with no park-local
+// owed is an error, since only the claimed CAS queues it;
+// - a stamp the claim shows superseded on this box (supersededOnBox: a
+// cold admission here issued it, and it is neither S nor the holder's
+// live session; spec §7.8, rev 7.11): retired the same way, against a
+// park-local queued here for it, so its inventory entries clear;
+// - any other stamp (live, unknown, another holder's): nothing is moved,
+// and the return stops for an operator (StuckReturnUnknownStamp); a
+// retry after the operator has accounted for it runs this again.
+//
+// A park-local owed with nothing listed (a replay after a retirement) is
+// completed. Then every pre-move artifact still at the flat paths that no
+// inventory entry claims is parked by the moves-owned journalled rename
+// (H5(b), parkLocalArtifacts), its plan and its renames inside the host
+// guard with the claim and the record read fresh there (fix round 1, ruling
+// 1), so a call that resumes after authority moved on renames nothing.
+func (ir *installRun) retirePreMove(ctx context.Context, captured hostguard.Captured, j *moves.Journal, mark func(step, value string) error) (restoreOutcome, error) {
cfg := ir.cfg()
- host, id := ir.rec.Host, ir.g.ID
+ host, id, phase := ir.rec.Host, ir.g.ID, ir.src.Phases[3]
stamp := captured.Stamp
- var chore *hostclaim.Obligation
- for i := range obs.Claim.Obligations {
- if o := &obs.Claim.Obligations[i]; o.Op == id && o.Kind == "park-local" && o.Box == cfg.Box {
- chore = o
+ for round := 0; ; round++ {
+ if round > 64 {
+ return restoreWaiting, fmt.Errorf("move %s: the source's inventory did not clear", id)
+ }
+ obs, err := ir.sync(ctx, phase)
+ if err != nil {
+ return restoreWaiting, err
}
+ c := obs.Claim
+ var retire []uint64
+ for _, x := range inventoryStamps(c, cfg.Box) {
+ switch {
+ case x == stamp:
+ retire = append([]uint64{x}, retire...)
+ case supersededOnBox(c, cfg.Box, x):
+ retire = append(retire, x)
+ default:
+ log.Printf("moves: %s: this box's inventory lists paths under stamp %d, which the return of %s cannot account for; nothing is moved", host, x, id)
+ ok, err := ir.refuse(ctx, refuseUnknownStamp)
+ if err != nil || !ok {
+ return restoreWaiting, err
+ }
+ return restoreMarked, nil
+ }
+ }
+ var chore *hostclaim.Obligation
+ for i := range c.Obligations {
+ if o := &c.Obligations[i]; o.Op == id && o.Kind == "park-local" && o.Box == cfg.Box {
+ chore = o
+ }
+ }
+ if len(retire) == 0 {
+ if chore != nil {
+ if _, err := hostclaim.Update(ctx, ir.s, obs, hostclaim.CompleteLocalPark(id, cfg.Box, stamp)); err != nil {
+ return restoreWaiting, fmt.Errorf("complete the source's local park: %w", err)
+ }
+ }
+ break
+ }
+ x := retire[0]
+ if chore == nil {
+ if x == stamp {
+ return restoreWaiting, fmt.Errorf("move %s: this box's inventory lists the source's pre-move paths at stamp %d, and the claim owes no park-local for them", id, stamp)
+ }
+ if obs, err = hostclaim.Update(ctx, ir.s, obs, hostclaim.QueueLocalPark(id, cfg.Box, obligationID)); err != nil {
+ return restoreWaiting, fmt.Errorf("queue the local park of stamp %d: %w", x, err)
+ }
+ for i := range obs.Claim.Obligations {
+ if o := &obs.Claim.Obligations[i]; o.Op == id && o.Kind == "park-local" && o.Box == cfg.Box {
+ chore = o
+ }
+ }
+ if chore == nil {
+ return restoreWaiting, fmt.Errorf("move %s: the local park of stamp %d was not queued", id, x)
+ }
+ }
+ if err := ir.local.RetireArtifacts(ctx, host, captured, chore.ID, id, x); err != nil {
+ return restoreWaiting, fmt.Errorf("retire the source's artifacts of stamp %d: %w", x, err)
+ }
+ log.Printf("moves: %s: the source's artifacts of stamp %d are retired", host, x)
}
- registered := hostclaim.InventoryHasStamp(obs.Claim, cfg.Box, stamp)
- switch {
- case registered && chore != nil:
- if err := ir.local.RetireArtifacts(ctx, host, captured, chore.ID, id, stamp); err != nil {
- return fmt.Errorf("retire the source's pre-move installation: %w", err)
+ if err := ir.fault("before-" + installStepPreMove); err != nil {
+ return restoreWaiting, err
+ }
+ paths := installArtifactPaths(host)
+ unclaimed := func(c *hostclaim.Claim) error {
+ for _, p := range paths {
+ if _, ok := c.Inventory[cfg.Box].Paths[p]; ok {
+ return fmt.Errorf("%w: the inventory claims %s", errLandingChanged, p)
+ }
}
- log.Printf("moves: %s: the source's pre-move installation (stamp %d) is retired", host, stamp)
- case registered:
- return fmt.Errorf("move %s: this box's inventory lists the source's pre-move paths at stamp %d, and the claim owes no park-local for them", id, stamp)
- case chore != nil:
- if _, err := hostclaim.Update(ctx, ir.s, obs, hostclaim.CompleteLocalPark(id, cfg.Box, stamp)); err != nil {
- return fmt.Errorf("complete the source's local park: %w", err)
+ return nil
+ }
+ err := ir.guardedWith(ctx, captured, phase, unclaimed, func() error {
+ _, err := parkLocalArtifacts(ctx, j, mark, localArtifactPark{DataDir: cfg.DataDir, Incarnation: captured.Incarnation, Host: host,
+ Op: ir.src.GenerationOp, Step: installStepPreMove, Paths: paths, NewID: obligationID, Fault: ir.fault})
+ return err
+ })
+ if err != nil {
+ return restoreWaiting, fmt.Errorf("park the source's pre-move artifacts: %w", err)
+ }
+ return restoreInstalled, nil
+}
+
+// inventoryStamps are the distinct stamps box's inventory tags paths with,
+// in order.
+func inventoryStamps(c *hostclaim.Claim, box string) []uint64 {
+ seen := map[uint64]bool{}
+ var out []uint64
+ for _, s := range c.Inventory[box].Paths {
+ if !seen[s] {
+ seen[s] = true
+ out = append(out, s)
}
}
- if _, err := parkLocalArtifacts(ctx, j, mark, localArtifactPark{DataDir: cfg.DataDir, Incarnation: captured.Incarnation, Host: host,
- Op: ir.src.GenerationOp, Step: installStepPreMove, Paths: installArtifactPaths(host), NewID: obligationID, Fault: ir.fault}); err != nil {
- return fmt.Errorf("park the source's pre-move artifacts: %w", err)
+ sort.Slice(out, func(i, k int) bool { return out[i] < out[k] })
+ return out
+}
+
+// supersededOnBox reports whether the claim shows stamp as an installation
+// stamp this holder issued on box and has since moved past: a cold
+// admission's stamp is its record's session (hostclaim.AdmitWriter), so a
+// cold admission record on box with that session names it, and it is
+// neither the holder's stamp nor its live writer session. A stamp the claim
+// has no such record of is not this box's to retire.
+func supersededOnBox(c *hostclaim.Claim, box string, stamp uint64) bool {
+ h := c.Holder
+ if h == nil || stamp == 0 || stamp == h.Stamp || stamp == h.Session {
+ return false
+ }
+ for _, a := range h.Admissions {
+ if a.Mode == "cold" && a.Box == box && a.Session == stamp {
+ return true
+ }
}
- return nil
+ return false
}
// configured (§7.6, §9.1, §9.2): the held row rendered (RenderHeld, the
@@ -963,6 +1077,8 @@ func recoveryInstallSource(w *movesWorker, s release.Store, rec provision.Record
reason = provision.StuckReturnFilesOccupied
case refuseInstallMismatch:
reason = provision.StuckReturnInstallMismatch
+ case refuseUnknownStamp:
+ reason = provision.StuckReturnUnknownStamp
default:
return false, "", fmt.Errorf("move %s: a return's install has no refusal %d", g.ID, why)
}
diff --git a/internal/host/movesinstall_test.go b/internal/host/movesinstall_test.go
index 0c9c162ad..631dc7fc1 100644
--- a/internal/host/movesinstall_test.go
+++ b/internal/host/movesinstall_test.go
@@ -665,3 +665,161 @@ func TestStandInViewTakesTheRole(t *testing.T) {
t.Fatalf("no role: %v", err)
}
}
+
+// Fix round 1, ruling 1: the moves-owned rename of the source's pre-move
+// artifacts, its plan and its renames, runs inside the host guard, with the
+// claim and the record read fresh there. Authority that changes before it
+// (another executor, or the record no longer this return at local) leaves
+// every live path where it is and journals no plan.
+func TestDirtyReturnRetirementIsGuarded(t *testing.T) {
+ g := provision.MoveGuard{ID: p1ID}
+ for name, change := range map[string]func(t *testing.T, h *returnHarness){
+ "the claim's executor": func(t *testing.T, h *returnHarness) {
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Op.Exec = "uk-box" })
+ },
+ "the record": func(t *testing.T, h *returnHarness) {
+ g.Incarnation = h.inc
+ if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveMarkStuck(g, provision.StuckReturnFilesOccupied)); err != nil {
+ t.Fatal(err)
+ }
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ h := newReturnHarness(t, false)
+ changed := false
+ h.landFault = func(p string) error {
+ if p == "before-"+installStepPreMove && !changed {
+ changed = true
+ change(t, h)
+ }
+ return nil
+ }
+ if _, err := h.install(t); err == nil {
+ t.Fatal("the rename ran after its authority changed")
+ }
+ if !changed {
+ t.Fatalf("the rename's guard point was never reached: %v", h.landHits)
+ }
+ db := provision.DefaultDBPath(h.sdir, p1Host)
+ for path, body := range map[string]string{db: "the stopped database", db + ".gen": "cur\n", db + "-wal": "a stale wal",
+ filepath.Join(h.sdir, p1Host, "upload.txt"): "hello", filepath.Join(h.sdir, p1Host+".env"): "APP_MODE=prod\n"} {
+ if got, err := os.ReadFile(path); err != nil || string(got) != body {
+ t.Fatalf("%s moved: %q %v", path, got, err)
+ }
+ }
+ if j := h.journal(t); j.Done(installStepPreMove+"-plan") || j.Done(installStepPreMove) {
+ t.Fatalf("a plan was journalled: %v", j.Steps)
+ }
+ if parked := h.parkedFiles(t); len(parked) != 0 {
+ t.Fatalf("parked: %v", parked)
+ }
+ })
+ }
+}
+
+// Fix round 1, ruling 2 (§7.8, rev 7.11): a leftover tagged with a stamp the
+// claim shows as superseded on this box (a cold admission here that issued
+// it, now neither the holder's stamp nor its live session) is retired
+// through park.Local, so its inventory entries clear before the return's
+// installation is registered; the moves-owned rename then finds nothing of
+// it to move.
+func TestDirtyReturnRetiresRetiredStampLeftovers(t *testing.T) {
+ h := newReturnHarness(t, true)
+ wal, shadow := p1Host+".db-wal", ".activator-"+p1Host+".db-litestream"
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ inv := c.Inventory["ie-box"]
+ inv.Paths[wal], inv.Paths[shadow] = 5, 5
+ c.Inventory["ie-box"] = inv
+ c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "ie", Session: 5, Box: "ie-box", Generation: "g5", Mode: "cold", Covered: true})
+ })
+ pinned := h.read(t, p1Host).Move.Inputs
+ if refused, err := h.install(t); err != nil || refused != "" {
+ t.Fatalf("install: %q %v", refused, err)
+ }
+ h.installedReturn(t, pinned)
+ parked := h.parkedFiles(t)
+ for suffix, body := range map[string]string{"/artifacts/" + wal: "a stale wal", "/artifacts/" + shadow + "/generation": "shadow"} {
+ if !parkedHas(parked, suffix, body) {
+ t.Fatalf("%s was not parked (%v)", suffix, parked)
+ }
+ }
+ // Retired by park.Local, not by the rename: the rename's plan found
+ // them gone.
+ plan, err := journalledLocalPlan(h.journal(t), installStepPreMove)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, it := range plan.Items {
+ if (it.Path == wal || it.Path == shadow || it.Path == p1Host+".db") && it.ID != "" {
+ t.Fatalf("the rename planned %s, which the inventory claimed", it.Path)
+ }
+ }
+}
+
+// Fix round 1, ruling 2: a leftover tagged with any other stamp (live,
+// unknown, or another holder's) stops the return for an operator before
+// anything moves, with a recorded reason; once the operator has dealt with
+// it, move --retry (MoveRetry) goes back to local and the install proceeds.
+func TestDirtyReturnUnknownStampIsStuckUntilRetried(t *testing.T) {
+ for name, stamp := range map[string]uint64{"unknown": 9, "the live session": 0} {
+ t.Run(name, func(t *testing.T) {
+ h := newReturnHarness(t, true)
+ wal := p1Host + ".db-wal"
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ s := stamp
+ if s == 0 {
+ // The holder's live writer session on this box: never a
+ // leftover, even with a cold admission record here.
+ s = c.Holder.Session
+ if s == 0 || s == c.Holder.Stamp {
+ s = 11
+ c.Holder.Session = s
+ }
+ c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "ie", Session: s, Box: "ie-box", Generation: "g-live", Mode: "cold", Covered: true})
+ }
+ inv := c.Inventory["ie-box"]
+ inv.Paths[wal] = s
+ c.Inventory["ie-box"] = inv
+ })
+ pinned := h.read(t, p1Host).Move.Inputs
+ refused, err := h.install(t)
+ if err != nil || refused != provision.StuckReturnUnknownStamp {
+ t.Fatalf("install: %q %v", refused, err)
+ }
+ m, rb := h.rollback(t)
+ if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnUnknownStamp || rb.StuckAt != provision.RbSourceLocal {
+ t.Fatalf("record: %+v %+v", m, rb)
+ }
+ db := provision.DefaultDBPath(h.sdir, p1Host)
+ for path, body := range map[string]string{db: "the stopped database", db + ".gen": "cur\n", db + "-wal": "a stale wal"} {
+ if got, err := os.ReadFile(path); err != nil || string(got) != body {
+ t.Fatalf("%s moved: %q %v", path, got, err)
+ }
+ }
+ if parked := h.parkedFiles(t); len(parked) != 0 {
+ t.Fatalf("parked before the refusal: %v", parked)
+ }
+ if !hostclaim.InventoryHasStamp(h.claim(t), "ie-box", 1) {
+ t.Fatal("S was retired before the refusal")
+ }
+ // The operator accounts for the file (here: its stray entry is
+ // dropped), then retries.
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ inv := c.Inventory["ie-box"]
+ delete(inv.Paths, wal)
+ c.Inventory["ie-box"] = inv
+ })
+ g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
+ if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetry(g)); err != nil {
+ t.Fatal(err)
+ }
+ if refused, err := h.install(t); err != nil || refused != "" {
+ t.Fatalf("install after the retry: %q %v", refused, err)
+ }
+ h.installedReturn(t, pinned)
+ if !parkedHas(h.parkedFiles(t), "/artifacts/"+wal, "a stale wal") {
+ t.Fatal("the WAL was not parked after the retry")
+ }
+ })
+ }
+}
diff --git a/internal/host/moveslandinstall.go b/internal/host/moveslandinstall.go
index f5bd8b424..1d8825d09 100644
--- a/internal/host/moveslandinstall.go
+++ b/internal/host/moveslandinstall.go
@@ -70,10 +70,21 @@ func (ir *installRun) capture(holder hostclaim.Holder) (hostguard.Captured, erro
// install; it must not call anything that takes the guard itself
// (park.Local's retirement and inventory), nor wait on the network.
func (ir *installRun) guarded(ctx context.Context, captured hostguard.Captured, phase string, fn func() error) error {
+ return ir.guardedWith(ctx, captured, phase, nil, fn)
+}
+
+// guardedWith is guarded with one more check of the fresh claim (nil for
+// none), run after the install's own.
+func (ir *installRun) guardedWith(ctx context.Context, captured hostguard.Captured, phase string, extra func(*hostclaim.Claim) error, fn func() error) error {
check := func(c *hostclaim.Claim) error {
if err := ir.src.ClaimCheck(phase)(c); err != nil {
return err
}
+ if extra != nil {
+ if err := extra(c); err != nil {
+ return err
+ }
+ }
if c.Holder == nil || c.Holder.Stamp != captured.Stamp {
return fmt.Errorf("%w: the claim's holder is not at the installation stamp %d", errLandingChanged, captured.Stamp)
}