carlos / platform Public

host: the rollback parks only its own leftovers and frees the build lease; restore and install at configured (Plan 4 final review)

Finding 1: the target's moves-owned rename (parkLeftovers) now moves a
leftover only on a record that this op created it: the landing's placement
receipt or staged identities for the files directory and env, the record's
installed_db_sha256 for the database and what sits beside it, and the held
render's config ack (this op, this box, the file's sha256) for the
rendered config. Anything else, a files directory or env the landing
refused as occupied included, is recorded in the plan as unowned, logged
and left where it is.

Finding 2: the location lease a failed build keeps across passes is
released when the rollback decision takes the landing, and on every entry
to the target's half of the rollback, whoever decided it.

Finding 3: the installer restores and installs inside configured, before
the CAS that writes restored (G8, J24). The restoring and landing rows now
grant restore and install at configured and nothing at restored. The
dirty return's and the landing's own writes are checked end to end through
hostbox.Policy.Check with MovesClaimPolicy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 2a19f4ff57059810d8ccc6babe288d1da03014c1 parent 7858c2b

7 files changed, +647 −47

internal/host/movesland.go +8 −1
internal/host/moveslandpark.go +3 −1
internal/host/movespolicy.go +23 −16
internal/host/movespolicy_test.go +31 −18
internal/host/movesretire.go +30 −4
internal/host/movesrollback.go +117 −7
internal/host/movesrollback_finalfix_test.go +435 −0

diff --git a/internal/host/movesland.go b/internal/host/movesland.go

index 5657dfe77..7087ab6e5 100644

--- a/internal/host/movesland.go

+++ b/internal/host/movesland.go

@@ -141,8 +141,15 @@ func landingArmed(cfg moveCfg) bool {

func (w *movesWorker) landingPass(ctx context.Context, s release.Store, rec provision.Record) error {

cfg := w.cfg

// The rollback decision (movesrollback.go, J10, J11): a recorded mark

- // or the landing's deadline, at any phase, armed or not.

+ // or the landing's deadline, at any phase, armed or not. A landing the

+ // decision took is abandoned, and with it the location's lease a build

+ // keeps across passes (building through published): it is released

+ // here, before the pass ends, or the target's cleaning could never take

+ // the same lease (the final review's finding 2).

if done, err := w.rollBackLanding(ctx, s, rec); done || err != nil {

+ if done {

+ releaseOwnLease(w, rec.Host)

+ }

return err

}

if !landingArmed(cfg) {

diff --git a/internal/host/moveslandpark.go b/internal/host/moveslandpark.go

index 9c99185a9..2c08a40b0 100644

--- a/internal/host/moveslandpark.go

+++ b/internal/host/moveslandpark.go

@@ -185,7 +185,9 @@ func (ir *installRun) releaseLease() { releaseOwnLease(ir.w, ir.rec.Host) }

// releaseOwnLease releases host's own-location lease on w, if held. The

// landing's pass settles it at its end (settleLease); a return's caller

-// does the same once its install call returns.

+// does the same once its install call returns; a rollback that abandons

+// the landing releases it when the decision takes the record and again on

+// every entry to the target's half (rollbackTarget).

func releaseOwnLease(w *movesWorker, host string) {

w.mu.Lock()

held := w.landLeases[host]

diff --git a/internal/host/movespolicy.go b/internal/host/movespolicy.go

index 7c9e115a0..6b6bf7885 100644

--- a/internal/host/movespolicy.go

+++ b/internal/host/movespolicy.go

@@ -34,9 +34,11 @@ import (

// participant, intent.target, and not intent.source; in the move's

// target region (intent.to); and not in lost;

// - the landing's phase permits the action (movesPhaseRights): held-timers

-// from local, held-config and held-release from configured, restore and

-// install from restored. These are the phases Plan 2 built (checking to

-// restored). The probe's rights at probing (spawn, seed, writer, and the

+// at local; held-config, held-release, restore and install at

+// configured, the phase whose work installs the database before its

+// closing CAS writes restored (G8; the final review's finding 3), so

+// restored, the install complete, grants none of them. These are the

+// phases Plan 2 built (checking to restored). The probe's rights at probing (spawn, seed, writer, and the

// held render before every probe, §7.7) and the ordinary wake rights at

// committed (§19.1) are Plan 3's rows, added when the probe is built;

// - the identity fits the action: a held-config write names the config

@@ -65,9 +67,10 @@ import (

// holder incarnation: rolling_back for this rollback (Rollback.ID is

// intent.rollback) under restoring, reverting under reverting;

// - the row permits the action. restoring is keyed by the record's

-// Rollback.SourcePhase: local grants held-timers, configured held-config

-// and held-release, restored restore and install, each under the claim

-// at claimed, for a dirty rollback, with the holder at the source's

+// Rollback.SourcePhase: local grants held-timers, configured held-config,

+// held-release, restore and install (the installer restores inside

+// configured, before the CAS that writes restored, G8; restored grants

+// nothing), each under the claim at claimed, for a dirty rollback, with the holder at the source's

// pre-move stamp S (op.journal.source_stamp); probing grants the held

// config render and the release materialization before every probe

// (§7.7) and spawn, under the claim at probing. reverting is keyed by

@@ -188,11 +191,13 @@ const (

)

// movesPhaseRights is the landing's table: the actions each phase permits

-// (§7.6). A phase or action not listed is refused.

+// (§7.6), keyed by the phase whose work makes the write. configured renders,

+// gates the release and installs the database; its closing CAS writes

+// restored (G8), so restored grants nothing. A phase or action not listed is

+// refused.

var movesPhaseRights = map[string]map[string]bool{

provision.LandLocal: {movesActionHeldTimers: true},

- provision.LandConfigured: {movesActionHeldConfig: true, movesActionHeldRelease: true},

- provision.LandRestored: {movesActionRestore: true, movesActionInstall: true},

+ provision.LandConfigured: {movesActionHeldConfig: true, movesActionHeldRelease: true, movesActionRestore: true, movesActionInstall: true},

}

// movesPolicyActions is every action the policy knows. Anything else is

@@ -322,14 +327,16 @@ type recoveryRow struct {

}

// restoringRights is the source's table during a rollback, keyed by the

-// record's Rollback.SourcePhase (§7.8 source table). The claim stays at

-// claimed through the dirty return's install (Task 3) and mirrors probing

-// (J1). A source phase not listed grants nothing.

+// record's Rollback.SourcePhase (§7.8 source table) at the phase whose work

+// makes the write: the installer restores and installs inside configured,

+// before the CAS that writes restored (G8, J24), so restored grants nothing.

+// The claim stays at claimed through the dirty return's install (Task 3) and

+// mirrors probing (J1). A source phase not listed grants nothing.

var restoringRights = map[string]recoveryRow{

- provision.RbSourceLocal: {claimPhase: hostclaim.PhaseClaimed, install: true, actions: map[string]bool{movesActionHeldTimers: true}},

- provision.RbSourceConfigured: {claimPhase: hostclaim.PhaseClaimed, install: true, actions: map[string]bool{movesActionHeldConfig: true, movesActionHeldRelease: true}},

- provision.RbSourceRestored: {claimPhase: hostclaim.PhaseClaimed, install: true, actions: map[string]bool{movesActionRestore: true, movesActionInstall: true}},

- provision.RbSourceProbing: {claimPhase: hostclaim.PhaseProbing, actions: map[string]bool{movesActionHeldConfig: true, movesActionHeldRelease: true, movesActionSpawn: true}},

+ provision.RbSourceLocal: {claimPhase: hostclaim.PhaseClaimed, install: true, actions: map[string]bool{movesActionHeldTimers: true}},

+ provision.RbSourceConfigured: {claimPhase: hostclaim.PhaseClaimed, install: true, actions: map[string]bool{

+ movesActionHeldConfig: true, movesActionHeldRelease: true, movesActionRestore: true, movesActionInstall: true}},

+ provision.RbSourceProbing: {claimPhase: hostclaim.PhaseProbing, actions: map[string]bool{movesActionHeldConfig: true, movesActionHeldRelease: true, movesActionSpawn: true}},

}

// revertingRights is the source's table during a revert, keyed by the

diff --git a/internal/host/movespolicy_test.go b/internal/host/movespolicy_test.go

index 490e05cc0..1cce1d76f 100644

--- a/internal/host/movespolicy_test.go

+++ b/internal/host/movespolicy_test.go

@@ -98,8 +98,7 @@ var mpActions = []string{"held-config", "held-release", "held-timers", "restore"

func TestMovesClaimPolicyGrantsByPhase(t *testing.T) {

granted := map[string]map[string]bool{

provision.LandLocal: {"held-timers": true},

- provision.LandConfigured: {"held-config": true, "held-release": true},

- provision.LandRestored: {"restore": true, "install": true},

+ provision.LandConfigured: {"held-config": true, "held-release": true, "restore": true, "install": true},

}

phases := append(provision.LandingOrder(), "committed")

moves, reader := mpPolicy()

@@ -512,7 +511,7 @@ func TestMovesClaimPolicyBindsHeldInputsToThePins(t *testing.T) {

})

t.Run("the record is read for the held config and release only", func(t *testing.T) {

moves, reader := mpPolicy()

- for phase, action := range map[string]string{provision.LandLocal: "held-timers", provision.LandRestored: "restore", provision.LandChecking: "spawn"} {

+ for phase, action := range map[string]string{provision.LandLocal: "held-timers", provision.LandConfigured: "restore", provision.LandRestored: "install", provision.LandChecking: "spawn"} {

c := mpClaim(hostclaim.KindLanding, phase)

moves(t.Context(), c, mpSelection(c, action), hostwrite.Identity{})

}

@@ -807,16 +806,16 @@ func rpRefused(t *testing.T, err error, reason string) {

// restoring, the source's return (J12): every action against every source

// phase of the rollback, under the claim phase that phase runs under. The

-// dirty install's held writes are granted at their phases (timers at local,

-// config and release at configured, restore and install at restored), and

-// the held config render, the release materialization and the spawn at

-// probing (§7.7); nothing else. A record

-// phase under the wrong claim phase is refused.

+// dirty install's held writes are granted at the phases that write them

+// (timers at local; config, release, restore and install at configured,

+// whose closing CAS writes restored, G8 and the final review's finding 3),

+// and the held config render, the release materialization and the spawn at

+// probing (§7.7); nothing else, restored included. A record phase under the

+// wrong claim phase is refused.

func TestMovesClaimPolicyRestoringRows(t *testing.T) {

granted := map[string]map[string]bool{

provision.RbSourceLocal: {"held-timers": true},

- provision.RbSourceConfigured: {"held-config": true, "held-release": true},

- provision.RbSourceRestored: {"restore": true, "install": true},

+ provision.RbSourceConfigured: {"held-config": true, "held-release": true, "restore": true, "install": true},

provision.RbSourceProbing: {"held-config": true, "held-release": true, "spawn": true},

}

grants := 0

@@ -851,7 +850,7 @@ func TestMovesClaimPolicyRestoringRows(t *testing.T) {

for _, tc := range []struct{ record, claim, action string }{

{provision.RbSourceLocal, hostclaim.PhaseProbing, "held-timers"},

{provision.RbSourceConfigured, hostclaim.PhaseReleased, "held-config"},

- {provision.RbSourceRestored, hostclaim.PhaseProbed, "install"},

+ {provision.RbSourceConfigured, hostclaim.PhaseProbed, "install"},

{provision.RbSourceProbing, hostclaim.PhaseClaimed, "spawn"},

{provision.RbSourceProbing, hostclaim.PhaseProbed, "held-config"},

} {

@@ -923,7 +922,7 @@ func TestMovesClaimPolicyRecoveryRefusesOtherRoles(t *testing.T) {

bases := map[string]cell{

"restoring local": restoring(provision.RbSourceLocal, "held-timers"),

"restoring configured": restoring(provision.RbSourceConfigured, "held-config"),

- "restoring restored": restoring(provision.RbSourceRestored, "install"),

+ "restoring install": restoring(provision.RbSourceConfigured, "install"),

"restoring probing": restoring(provision.RbSourceProbing, "spawn"),

"reverting probing": reverting,

}

@@ -1023,7 +1022,11 @@ func TestMovesClaimPolicyRecoveryRefusesOtherRoles(t *testing.T) {

rpRefused(t, moves(t.Context(), c, rpSelection(c, "spawn"), rpIdentity("spawn")), "")

})

t.Run("the dirty install rows need a dirty rollback", func(t *testing.T) {

- for phase, action := range map[string]string{provision.RbSourceLocal: "held-timers", provision.RbSourceConfigured: "held-config", provision.RbSourceRestored: "restore"} {

+ for _, tc := range []struct{ phase, action string }{

+ {provision.RbSourceLocal, "held-timers"}, {provision.RbSourceConfigured, "held-config"},

+ {provision.RbSourceConfigured, "restore"}, {provision.RbSourceConfigured, "install"},

+ } {

+ phase, action := tc.phase, tc.action

rec := rpRestoringRecord(phase)

rec.Move.Rollback.Dirty = false

c := rpRestoringClaim(hostclaim.PhaseClaimed)

@@ -1210,12 +1213,14 @@ func TestMovesClaimPolicyRestoringThroughPolicyCheck(t *testing.T) {

p := hostbox.Policy{Moves: MovesClaimPolicy((&mpReader{rec: rec}).read), Held: func(context.Context, string) (string, error) { return hold, nil }}

return p.Check(t.Context(), c, s, id)

}

- for phase, action := range map[string]string{

- provision.RbSourceLocal: "held-timers",

- provision.RbSourceConfigured: "held-config",

- provision.RbSourceRestored: "install",

- provision.RbSourceProbing: "spawn",

+ for _, tc := range []struct{ phase, action string }{

+ {provision.RbSourceLocal, "held-timers"},

+ {provision.RbSourceConfigured, "held-config"},

+ {provision.RbSourceConfigured, "restore"},

+ {provision.RbSourceConfigured, "install"},

+ {provision.RbSourceProbing, "spawn"},

} {

+ phase, action := tc.phase, tc.action

c := rpRestoringClaim(rpClaimPhase(phase))

s := rpSelection(c, action)

if err := through(rpRB, rpRestoringRecord(phase), c, s, rpIdentity(action)); err != nil {

@@ -1237,6 +1242,14 @@ func TestMovesClaimPolicyRestoringThroughPolicyCheck(t *testing.T) {

if err := through(rpRB, rpRestoringRecord(provision.RbSourceConfigured), c, s, hostwrite.Identity{}); !errors.Is(err, ErrMovesRefused) {

t.Fatalf("held-timers at configured: %v", err)

}

+ // restored is written by the CAS that ends configured (G8): the install

+ // is done, and nothing is restored or installed there.

+ for _, action := range []string{"restore", "install"} {

+ s = rpSelection(c, action)

+ if err := through(rpRB, rpRestoringRecord(provision.RbSourceRestored), c, s, hostwrite.Identity{}); !errors.Is(err, ErrMovesRefused) {

+ t.Fatalf("%s at restored: %v", action, err)

+ }

}

// The release is materialized before every recovery probe (§7.7; Task 3b

diff --git a/internal/host/movesretire.go b/internal/host/movesretire.go

index 9dd91bf26..81e6f0d30 100644

--- a/internal/host/movesretire.go

+++ b/internal/host/movesretire.go

@@ -40,6 +40,12 @@ type localArtifactPark struct {

NewID func() string

// Fault is the test seam, after the plan and after each rename.

Fault func(point string) error

+ // Owns, when set, is asked about each candidate present when the plan

+ // is made, with its identity: false records it in the plan as not this

+ // park's (Unowned), so it is logged and left where it is, then and on

+ // every replay. The rollback's cleaning passes its ownership evidence

+ // here (the final review's finding 1); nil owns every candidate.

+ Owns func(path, id string) (bool, error)

}

// localParkPlan is what is journalled before the first rename: the

@@ -53,6 +59,10 @@ type localParkPlan struct {

type localParkItem struct {

Path string `json:"path"`

ID string `json:"id,omitempty"`

+ // Unowned is a candidate that was present when the plan was made but

+ // that the park has no record of creating (localArtifactPark.Owns): ID

+ // is empty, and it is never moved.

+ Unowned bool `json:"unowned,omitempty"`

}

// localParkReceipt sits beside artifacts/ in the retirement, so a later

@@ -123,9 +133,10 @@ func artifactPathOK(p string) bool {

// a fresh rename is (a stop between the rename and its syncs leaves the

// rename undurable, fix round 1), and is then done;

// - one still at its live path with its identity is renamed;

-// - one recorded absent, or whose live path now holds something with

-// another identity, is not this park's: it is left where it is, never

-// moved or removed;

+// - one recorded absent, or unowned (present, but with no record that

+// this park's op created it), or whose live path now holds something

+// with another identity, is not this park's: it is left where it is,

+// never moved or removed;

// - a destination that holds anything else is an error, and nothing more

// is moved.

func parkLocalArtifacts(ctx context.Context, j *moves.Journal, mark func(step, value string) error, p localArtifactPark) (string, error) {

@@ -187,6 +198,10 @@ func parkLocalArtifacts(ctx context.Context, j *moves.Journal, mark func(step, v

return "", err

}

src, dst := filepath.Join(p.DataDir, it.Path), filepath.Join(dest, it.Path)

+ if it.Unowned {

+ log.Printf("moves: %s: %s has no record that %s created it; it is left where it is", p.Host, it.Path, p.Op)

+ continue

+ }

if it.ID == "" {

if _, err := os.Lstat(src); err == nil {

log.Printf("moves: %s: %s appeared after the park recorded it absent; it is not this park's and is left where it is", p.Host, it.Path)

@@ -243,7 +258,8 @@ func parkLocalArtifacts(ctx context.Context, j *moves.Journal, mark func(step, v

return plan.Retirement, nil

}

-// plan records a fresh retirement id and each candidate's identity. The

+// plan records a fresh retirement id and each candidate's identity, or, for

+// a present candidate Owns refuses, that it is not this park's. The

// retirement's directory must not exist yet: the id is this park's alone.

func (p localArtifactPark) plan() (localParkPlan, error) {

plan := localParkPlan{Retirement: p.NewID(), Incarnation: p.Incarnation}

@@ -263,6 +279,16 @@ func (p localArtifactPark) plan() (localParkPlan, error) {

if err != nil {

return plan, err

}

+ if id != "" && p.Owns != nil {

+ owned, err := p.Owns(path, id)

+ if err != nil {

+ return plan, fmt.Errorf("the ownership of %s: %w", path, err)

+ }

+ if !owned {

+ plan.Items = append(plan.Items, localParkItem{Path: path, Unowned: true})

+ continue

+ }

plan.Items = append(plan.Items, localParkItem{Path: path, ID: id})

}

return plan, nil

diff --git a/internal/host/movesrollback.go b/internal/host/movesrollback.go

index d2c1e447f..3cef68bcc 100644

--- a/internal/host/movesrollback.go

+++ b/internal/host/movesrollback.go

@@ -14,6 +14,7 @@ import (

"strings"

"time"

+ "github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/hostclaim"

"github.com/carlosframework/platform/internal/hostguard"

"github.com/carlosframework/platform/internal/lease"

@@ -236,6 +237,12 @@ func (w *movesWorker) rollbackTarget(ctx context.Context, s release.Store, rec p

if m == nil || m.Legacy() || m.State != provision.MoveRollingBack || m.Rollback == nil || m.To != cfg.Region || m.From == cfg.Region {

return nil

}

+ // The landing is abandoned: a location lease its build still holds on

+ // this worker (kept across passes from building through published) is

+ // released on entry, whoever decided the rollback, so the cleaning's

+ // park engine can take it (the final review's finding 2). Idempotent:

+ // nothing held is nothing to release.

+ releaseOwnLease(w, rec.Host)

if !rollbackTargetArmed(cfg) {

w.notice(rec.Host+"#rollback", "moves: %s: the target's half of rollback %s waits: this box's rollback is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.Rollback.ID)

return nil

@@ -1253,12 +1260,18 @@ func (r *rollbackRun) installation(ctx context.Context, ours bool) (string, erro

}

// parkLeftovers is the moves-owned rename (H5(b), parkLocalArtifacts) of

-// every artifact the target still has at the flat paths (the installation's,

-// the rendered config and the sidecar's env), its plan and its renames inside

-// the host guard, which retires the installation record at F if it is still

-// active (an unregistered target), with the claim read fresh there: this box

-// no longer executes and no longer takes part in the claim's op, and no

-// inventory entry claims a path the rename would move.

+// the artifacts this op created that the target still has at the flat paths,

+// its plan and its renames inside the host guard, which retires the

+// installation record at F if it is still active (an unregistered target),

+// with the claim read fresh there: this box no longer executes and no longer

+// takes part in the claim's op, and no inventory entry claims a path the

+// rename would move.

+//

+// The candidates are the installation's paths, the rendered config and the

+// sidecar's env (sourceArtifactPaths), but a candidate is moved only on a

+// record that this op created it (rollbackOwnership; the final review's

+// finding 1). Anything else, a files directory or env the landing refused as

+// occupied (RollbackFilesOccupied) included, is logged and left where it is.

func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark func(step, value string) error) error {

if j.Done(rbStepLocal) {

return nil

@@ -1266,6 +1279,16 @@ func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark

cfg := r.cfg()

host, id := r.rec.Host, r.g.ID

paths := sourceArtifactPaths(host)

+ var owns func(path, identity string) (bool, error)

+ if _, planned := j.Steps[rbStepLocal+"-plan"]; !planned {

+ // A journalled plan already recorded what is the op's; the evidence

+ // is read (the config ack is a store read) only before the plan.

+ ev, err := r.ownership(ctx, j)

+ if err != nil {

+ return err

+ }

+ owns = ev.owns

+ }

var incarnation string

check := func(c *hostclaim.Claim) error {

if c.Op.Exec == cfg.Box {

@@ -1291,7 +1314,7 @@ func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark

return fmt.Errorf("move %s: the claim names another incarnation", id)

}

_, err := parkLocalArtifacts(ctx, j, mark, localArtifactPark{DataDir: cfg.DataDir, Incarnation: incarnation, Host: host,

- Op: id, Step: rbStepLocal, Paths: paths, NewID: obligationID, Fault: r.fault})

+ Op: id, Step: rbStepLocal, Paths: paths, NewID: obligationID, Fault: r.fault, Owns: owns})

return err

}

g := r.guard()

@@ -1312,6 +1335,93 @@ func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark

return g.RetireInstallationWith(ctx, host, hostguard.Captured{Incarnation: r.g.Incarnation, Stamp: f}, check, work)

}

+// rollbackOwnership is the record that this op created a path the target's

+// cleaning may park (the final review's finding 1). The registered

+// installation, the paths the landing listed under F, is park.Local's and is

+// retired before the moves-owned rename runs; what is left at the flat paths

+// is parked only on one of these:

+//

+// - the files directory and <host>.env: the landing's placement receipt

+// (files_placed), or before it the staged extraction's identities

+// (staged), which a rename keeps: the live path must have that identity;

+// - the database, its marker, WAL, shared memory, quarantine and

+// litestream shadow: the landing installed its database, as the record

+// says (landing.installed_db_sha256, stamped by the CAS that ends

+// configured); the install refused a WAL or shared memory beside an

+// absent database, so what sits beside it since is the op's instance's;

+// - the rendered config, <host>.config.env: the held render's ack on this

+// box names this op, and the file's sha256 is the one the ack recorded;

+// - the sidecar's env: never, since a moved host has no sidecar.

+type rollbackOwnership struct {

+ host, dataDir string

+ placed stagedFiles // the landing's receipt, or its staged identities

+ installed bool

+ configSHA256 string // the held render's ack, "sha256:<hex>", or ""

+// ownership reads the evidence: the landing's journal, the record and the

+// held render's config ack on this box.

+func (r *rollbackRun) ownership(ctx context.Context, j *moves.Journal) (rollbackOwnership, error) {

+ cfg := r.cfg()

+ host, id := r.rec.Host, r.g.ID

+ ev := rollbackOwnership{host: host, dataDir: cfg.DataDir}

+ for _, step := range []string{landStepFilesPlaced, landStepStaged} {

+ if _, ok := j.Steps[step]; !ok {

+ continue

+ }

+ placed, err := journalledStaged(j, step)

+ if err != nil {

+ return ev, err

+ }

+ ev.placed = placed

+ break

+ }

+ if l := r.rec.Move.Landing; l != nil && l.InstalledDBSHA256 != "" {

+ ev.installed = true

+ }

+ body, _, err := r.s.Get(ctx, appconfig.ConfigAckKey(cfg.Box, host))

+ switch {

+ case errors.Is(err, os.ErrNotExist):

+ case err != nil:

+ return ev, fmt.Errorf("read the held render's config ack: %w", err)

+ default:

+ var a appconfig.ConfigAck

+ if err := json.Unmarshal(body, &a); err != nil {

+ log.Printf("moves: %s: the config ack is unreadable (%v); the rendered config is not counted as %s's", host, err, id)

+ } else if a.MoveOp == id && a.Box == cfg.Box && a.Host == host && a.FileSHA256 != "" {

+ ev.configSHA256 = "sha256:" + a.FileSHA256

+ }

+ return ev, nil

+// owns answers localArtifactPark.Owns for the target's leftovers.

+func (ev rollbackOwnership) owns(path, identity string) (bool, error) {

+ db := ev.host + ".db"

+ switch path {

+ case ev.host:

+ return ev.placed.Files != "" && identity == ev.placed.Files, nil

+ case ev.host + ".env":

+ return ev.placed.Env != "" && identity == ev.placed.Env, nil

+ case ev.host + ".config.env":

+ if ev.configSHA256 == "" {

+ return false, nil

+ }

+ full := filepath.Join(ev.dataDir, path)

+ if fi, err := os.Lstat(full); err != nil || !fi.Mode().IsRegular() {

+ return false, nil // a link or anything else is not the render's file

+ }

+ got, err := fileSHA256(full)

+ if err != nil {

+ return false, err

+ }

+ return got == ev.configSHA256, nil

+ case db, db + ".gen", db + "-wal", db + "-shm", db + ".corrupt", ".activator-" + db + "-litestream":

+ return ev.installed, nil

+ }

+ return false, nil

// deleteRow deletes the row and its timers rows in one registry transaction

// (registry.DeleteIfHoldWithTimers, J12), only while the rollback holds the

// row, under the host's hold lock with the record re-read there; the journal

diff --git a/internal/host/movesrollback_finalfix_test.go b/internal/host/movesrollback_finalfix_test.go

new file mode 100644

index 000000000..24ca5662e

--- /dev/null

+++ b/internal/host/movesrollback_finalfix_test.go

@@ -0,0 +1,435 @@

+package host

+import (

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "errors"

+ "fmt"

+ "io/fs"

+ "os"

+ "path/filepath"

+ "sort"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/appconfig"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostguard"

+ "github.com/carlosframework/platform/internal/hostwrite"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+// The final whole-branch review of Plan 4 (final-p4a-review.md, rulings in

+// final-fix-rulings.md): the target's cleaning moves only what this op

+// created, an abandoned build's location lease is released when the

+// rollback starts, and the install's restore and install rights are granted

+// at the phase that performs them.

+// treeSnapshot records every entry under the named paths (relative to dir)

+// by identity, mode and bytes, so a test can tell a path left in place from

+// one moved away and back or rewritten.

+func treeSnapshot(t *testing.T, dir string, names ...string) map[string]string {

+ t.Helper()

+ out := map[string]string{}

+ for _, name := range names {

+ err := filepath.WalkDir(filepath.Join(dir, name), func(p string, d fs.DirEntry, err error) error {

+ if err != nil {

+ return err

+ }

+ rel, _ := filepath.Rel(dir, p)

+ fi, err := os.Lstat(p)

+ if err != nil {

+ return err

+ }

+ id, err := pathIdentity(p)

+ if err != nil {

+ return err

+ }

+ body := ""

+ if fi.Mode().IsRegular() {

+ b, err := os.ReadFile(p)

+ if err != nil {

+ return err

+ }

+ body = string(b)

+ }

+ out[rel] = fmt.Sprintf("%s %s %q", id, fi.Mode(), body)

+ return nil

+ })

+ if err != nil {

+ t.Fatalf("snapshot %s: %v", name, err)

+ }

+ return out

+func sameSnapshot(a, b map[string]string) bool {

+ if len(a) != len(b) {

+ return false

+ }

+ for k, v := range a {

+ if b[k] != v {

+ return false

+ }

+ return true

+// cleanTarget drives a rolling_back record from the decision to cleaned on

+// the target: the release, the source's recovery, the cleaning.

+func (h *rollbackHarness) cleanTarget(t *testing.T) *provision.MoveRollback {

+ t.Helper()

+ h.rollbackPass(t)

+ if rb := h.rb(t); rb.TargetPhase != provision.RbTargetReleased && rb.TargetPhase != provision.RbTargetReturned {

+ t.Fatalf("the release: %+v", rb)

+ }

+ h.recovered(t)

+ h.rollbackPass(t)

+ rb := h.rb(t)

+ if rb.TargetPhase != provision.RbTargetCleaned || rb.TargetParkedAt == nil {

+ t.Fatalf("the cleaning did not complete: %+v (lease table %v)", rb, h.leases.holding())

+ }

+ return rb

+// Finding 1: files the landing refused as foreign (an unregistered files

+// directory and env file, RollbackFilesOccupied) stay where they are, byte

+// for byte, through the rollback's cleaning, which still completes.

+func TestRollbackCleaningKeepsWhatTheLandingRefused(t *testing.T) {

+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}

+ h.flip(t)

+ h.sentinels(t)

+ foreign := []string{p1Host, p1Host + ".env"}

+ before := treeSnapshot(t, h.tdir, foreign...)

+ h.land(t)

+ if m := h.read(t, p1Host).Move; m.Reason != provision.RollbackFilesOccupied || m.LandingPhase != provision.LandLocal {

+ t.Fatalf("the landing did not refuse the foreign files: %+v", m)

+ }

+ h.rollbackPass(t) // the decision converts the mark

+ rolledBack(t, h.landingHarness, provision.RollbackFilesOccupied, provision.LandLocal)

+ h.cleanTarget(t)

+ h.sentinelsIntact(t, "after the cleaning")

+ if after := treeSnapshot(t, h.tdir, foreign...); !sameSnapshot(before, after) {

+ t.Fatalf("the foreign files changed:\nbefore %v\nafter %v", before, after)

+ }

+ // Nothing of theirs is in the parked tree, and the op's journal is.

+ var parkedForeign []string

+ filepath.WalkDir(filepath.Join(h.tdir, ".parked"), func(p string, d fs.DirEntry, err error) error {

+ if err == nil && (filepath.Base(p) == "sentinel.txt" || filepath.Base(p) == p1Host+".env") {

+ parkedForeign = append(parkedForeign, p)

+ }

+ return nil

+ })

+ if len(parkedForeign) != 0 {

+ t.Fatalf("foreign files parked: %v", parkedForeign)

+ }

+ if _, err := os.Lstat(moves.JournalDir(h.tdir, p1Host, p1ID)); !errors.Is(err, os.ErrNotExist) {

+ t.Fatalf("the op journal is still live: %v", err)

+ }

+// Finding 1, the other half: what the op created is still parked by the

+// cleaning, on its ownership evidence, and what it did not create is left.

+// The landing placed its files and env (the placement receipt) and stopped

+// before it registered them under F; a WAL and a rendered config were

+// there that it never wrote. The cleaning parks the op's two and leaves the

+// other two. Once the landing has installed its database and rendered its

+// config (the restored CAS and the render's ack), the config and the

+// database's WAL are the op's and are parked.

+func TestRollbackCleaningParksWhatTheOpCreated(t *testing.T) {

+ t.Run("placed, not registered", func(t *testing.T) {

+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}

+ h.flip(t)

+ db := provision.DefaultDBPath(h.tdir, p1Host)

+ for name, body := range map[string]string{db + "-wal": "a wal the landing never wrote", filepath.Join(h.tdir, p1Host+".config.env"): "OTHER=1\n"} {

+ if err := os.WriteFile(name, []byte(body), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ foreign := []string{p1Host + ".db-wal", p1Host + ".config.env"}

+ before := treeSnapshot(t, h.tdir, foreign...)

+ h.landAt = "after-extracted"

+ h.crashLand(t)

+ j := h.landJournal(t)

+ if !j.Done(landStepFilesPlaced) || j.Done(landStepInventory) {

+ t.Fatalf("the fixture: %v", j.Steps)

+ }

+ if c := h.claim(t); hostclaim.InventoryHasStamp(c, "uk-box", c.Holder.Stamp) {

+ t.Fatal("the fixture registered the landing's paths")

+ }

+ h.rollBack(t)

+ h.cleanTarget(t)

+ for _, p := range []string{p1Host, p1Host + ".env"} {

+ if _, err := os.Lstat(filepath.Join(h.tdir, p)); !errors.Is(err, os.ErrNotExist) {

+ t.Fatalf("the landing's own %s is still live: %v", p, err)

+ }

+ parked := parkedUnder(t, h.tdir)

+ if !parkedHas(parked, "/artifacts/"+p1Host+"/upload.txt", "hello") {

+ t.Fatalf("the landing's files were not parked: %v", parked)

+ }

+ if after := treeSnapshot(t, h.tdir, foreign...); !sameSnapshot(before, after) {

+ t.Fatalf("what the op did not create changed:\nbefore %v\nafter %v", before, after)

+ }

+ })

+ t.Run("installed and rendered", func(t *testing.T) {

+ h := newRollbackHarness(t)

+ db := provision.DefaultDBPath(h.tdir, p1Host)

+ config := filepath.Join(h.tdir, p1Host+".config.env")

+ if err := os.WriteFile(db+"-wal", []byte("the landing's wal"), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ rendered := []byte("RENDERED=1\n")

+ if err := os.WriteFile(config, rendered, 0o600); err != nil {

+ t.Fatal(err)

+ }

+ sum := sha256.Sum256(rendered)

+ if err := appconfig.WriteConfigAck(context.Background(), h.s, appconfig.ConfigAck{Schema: appconfig.ConfigAckSchema, Box: "uk-box", Host: p1Host,

+ At: h.clock.Now(), MoveOp: p1ID, FileSHA256: hex.EncodeToString(sum[:])}); err != nil {

+ t.Fatal(err)

+ }

+ h.rollBack(t)

+ h.cleanTarget(t)

+ for _, p := range sourceArtifactPaths(p1Host) {

+ if _, err := os.Lstat(filepath.Join(h.tdir, p)); !errors.Is(err, os.ErrNotExist) {

+ t.Fatalf("%s is still live: %v", p, err)

+ }

+ parked := parkedUnder(t, h.tdir)

+ if !parkedHas(parked, "/artifacts/"+p1Host+".db-wal", "the landing's wal") || !parkedHas(parked, "/artifacts/"+p1Host+".config.env", "RENDERED=1\n") {

+ t.Fatalf("the op's leftovers were not parked: %v", parked)

+ }

+ })

+ t.Run("a config the render did not write", func(t *testing.T) {

+ h := newRollbackHarness(t)

+ config := filepath.Join(h.tdir, p1Host+".config.env")

+ if err := os.WriteFile(config, []byte("ANOTHER=1\n"), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ // The ack names this op, but another file's bytes.

+ sum := sha256.Sum256([]byte("RENDERED=1\n"))

+ if err := appconfig.WriteConfigAck(context.Background(), h.s, appconfig.ConfigAck{Schema: appconfig.ConfigAckSchema, Box: "uk-box", Host: p1Host,

+ At: h.clock.Now(), MoveOp: p1ID, FileSHA256: hex.EncodeToString(sum[:])}); err != nil {

+ t.Fatal(err)

+ }

+ before := treeSnapshot(t, h.tdir, p1Host+".config.env")

+ h.rollBack(t)

+ h.cleanTarget(t)

+ if after := treeSnapshot(t, h.tdir, p1Host+".config.env"); !sameSnapshot(before, after) {

+ t.Fatalf("a config the render did not write changed:\nbefore %v\nafter %v", before, after)

+ }

+ })

+// parkedUnder maps every regular file under dir's .parked/ to its body.

+func parkedUnder(t *testing.T, dir string) map[string]string {

+ t.Helper()

+ out := map[string]string{}

+ err := filepath.WalkDir(filepath.Join(dir, ".parked"), func(p string, d fs.DirEntry, err error) error {

+ if err != nil || d.IsDir() {

+ return err

+ }

+ body, err := os.ReadFile(p)

+ if err != nil {

+ return err

+ }

+ out[p] = string(body)

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ return out

+// Finding 2: a build that failed leaves the location's lease held across

+// passes, as the landing designs; once the landing is rolled back that

+// lease is abandoned and is released, whether the rollback is decided by

+// this worker's landing pass or outside it. The same worker then cleans,

+// and its park engine acquires the location's lease.

+func TestRollbackReleasesTheAbandonedBuildLease(t *testing.T) {

+ failedBuild := func(t *testing.T) *rollbackHarness {

+ t.Helper()

+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}

+ h.flip(t)

+ h.landAt = "after-build"

+ h.crashLand(t)

+ if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandBuilding {

+ t.Fatalf("the fixture: %+v", m)

+ }

+ if held := h.leases.holding(); len(held) != 1 {

+ t.Fatalf("the failed build holds %v, want the location's lease", held)

+ }

+ // The same worker carries on: its lease map is the build's.

+ h.rw = h.lw

+ return h

+ }

+ t.Run("decided in the landing pass", func(t *testing.T) {

+ h := failedBuild(t)

+ h.rollBack(t)

+ if held := h.leases.holding(); len(held) != 0 {

+ t.Fatalf("the rollback kept the build's lease: %v", held)

+ }

+ h.cleanTarget(t)

+ })

+ t.Run("decided outside the landing pass", func(t *testing.T) {

+ h := failedBuild(t)

+ // Another worker decides the rollback (the decision is not this

+ // worker's pass); the target's next pass enters the rollback.

+ h.clock.Advance(cutLandingWindow + time.Minute)

+ other := &movesWorker{mirrors: map[string]lease.MirrorStore{}, cfg: h.rollbackCfg(t)}

+ if done, err := other.rollBackLanding(context.Background(), h.s, h.read(t, p1Host)); !done || err != nil {

+ t.Fatalf("the outside decision: %v %v", done, err)

+ }

+ if held := h.leases.holding(); len(held) != 1 {

+ t.Fatalf("the fixture: %v", held)

+ }

+ h.rollbackPass(t)

+ if held := h.leases.holding(); len(held) != 0 {

+ t.Fatalf("the rollback's entry kept the build's lease: %v", held)

+ }

+ h.rollbackPass(t) // idempotent: nothing held, nothing to release

+ h.cleanTarget(t)

+ })

+// policyProbe checks, from inside an installer's own write hooks, that each

+// write it makes is one hostbox.Policy.Check grants with MovesClaimPolicy as

+// Policy.Moves: the selection is made as hostbox.WorkSource.Select makes it

+// (the live claim's op, kind and phase, the installation record's capture),

+// the row's hold is the registry's, and the record is read from the store.

+type policyProbe struct {

+ t *testing.T

+ s release.Store

+ dataDir, box, region string

+ hold func() string

+ phase func(*provision.Record) string

+ granted, refused []string

+func (p *policyProbe) check(action string, id hostwrite.Identity) {

+ ctx := context.Background()

+ obs, err := hostclaim.Read(ctx, p.s, p1Host)

+ if err != nil || obs.Absent {

+ p.t.Errorf("read the claim: %v", err)

+ return

+ }

+ captured, err := hostguard.Capture(p.dataDir, p1Host)

+ if err != nil {

+ p.t.Errorf("capture: %v", err)

+ return

+ }

+ rec, err := provision.GetRecord(ctx, p.s, "bab", "app", p1Host)

+ if err != nil {

+ p.t.Errorf("read the record: %v", err)

+ return

+ }

+ c := obs.Claim

+ sel := hostbox.Selection{Host: p1Host, Action: action, Box: p.box, Region: p.region, Captured: captured, Kind: c.Op.Kind, Op: c.Op.ID,

+ Phase: c.Op.Phase, Generation: c.Holder.Generation, Session: c.Holder.Session}

+ pol := hostbox.Policy{Moves: MovesClaimPolicy(MovesRecordsFrom(p.s)), Held: func(context.Context, string) (string, error) { return p.hold(), nil }}

+ cell := p.phase(rec) + "/" + action

+ if err := pol.Check(ctx, c, sel, id); err != nil {

+ p.refused = append(p.refused, cell+": "+err.Error())

+ return

+ }

+ p.granted = append(p.granted, cell)

+// wrap puts the probe in front of the installer's write hooks: the timers

+// import (held-timers), the held render (held-config), the release gate

+// (held-release) and the generation marker the database install writes

+// inside its guard (restore and install).

+func (p *policyProbe) wrap(c *moveCfg) {

+ timers, render, ready, gen := c.ReplaceTimers, c.RenderHeld, c.ReleaseReady, c.WriteGen

+ c.ReplaceTimers = func(ctx context.Context, host string, rows []registry.Timer) error {

+ p.check(movesActionHeldTimers, hostwrite.Identity{})

+ return timers(ctx, host, rows)

+ }

+ c.RenderHeld = func(ctx context.Context, s release.Store, rt registry.Route, op string) (appconfig.HeldRender, error) {

+ hr, err := render(ctx, s, rt, op)

+ if err == nil {

+ p.check(movesActionHeldConfig, HeldConfigIdentity(hr.BundleETags, hr.OverlayETag))

+ }

+ return hr, err

+ }

+ c.ReleaseReady = func(ctx context.Context, s release.Store, rec provision.Record, v string) moves.Check {

+ p.check(movesActionHeldRelease, hostwrite.Identity{Release: v})

+ return ready(ctx, s, rec, v)

+ }

+ c.WriteGen = func(path, generation string) error {

+ p.check(movesActionRestore, hostwrite.Identity{})

+ p.check(movesActionInstall, hostwrite.Identity{})

+ return gen(path, generation)

+ }

+func (p *policyProbe) want(t *testing.T, cells ...string) {

+ t.Helper()

+ if len(p.refused) != 0 {

+ t.Fatalf("Policy.Check refused the installer's own writes:\n%s", strings.Join(p.refused, "\n"))

+ }

+ got := map[string]bool{}

+ for _, c := range p.granted {

+ got[c] = true

+ }

+ var missing []string

+ for _, c := range cells {

+ if !got[c] {

+ missing = append(missing, c)

+ }

+ sort.Strings(missing)

+ if len(missing) != 0 {

+ t.Fatalf("never checked %v (granted %v)", missing, p.granted)

+ }

+// Finding 3 (J24, G8): the dirty return's installer, end to end, writes

+// only what hostbox.Policy.Check grants under MovesClaimPolicy at the phase

+// it writes in, with its row held by the rollback id: held-timers at local;

+// held-config, held-release, restore and install at configured, whose

+// closing CAS writes restored.

+func TestDirtyReturnWritesPassThePolicy(t *testing.T) {

+ h := newReturnHarness(t, true)

+ p := &policyProbe{t: t, s: h.s, dataDir: h.sdir, box: "ie-box", region: "ie", hold: func() string { return h.srcRoute.HoldOp },

+ phase: func(rec *provision.Record) string { return rec.Move.Rollback.SourcePhase }}

+ p.wrap(&h.rcfg)

+ pinned := h.read(t, p1Host).Move.Inputs

+ if refused, err := h.install(t); err != nil || refused != "" {

+ t.Fatalf("install: %q %v", refused, err)

+ }

+ h.installedReturn(t, pinned)

+ p.want(t, "local/held-timers", "configured/held-config", "configured/held-release", "configured/restore", "configured/install")

+// The landing's rows had the same off-by-one: its installer restores and

+// installs inside configured too. Its writes, end to end, pass

+// hostbox.Policy.Check under MovesClaimPolicy with the row held by the move.

+func TestLandingWritesPassThePolicy(t *testing.T) {

+ h := newLandingHarness(t)

+ h.flip(t)

+ p := &policyProbe{t: t, s: h.s, dataDir: h.tdir, box: "uk-box", region: "uk", hold: func() string { return h.route.HoldOp },

+ phase: func(rec *provision.Record) string { return rec.Move.LandingPhase }}

+ if h.lw == nil {

+ h.lw = &movesWorker{mirrors: map[string]lease.MirrorStore{}}

+ }

+ for i := 0; i < 4 && h.read(t, p1Host).Move.LandingPhase != provision.LandRestored; i++ {

+ h.lw.cfg = h.landingCfg(t)

+ p.wrap(&h.lw.cfg)

+ if err := movesPass(context.Background(), h.s, h.lw); err != nil {

+ t.Fatal(err)

+ }

+ if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandRestored {

+ t.Fatalf("the landing did not install: %+v", m)

+ }

+ p.want(t, "local/held-timers", "configured/held-config", "configured/held-release", "configured/restore", "configured/install")