42ae89c74 · carlos/platform

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index 15293fe12..e968b0612 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -39,6 +39,7 @@ const movesCopyRecordPath = "docs/superpowers/plans/2026-10-06-moves-phase1-copy

// "Gate and copy" task appends its ids here as it appends its rows to the

// record. The record and this literal must agree both ways.

var movesCopyIDs = []string{

+ "MV-N391",

"HC5-N1", "HC5-N2", "HC5-N3",

"M-C1", "M-C2", "M-C3", "M-C4", "M-C5", "M-C6", "M-C7", "M-C8", "M-C9",

"M-C10", "M-C11", "M-C12", "M-C13", "M-C14", "M-C15", "M-C16", "M-C17",

@@ -1150,6 +1151,7 @@ func TestMovesTextAppliedPlaceholders(t *testing.T) {

// copy" task appends the files it touched; TestMovesScanFilesCoverTheDiff

// fails when the list falls behind.

var movesScanFiles = []string{

+ "internal/hostops/move_fence_release.go",

"internal/host/movessources.go",

"internal/host/movesrefusals.go",

"internal/console/movesrecovery.go",

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index a253ae9ed..bba2792f9 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -397,3 +397,9 @@ Built, but the gate could not read the row (3). Paul adopted three notation rule

The 16 rows added after that review (MV-N184 to MV-N186, MV-N300 to MV-N312) were approved by Paul on 2026-10-09 with no edits.

The 58 rows that were still pending on 2026-10-11 (MV-N187, MV-N313 to MV-N337, MV-N319, MV-N350 to MV-N381 and MV-N385 to MV-N390, without the withdrawn MV-N388) were approved by Paul on 2026-10-11 with no edits.

+

+## Claim Plan 12 rev 3

+

+|---|---|---|---|

diff --git a/internal/activator/claimwake.go b/internal/activator/claimwake.go

index 8c344f5fc..d4e0710df 100644

--- a/internal/activator/claimwake.go

+++ b/internal/activator/claimwake.go

@@ -66,6 +66,9 @@ type startPolicy = StartPolicy

// Admit reserves the first serving attempt and binds its resulting cold session.

// Prepared probes instead consume the moves owner's single-use admission.

type ClaimRuntime struct {

+ // ResumedAdmission verifies a journalled recovery that persisted prelaunch

+ // under the existing session during this admission call.

+ ResumedAdmission func(context.Context, hostclaim.Observed, hostclaim.Observed) error

RecoveryAttemptsReady func() bool

FailedAdmission func(context.Context, hostclaim.Observed, error) error

Box string

@@ -301,9 +304,17 @@ func (a *Service) doClaimWake(ctx context.Context, e *actEntry) (*wokenInstance,

if err != nil {

return nil, err

}

- if admitted.Claim == nil || admitted.Claim.Holder == nil || observed.Claim.Holder == nil || admitted.Claim.Holder.Incarnation != observed.Claim.Holder.Incarnation || admitted.Claim.Holder.Session <= observed.Claim.Holder.Session || admitted.Claim.Holder.Generation == observed.Claim.Holder.Generation {

+ if admitted.Claim == nil || admitted.Claim.Holder == nil || observed.Claim.Holder == nil || admitted.Claim.Holder.Incarnation != observed.Claim.Holder.Incarnation {

return nil, hostbox.ErrChanged

}

+ if admitted.Claim.Holder.Session <= observed.Claim.Holder.Session || admitted.Claim.Holder.Generation == observed.Claim.Holder.Generation {

+ if !recoveryProbeKind(observed.Claim.Op.Kind) || observed.ETag == "" || admitted.ETag != observed.ETag || admitted.Host != observed.Host || admitted.Claim.Holder.Session != observed.Claim.Holder.Session || admitted.Claim.Holder.Generation != observed.Claim.Holder.Generation || r.ResumedAdmission == nil {

+ return nil, hostbox.ErrChanged

+ }

+ if err := r.ResumedAdmission(ctx, observed, admitted); err != nil {

+ return nil, err

+ }

e.route = rt

if e.recoveryPolicy != nil {

e.probeAdmitted = admitted

diff --git a/internal/activator/plan12_resume_test.go b/internal/activator/plan12_resume_test.go

new file mode 100644

index 000000000..81a628e4d

--- /dev/null

+++ b/internal/activator/plan12_resume_test.go

@@ -0,0 +1,48 @@

+package activator

+

+import (

+ "context"

+ "errors"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostwrite"

+ "github.com/carlosframework/platform/internal/registry"

+)

+

+func TestPlan12RecoveryResumeReachesPrepare(t *testing.T) {

+ for _, mode := range []string{"verified", "missing", "refused", "changed"} {

+ t.Run(mode, func(t *testing.T) {

+ o := hostclaim.Observed{Host: "x.example", ETag: "etag", Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc", Session: 3, Stamp: 3, Generation: "gen", WriterBox: "box"}, Op: hostclaim.Op{ID: "move", Kind: hostclaim.KindRestoring, Phase: hostclaim.PhaseProbing, Exec: "box"}}}

+ prepared := errors.New("prepare reached")

+ r := &ClaimRuntime{Box: "box", Read: func(context.Context, string) (hostclaim.Observed, error) { return o, nil }, StopForeign: func(context.Context, string) error { return nil }, Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) error { return nil }, Admit: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (hostclaim.Observed, error) {

+ next := o

+ if mode == "changed" {

+ next.ETag = "changed"

+ }

+ return next, nil

+ }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, startPolicy) (ClaimStart, error) {

+ return ClaimStart{}, prepared

+ }, FailedAdmission: func(context.Context, hostclaim.Observed, error) error { return nil }}

+ if mode != "missing" {

+ r.ResumedAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error {

+ if mode == "refused" {

+ return hostbox.ErrChanged

+ }

+ return nil

+ }

+ s := &Service{cfg: Config{HostWrites: hostwrite.Config{Enabled: true}, Claims: r}}

+ _, err := s.doClaimWake(t.Context(), &actEntry{host: o.Host, route: registry.Route{Host: o.Host}, recoveryPolicy: &StartPolicy{Probe: true, MaxStarts: 5, Window: 30 * time.Second}})

+ if mode == "verified" {

+ if !errors.Is(err, prepared) {

+ t.Fatalf("resume did not reach prepare: %v", err)

+ }

+ } else if !errors.Is(err, hostbox.ErrChanged) {

+ t.Fatalf("unverified resume: %v", err)

+ }

+ })

+ }

+}

diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go

index 5866ca06f..ee731d684 100644

--- a/internal/console/claim_authority_census_test.go

+++ b/internal/console/claim_authority_census_test.go

@@ -25,10 +25,10 @@ func claimAuthoritySites(t *testing.T) []string {

"NewClamAVScanner": true, "NewRoute53Writer": true, "BeginExclusive": true,

"ResumeExclusive": true, "RegisterStart": true, "FinishWakeRetryReset": true, "ReservePurge": true,

"ConsoleMove": true, "InspectMoveLoss": true, "DeclareMoveLoss": true,

- "RunRecovery": true, "runMoveRecovery": true, "captureMoveStartBase": true, "resolveUncertain": true, "ConvertMovingToRelocating": true,

+ "RunRecovery": true, "runMoveRecovery": true, "captureMoveStartBase": true, "resolveUncertain": true, "ConvertMovingToRelocating": true, "ReleaseFencedTarget": true, "VerifyMoveTargetFence": true, "ReportAutomaticLosses": true,

// The console's move pass (moves Plan 5 Task 7): every write of the

// move-recovery grant, and the lost target's stamp.

- "ResolveStart": true, "Convert": true, "Claim": true, "Record": true, "StampLostMoveTargetParked": true, "stampLostMoveTarget": true,

+ "ResolveStart": true, "Convert": true, "Claim": true, "Record": true, "StampLostMoveTargetParked": true,

// The recovery pass's move log writer (Plan 6 Task 3, fix round 3

// ruling 1): minted inside RunRecovery for the console's outbox

// drainer; one create-only entry per call, under the entry's own

diff --git a/internal/console/claim_invariant_moves_test.go b/internal/console/claim_invariant_moves_test.go

index 94f14449f..da4263e7d 100644

--- a/internal/console/claim_invariant_moves_test.go

+++ b/internal/console/claim_invariant_moves_test.go

@@ -140,11 +140,11 @@ func TestClaimPublishMovesConsoleIsInertWithTheSwitchOff(t *testing.T) {

// recovery pass skips move claims as main does.

func TestClaimMovesConsoleBindsTheMovePass(t *testing.T) {

off := newClaimCase(t).srv

- if off.cfg.ClaimMoveRecovery != nil || off.cfg.HostClaimRecovery == nil || off.cfg.HostClaimRecovery.Moves != nil || off.stampLostMoveTarget != nil {

+ if off.cfg.ClaimMoveRecovery != nil || off.cfg.HostClaimRecovery == nil || off.cfg.HostClaimRecovery.Moves != nil {

t.Fatal("a moves-off console bound the move pass")

}

on := newClaimCaseWith(t, claimMovesOn).srv

- if on.cfg.ClaimMoveRecovery == nil || on.cfg.HostClaimRecovery == nil || on.cfg.HostClaimRecovery.Moves == nil || on.stampLostMoveTarget == nil {

+ if on.cfg.ClaimMoveRecovery == nil || on.cfg.HostClaimRecovery == nil || on.cfg.HostClaimRecovery.Moves == nil {

t.Fatal("a moves-on console did not bind the move pass")

}

if on.cfg.HostClaimRecovery.MoveFenceAuthority == nil {

diff --git a/internal/console/claim_runtime.go b/internal/console/claim_runtime.go

index f70b793ee..f4fa9ce18 100644

--- a/internal/console/claim_runtime.go

+++ b/internal/console/claim_runtime.go

@@ -115,14 +115,7 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {

// switch on, and only now that every box close is record first

// (ruling M4, P3-6; TestClaimMoveRecoveryWiringWaitsForRecordFirstOwners).

s.cfg.ClaimMoveRecovery = s.movesRecovery

- // The lost target's target_parked_at (p10 Task G) is written

- // outside the grant by the lifecycle record writer, which the

- // conditional record store carries and the data boundary (an app

- // permit's) does not. The pass gets this one transition, never

- // the store.

- s.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

- return hostops.StampLostMoveTargetParked(ctx, records, host, now)

- }

+

}

var moveLogs func(context.Context, *hostops.MoveLogWriter) error

if s.cfg.MovesEnabled {

@@ -131,6 +124,9 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {

moveLogs = s.drainMoveLog

}

s.cfg.HostClaimRecovery = &hostops.Recovery{

+ ReportAutomaticLosses: func(ctx context.Context, host string) error {

+ return a.Control.ReportAutomaticLosses(ctx, a.Signer, host)

+ },

Moves: s.cfg.ClaimMoveRecovery,

MoveLogs: moveLogs,

MoveFenceAuthority: hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: c.Now},

diff --git a/internal/console/movesrecovery.go b/internal/console/movesrecovery.go

index c2ebae1f3..2a04b2192 100644

--- a/internal/console/movesrecovery.go

+++ b/internal/console/movesrecovery.go

@@ -10,7 +10,6 @@ import (

"fmt"

"log"

"os"

- "slices"

"time"

"github.com/carlosframework/platform/internal/alerting"

@@ -52,8 +51,8 @@ import (

// 5. The deadline: a landing past the earlier of its cut and landing

// deadlines is rolled back by the record (J11, J9's recorded reason).

// 6. A lost target released with nothing of its box or region owed:

-// target_parked_at, through hostops.StampLostMoveTargetParked over the

-// conditional record store (stampLostMoveTarget), outside the grant

+// target_parked_at, through the

+// record-sequenced recovery grant (StampLostMoveTargetParked)

// (p10 Task G).

// 7. A close whose stamps are in and whose op owes nothing: the record's

// close (close, rolled-back-close or revert-close). The boxes close on

@@ -198,11 +197,7 @@ func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision {

case m.State == provision.MoveLanding && op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted && moveDeadlinePassed(m, in.Now):

d.Action, d.Record = moveActRecord, hostops.RecoveryMoveDeadlineRollback

case m.State == provision.MoveRollingBack && rb != nil && rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty && rb.TargetParkedAt == nil &&

- (op.Kind == hostclaim.KindRestoring || op.Kind == hostclaim.KindLanding) && m.To != "" && op.Intent["target"] != "" && slices.Contains(c.Lost, op.Intent["target"]) &&

- owedUnderOp(c, op.ID, func(o hostclaim.Obligation) bool {

- t := op.Intent["target"]

- return o.Box == t || o.Owner == "box:"+t || o.Region == m.To || o.Owner == "region:"+m.To

- }) == 0:

+ hostclaim.LostMoveTargetParked(c, m.To):

d.Action = moveActStampLost

case owed > 0:

case m.State == provision.MoveCommitted && m.OpenedAt != nil && m.SourceParkedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseCommitted:

@@ -355,10 +350,7 @@ func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant

}

return nil

case moveActStampLost:

- if s.stampLostMoveTarget == nil {

- return hostops.ErrContract

- }

- return s.stampLostMoveTarget(ctx, o.Host, now)

+ return g.StampLostMoveTargetParked(ctx)

case moveActClose:

if !s.moveCloseOverdue(o, d.Record, now) {

return nil

diff --git a/internal/console/movesrecovery_test.go b/internal/console/movesrecovery_test.go

index bd856c367..81d298b36 100644

--- a/internal/console/movesrecovery_test.go

+++ b/internal/console/movesrecovery_test.go

@@ -38,7 +38,7 @@ var mvT0 = time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)

func mvClaim(kind, phase string, mut ...func(*hostclaim.Claim)) *hostclaim.Claim {

c := &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"},

Op: hostclaim.Op{Kind: kind, Phase: phase, ID: "mv", Exec: "ie-box", Participants: []string{"ie-box", "uk-box"}, StartedAt: mvT0,

- Intent: map[string]string{"source": "ie-box", "target": "uk-box"}}}

+ Intent: map[string]string{"source": "ie-box", "target": "uk-box", "to": "uk"}}}

for _, m := range mut {

m(c)

}

@@ -125,7 +125,7 @@ func TestRecoveryPassOneActionPerObservation(t *testing.T) {

t.Fatalf("the cut deadline: %+v", d)

}

// A lost, released target: the stamp, never the close it enables.

- lost := func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"} }

+ lost := func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"}; c.Op.Participants = []string{"ie-box"} }

rb := func(r *provision.Record) {

r.Move.Rollback.TargetPhase, r.Move.Rollback.SourcePhase = provision.RbTargetReleased, provision.RbSourceOpened

r.Move.Rollback.RecoveredAt, r.Move.Rollback.OpenedAt = mvAt(0), mvAt(0)

@@ -201,15 +201,6 @@ func newMovePassWorld(t *testing.T, fx *consoleFixture, id string) *movePassWorl

t.Helper()

w := &movePassWorld{fx: fx, id: id, clock: time.Now().UTC()}

fx.srv.movesClock = func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.clock }

- // As ConfigureClaimRuntime binds it: the lost target's stamp over the

- // conditional record store under the fixture's data boundary.

- records, err := hostops.NewConditionalRecordStore(fx.rel, fx.pub)

- if err != nil {

- t.Fatal(err)

- }

- fx.srv.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

- return hostops.StampLostMoveTargetParked(ctx, records, host, now)

- }

alerting.Arm(&alerting.Sinks{WebhookURL: "http://pages.test/hook", Post: func(_ string, body []byte) error {

var p pagePayload

if err := json.Unmarshal(body, &p); err != nil {

@@ -465,6 +456,8 @@ func lostTargetRollback(t *testing.T, w *movePassWorld) {

w.setClaim(t, func(c *hostclaim.Claim) {

c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box"

c.Lost = []string{"uk-box"}

+ c.Op.Participants = []string{"ie-box"}

+ c.Op.Intent["to"] = "uk"

c.Obligations = nil

})

rb, at := newRollbackID(t), time.Now().UTC().Add(-time.Hour)

@@ -479,7 +472,7 @@ func lostTargetRollback(t *testing.T, w *movePassWorld) {

// TestRecoveryPassStampsALostTargetThenCloses (p10 Task G): with the target

// lost and released and nothing of its box or region owed, the pass stamps

-// target_parked_at outside the grant; the close follows once overdue, then

+// target_parked_at through the grant; the close follows once overdue, then

// the release.

func TestRecoveryPassStampsALostTargetThenCloses(t *testing.T) {

w := landingMovePass(t, "code-mvpass-lost")

diff --git a/internal/console/server.go b/internal/console/server.go

index a3fa0cfd6..7a60ea992 100644

--- a/internal/console/server.go

+++ b/internal/console/server.go

@@ -833,10 +833,6 @@ type Server struct {

movesClock func() time.Time

moveCloseMu sync.Mutex

moveCloseSeen map[string]time.Time

- // stampLostMoveTarget is hostops.StampLostMoveTargetParked over the

- // conditional record store, bound by ConfigureClaimRuntime with the

- // moves switch on; nil otherwise.

- stampLostMoveTarget func(ctx context.Context, host string, now time.Time) error

// moveReleaseFault, when set, fails the pass's claim release just before

// it is made (a TEST SEAM: a release that fails after its log entry is

// queued, movesrecovery.go).

diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt

index c43bbd3b4..3ea4c1055 100644

--- a/internal/console/testdata/claim-authority-sites.txt

+++ b/internal/console/testdata/claim-authority-sites.txt

@@ -12,6 +12,8 @@

../hostops/lifecycle_grant.go|RecoveryPass|w.RunRecovery(ctx, r.Pass)

../hostops/move_enrollment.go|RegisterMove|a.Control.RegisterStart(ctx, h.Account, h.App, appID, id, reg, a.Now())

../hostops/move_enrollment.go|StartClaimMoveRecord|h.captureMoveStartBase(ctx, seq)

+../hostops/move_fence_release.go|ReleaseFencedTarget|hostclaim.ReleaseFencedTarget(r)

+../hostops/move_fence_release.go|ReleaseFencedTarget|hostclaim.VerifyMoveTargetFence(ctx, g.w.raw, o, ref, g.authority)

../hostops/move_loss_authority.go|DeclareMoveLoss|a.Control.DeclareMoveLoss(ctx, a.Signer, a.Directory.Bucket, actor, account, app, host, box, in)

../hostops/move_loss_authority.go|InspectMoveLoss|a.Control.InspectMoveLoss(ctx, a.Signer, a.Directory.Bucket, actor, account, app, host, box)

../hostops/move_recovery_bridge.go|RecoverTerminatedMovingHost|hostclaim.ConvertMovingToRelocating(mh.Lifecycle(ctx), s, fresh, c.Op.ID, ref, authority, guard)

@@ -35,6 +37,7 @@

../hostops/purge_journal.go|DeletePurgeSweep|w.SavePurgeJournal(ctx, fresh, tag)

../hostops/purge_journal.go|PreparePurgeSweep|w.SavePurgeJournal(ctx, j, tag)

../hostops/purge_journal.go|VerifyPurgeSweep|w.SavePurgeJournal(ctx, j, tag)

+../hostops/recovery.go|Pass|r.ReportAutomaticLosses(ctx, host)

../hostops/recovery.go|Pass|runMoveLogDrain(ctx, r.MoveLogs)

../hostops/recovery.go|Pass|runMoveRecovery(ctx, obs, r.Moves, r.MoveFenceAuthority)

../hostops/sidecarwake.go|TouchSidecarWake|a.Control.RegisterStart(ctx, account, app, appID, id, reg, a.Now())

@@ -87,13 +90,13 @@ claim_restore.go|restoreClaimApp|w.BeginExclusive(ctx, acct.Sqid, app, appID, ho

claim_retire.go|retireClaimHostRecord|c.Control.BeginExclusive(ctx, ref.Account, ref.App, appID, begin, s.claimOwner)

claim_retire.go|retireClaimHostRecord|c.Control.ResumeExclusive(ctx, ref.Account, ref.App, appID, op.ID, hostops.OpDeleting, s.claimOwner)

claim_route_writes.go|appMutation|release.WithAppMutation(ctx, s.cfg.Releases, acct.Sqid, app, action, fn)

+claim_runtime.go|ConfigureClaimRuntime|a.Control.ReportAutomaticLosses(ctx, a.Signer, host)

claim_runtime.go|ConfigureClaimRuntime|hostops.BindMutationRunner(data, control, owner, s.mutationAppID)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewConditionalRecordStore(raw, s.cfg.ReleasePub)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewControlWriter(raw, s.cfg.ReleasePub, time.Now)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewObservedMutationBoundary(true, records, check, s.claimMutationObserver)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewProductionMutationCheck(raw, control)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewVersionedBoundary(true, s.cfg.DBVersions, raw, control, s.cfg.Bucket, dataBuckets, s.claimMutationObserver)

-claim_runtime.go|ConfigureClaimRuntime|hostops.StampLostMoveTargetParked(ctx, records, host, now)

claim_runtime.go|sweepOneClaimAdmission|c.Control.RecoverSQLTransaction(ctx, app.Account, app.Name, appID, id, s.store)

claim_sql_auto.go|withAutoAppSQL|s.WithAppMutation(pctx, []sqlGrant{permitGrant(pctx, s.claimBucket, acct.Sqid, scope)}, call)

claim_sql_grants.go|WithAppMutation|p.BindTransaction(database)

@@ -114,7 +117,7 @@ movesrecovery.go|movesRecovery|g.Convert(ctx, *in.Fence)

movesrecovery.go|movesRecovery|g.Record(ctx, d.Record)

movesrecovery.go|movesRecovery|g.ResolveStart(ctx)

-movesrecovery.go|movesRecovery|s.stampLostMoveTarget(ctx, o.Host, now)

+movesrecovery.go|movesRecovery|g.StampLostMoveTargetParked(ctx)

scanpass.go|storeLister|objstore.S3ClientFor(ctx, p.Region, p.Endpoint)

store.go|Open|sql.Open("sqlite", path)

store_gate.go|Connect|k.g.d.Open(k.g.dsn)

diff --git a/internal/host/claimassembly.go b/internal/host/claimassembly.go

index e07a49179..555a1449f 100644

--- a/internal/host/claimassembly.go

+++ b/internal/host/claimassembly.go

@@ -91,6 +91,7 @@ func WithClaimAssembly(ctx context.Context, a *ClaimAssembly) context.Context {

// ClaimAssembly owns the concrete wiring. Dependencies retain their contract

// names so a missing implementation cannot silently become permissive policy.

type ClaimAssembly struct {

+ LossAdmission *hostbox.LossAdmission

moveRead *claimMoveReadAccess

wakeUnavailable error

Journals *hostbox.JournalService

@@ -176,7 +177,7 @@ func (a ClaimAssembly) Build(ctx context.Context) (ClaimProviders, error) {

a.Provision = a.Reconcile.Configure

check := claimBoundIdentity(a.Identity.Verify, actor)

data := hostbox.RequestStore{Store: a.Data, Check: check}

- writer := hostbox.Writer{Claims: a.Claims, Data: data, Bucket: actor.Bucket, Region: actor.Region, Box: actor.Box, AdmissionEvidence: a.AdmissionEvidence, PublicationEvidence: a.PublicationEvidence, NewID: a.NewID}

+ writer := hostbox.Writer{Loss: a.LossAdmission, Claims: a.Claims, Data: data, Bucket: actor.Bucket, Region: actor.Region, Box: actor.Box, AdmissionEvidence: a.AdmissionEvidence, PublicationEvidence: a.PublicationEvidence, NewID: a.NewID}

a.bindClaimWakeStop()

if local, ok := a.Local.(*productionClaimLocal); ok {

if err := local.bind(ctx, &a); err != nil {

@@ -250,7 +251,7 @@ func (a ClaimAssembly) Build(ctx context.Context) (ClaimProviders, error) {

if err := a.startWakeResetWorker(ctx); err != nil {

return ClaimProviders{}, err

}

- return ClaimProviders{moveParks: &claimMoveParks{engine: regional.Regional.Engine, leases: leases, check: check}, Delivery: ClaimTimerDelivery(a.Producers, a.NewID), Selector: source, Wake: wake, Regional: regional, Stop: a.Stop, Provision: a.Provision, Close: a.Close}, nil

+ return ClaimProviders{moveTargetFence: claimBoxFenceVerifier(check, a.Claims, a.RetirementAuthority), moveParks: &claimMoveParks{local: claimMoveLocalAuthorizer(check, a.Authorize), engine: regional.Regional.Engine, leases: leases, check: check}, Delivery: ClaimTimerDelivery(a.Producers, a.NewID), Selector: source, Wake: wake, Regional: regional, Stop: a.Stop, Provision: a.Provision, Close: a.Close}, nil

}

type claimRegionalEnrollment struct {

@@ -347,3 +348,16 @@ func NewProductionRootClaimStop(ctx context.Context) (http.Handler, error) {

reconcile.Box, reconcile.Enrollment = actor.Box, actor.Enrollment

return ClaimRootHTTP{Journals: assembly.Journals, RequireNewWork: func(ctx context.Context) error { _, err := assembly.Identity.Verify(ctx); return err }, Launches: assembly.Launches, Stop: stop, Reconcile: reconcile, Roots: assembly.ManagedRoots}, nil

}

+

+func claimMoveLocalAuthorizer(check func(context.Context) error, owner func(context.Context, hostclaim.Observed, hostclaim.Obligation) error) ClaimLocalParkAuthorizer {

+ return func(ctx context.Context, c *hostclaim.Claim, ob hostclaim.Obligation) error {

+ if check == nil || owner == nil || c == nil || ob.Kind != "park-local" {

+ return hostwrite.ErrUnavailable

+ }

+ if err := check(ctx); err != nil {

+ return err

+ }

+ // Preserve the unwrapped owner used by the assembly's own park.Local.

+ return owner(ctx, hostclaim.Observed{Host: c.Host, Claim: c}, ob)

+ }

+}

diff --git a/internal/host/claimconstruction.go b/internal/host/claimconstruction.go

index 8d8c7031d..eee40d899 100644

--- a/internal/host/claimconstruction.go

+++ b/internal/host/claimconstruction.go

@@ -22,6 +22,7 @@ import (

"github.com/carlosframework/platform/internal/controlsock"

"github.com/carlosframework/platform/internal/hostbox"

"github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostfence"

"github.com/carlosframework/platform/internal/hostguard"

"github.com/carlosframework/platform/internal/hostops"

"github.com/carlosframework/platform/internal/hostwrite"

@@ -282,7 +283,12 @@ func OpenProductionClaimAssembly(ctx context.Context, store release.Store, r Cla

wake.AfterAdmission = coldAdmissionHook(guard, owners.Identity, controlsock.Path)

wake.RetireArtifacts = func(context.Context, hostclaim.Observed, ClaimLaunchInputs, string) error { return hostbox.ErrChanged }

if movesC1 {

- handover := claimCrossBoxHandover{Guard: guard, Evidence: store, Box: actor.Box}

+ handover := claimCrossBoxHandover{Guard: guard, Evidence: store, Box: actor.Box, Inventory: inventory, Identity: owners.Identity, Registry: r.Registry, Launches: launches}

+ if owners.RetirementAuthority != nil {

+ handover.PublicKey = owners.RetirementAuthority.Directory.PublicKey

+ handover.Directory = owners.RetirementAuthority.Directory

+ handover.FenceAuthority = hostfence.ClaimFenceAuthority{Directory: owners.RetirementAuthority.Directory, Now: time.Now}

+ }

var relocation *ClaimRelocationOwners

if groupD.Dispatch.Relocation != nil {

relocation = &owners.Relocation

@@ -331,6 +337,9 @@ func OpenProductionClaimAssembly(ctx context.Context, store release.Store, r Cla

stop.Inventory = inventory

stop.NewID = newID

assembly := &ClaimAssembly{moveRead: owners.moveRead, wakeUnavailable: owners.wakeUnavailable, Journals: journals, cleanupEnrollment: owners.cleanupEnrollment, RetirementAuthority: owners.RetirementAuthority, SharedPaths: claimGlobalSharedPaths(r), Mode: r.Mode, Producers: producers, Launches: launches, ManagedRoots: roots, Reconcile: reconcile, RootJournal: launches, Controllers: owners.Controllers, Claims: store, Guard: guard, Identity: owners.Identity, Local: owners.Local, Policy: hostbox.Policy{Inputs: owners.InputPolicy, Moves: owners.MovesPolicy, Relocation: groupD.Relocation, Held: hostbox.RegistryHoldPolicy(r.Registry)}.WithRefusals(owners.PolicyRefusals...), Data: owners.Data, Wake: wake, Stop: stop, Evidence: owners.Evidence, LeaseSet: owners.LeaseSet, View: owners.View, Authorize: owners.Authorize, AdmissionEvidence: admissionEvidence, PublicationEvidence: publicationEvidence, NewID: newID}

+ if movesC1 && owners.RetirementAuthority != nil {

+ assembly.LossAdmission = &hostbox.LossAdmission{PublicKey: owners.RetirementAuthority.Directory.PublicKey, Owner: owners.Moves.LossRecovery, Sources: owners.Moves.LossSources}

+ }

var closeMu sync.Mutex

closed := false

assembly.Close = func(ctx context.Context) error {

diff --git a/internal/host/claimhandover_crossbox.go b/internal/host/claimhandover_crossbox.go

index 3c1cb5e89..a2c9603dd 100644

--- a/internal/host/claimhandover_crossbox.go

+++ b/internal/host/claimhandover_crossbox.go

@@ -2,12 +2,15 @@ package host

import (

"context"

+ "crypto/ed25519"

"strconv"

+ "github.com/carlosframework/platform/internal/hostactors"

"github.com/carlosframework/platform/internal/hostbox"

"github.com/carlosframework/platform/internal/hostclaim"

"github.com/carlosframework/platform/internal/hostguard"

"github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

)

@@ -39,12 +42,23 @@ func claimAdmissionHandover(box string, local, crossBox func(context.Context, ho

// installation stamp (item 4), so the local record this box initialized for

// the landing installation is the one the session runs under.

type claimCrossBoxHandover struct {

- Guard hostguard.Guard

- Evidence release.Store

- Box string

+ Launches *hostbox.LaunchJournal

+ afterRecoveryRows func() error

+ PublicKey ed25519.PublicKey

+ FenceAuthority hostclaim.FenceAuthority

+ Directory hostactors.Directory

+ Guard hostguard.Guard

+ Evidence release.Store

+ Inventory *hostbox.StopInventoryDB

+ Registry *registry.Registry

+ Identity hostbox.IdentityProvider

+ Box string

}

func (h claimCrossBoxHandover) After(ctx context.Context, prior, admitted hostclaim.Observed) error {

+ if claimRecoveryObservation(prior) {

+ return h.afterRecovery(ctx, prior, admitted)

+ }

if h.Evidence == nil || h.Box == "" || prior.Claim == nil || prior.Claim.Holder == nil || admitted.Claim == nil || admitted.Claim.Holder == nil || prior.Host != admitted.Host || prior.Absent || admitted.Absent {

return hostbox.ErrChanged

}

@@ -115,6 +129,9 @@ func (h claimCrossBoxHandover) sourceEvidence(ctx context.Context, prior hostcla

}

func (h claimCrossBoxHandover) Before(ctx context.Context, prior hostclaim.Observed) error {

+ if claimRecoveryObservation(prior) {

+ return h.beforeRecovery(ctx, prior)

+ }

if _, err := h.sourceEvidence(ctx, prior); err != nil {

return err

}

@@ -163,13 +180,20 @@ func wireClaimAdmission(w *ClaimWakeAssembly, box string, local func(context.Con

return relocation.AfterAdmission(ctx, prior, admitted)

}

+ w.recoveryPrior = landing.pendingRecovery

+ w.recoveryPrelaunched = landing.completeRecovery

w.BeforeAdmission = claimPredecessorStop(w, box, before)

- w.AfterAdmission = claimAdmissionHandover(box, local, after)

+ w.AfterAdmission = claimAdmissionHandover(box, func(ctx context.Context, prior, admitted hostclaim.Observed) error {

+ if claimInitialSourceRecovery(prior, box) {

+ return landing.afterRecovery(ctx, prior, admitted)

+ }

+ return local(ctx, prior, admitted)

+ }, after)

}

func claimRemotePredecessor(prior hostclaim.Observed, box string) bool {

if prior.Claim == nil || prior.Claim.Holder == nil || prior.Claim.Holder.WriterBox == box {

return false

}

- return prior.Claim.Op.Kind == hostclaim.KindRelocating || (prior.Claim.Op.Kind == hostclaim.KindLanding && prior.Claim.Holder.WriterBox != "")

+ return (prior.Claim.Op.Kind == hostclaim.KindRestoring && prior.Claim.Holder.WriterBox != "" && prior.Claim.Holder.WriterBox == prior.Claim.Op.Intent["target"]) || prior.Claim.Op.Kind == hostclaim.KindRelocating || (prior.Claim.Op.Kind == hostclaim.KindLanding && prior.Claim.Holder.WriterBox != "")

}

diff --git a/internal/host/claimhandover_recovery.go b/internal/host/claimhandover_recovery.go

new file mode 100644

index 000000000..ef22a2841

--- /dev/null

+++ b/internal/host/claimhandover_recovery.go

@@ -0,0 +1,331 @@

+package host

+

+import (

+ "context"

+ "crypto/sha256"

+ "database/sql"

+ "encoding/hex"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostguard"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/registry"

+ "strconv"

+)

+

+func claimRecoveryObservation(o hostclaim.Observed) bool {

+ return o.Claim != nil && claimRecoveryKind(o.Claim.Op.Kind)

+}

+func recoveryPredecessor(o hostclaim.Observed) hostclaim.AdmissionRecord {

+ h := o.Claim.Holder

+ p := hostclaim.AdmissionRecord{Box: h.WriterBox, Session: h.Session, Generation: h.Generation}

+ if len(h.Admissions) > 0 {

+ p = h.Admissions[len(h.Admissions)-1]

+ }

+ return p

+}

+func (h claimCrossBoxHandover) recoveryEvidence(ctx context.Context, o hostclaim.Observed) error {

+ if !claimRecoveryObservation(o) || o.Absent || o.ETag == "" || o.Claim.Holder == nil || h.Evidence == nil || h.Box == "" {

+ return hostbox.ErrChanged

+ }

+ c := o.Claim

+ p := recoveryPredecessor(o)

+ op := c.Op

+ if claimInitialSourceRecovery(o, h.Box) {

+ return h.sourceRecoveryEvidence(ctx, o)

+ }

+ if op.ID == "" || op.Phase != hostclaim.PhaseProbing || op.Exec != h.Box || op.Intent["source"] != h.Box || p.Box == "" || p.Box == h.Box || p.Box != op.Intent["target"] || p.Box != c.Holder.WriterBox || p.Session != c.Holder.Session {

+ return hostbox.ErrChanged

+ }

+ lossID, err := hostclaim.OperationLossID(c)

+ if err != nil {

+ return err

+ }

+ if lossID != "" {

+ l, err := hostclaim.ReadLossRecord(ctx, h.Evidence, h.PublicKey, lossID)

+ if err != nil {

+ return err

+ }

+ r := l.Record

+ if r.State != "committed" || r.Host != o.Host || r.OpID != op.ID || r.HolderIncarnation != c.Holder.Incarnation || r.Box != p.Box {

+ return hostclaim.ErrLossRecord

+ }

+ for _, source := range r.WriteSources {

+ if source.Reason == "target" && source.Box == p.Box && source.Session == p.Session && source.Generation == p.Generation {

+ return nil

+ }

+ return hostclaim.ErrLossRecord

+ }

+ if _, ok := hostclaim.ConsumedMoveTargetFence(c); ok {

+ return nil

+ }

+ if op.Journal["fence."+p.Box] != "" {

+ var ref hostclaim.FenceRef

+ if json.Unmarshal([]byte(op.Journal["fence."+p.Box]), &ref) != nil || op.Intent["target_enrollment"] == "" || ref.Enrollment != op.Intent["target_enrollment"] {

+ return hostclaim.ErrFence

+ }

+ verified, err := hostclaim.VerifyFence(ctx, h.Evidence, h.FenceAuthority, ref)

+ if err != nil {

+ return err

+ }

+ if verified.Box() != p.Box || verified.Region() != op.Intent["to"] {

+ return hostclaim.ErrFence

+ }

+ return nil

+ }

+ if key := op.Journal["target_stop_receipt"]; key != "" {

+ return h.recoveryStopReceipt(ctx, o, p, key, op.Journal["target_stop_sha256"])

+ }

+ key := op.Journal[hostclaim.JournalEvidenceTarget]

+ expected := moves.StoppedKey(moves.ReportPrefix(c.Holder.Account, c.Holder.App, o.Host, op.ID), "target")

+ session, err := strconv.ParseUint(op.Journal[hostclaim.JournalTargetSession], 10, 64)

+ if err != nil || session != p.Session || op.Journal[hostclaim.JournalTargetGeneration] != p.Generation || key != expected {

+ return hostbox.ErrChanged

+ }

+ report, err := readStoppedReport(ctx, h.Evidence, key)

+ if err != nil {

+ return err

+ }

+ if report.Role != "target" || report.Box != p.Box || report.Host != o.Host || report.OpID != op.ID || report.Incarnation != c.Holder.Incarnation {

+ return hostbox.ErrChanged

+ }

+ return nil

+}

+func (h claimCrossBoxHandover) beforeRecovery(ctx context.Context, o hostclaim.Observed) error {

+ if err := h.recoveryEvidence(ctx, o); err != nil {

+ return err

+ }

+ old := o.Claim.Holder

+ return h.Guard.With(ctx, o.Host, hostguard.Captured{Incarnation: old.Incarnation, Stamp: old.Stamp}, func(c *hostclaim.Claim) error {

+ if c.Gen != o.Claim.Gen || c.Op.ID != o.Claim.Op.ID || c.Op.Kind != o.Claim.Op.Kind || c.Op.Phase != o.Claim.Op.Phase || c.Op.Exec != h.Box || c.Holder.Session != old.Session {

+ return hostbox.ErrChanged

+ }

+ return nil

+ }, func() error { return nil })

+}

+func (h claimCrossBoxHandover) afterRecovery(ctx context.Context, prior, admitted hostclaim.Observed) error {

+ if err := h.recoveryEvidence(ctx, prior); err != nil {

+ return err

+ }

+ if admitted.Absent || admitted.Claim == nil || admitted.Claim.Holder == nil || admitted.Host != prior.Host {

+ return hostbox.ErrChanged

+ }

+ old, next := prior.Claim.Holder, admitted.Claim.Holder

+ op, ac := prior.Claim.Op, admitted.Claim

+ if ac.Op.ID != op.ID || ac.Op.Kind != op.Kind || ac.Op.Phase != op.Phase || ac.Op.Exec != h.Box || next.Incarnation != old.Incarnation || next.Session <= old.Session || next.Stamp != next.Session || next.WriterBox != h.Box || len(next.Admissions) == 0 {

+ return hostbox.ErrChanged

+ }

+ p := recoveryPredecessor(prior)

+ a := next.Admissions[len(next.Admissions)-1]

+ if a.Session != next.Session || a.Box != h.Box || a.Mode != "cold" || a.Generation != next.Generation || a.Predecessor != p.Generation {

+ return hostbox.ErrChanged

+ }

+ lossID, err := hostclaim.OperationLossID(prior.Claim)

+ if err != nil {

+ return err

+ }

+ if lossID != "" && (a.LossID != lossID || a.Coverage != "" || (p.Session != 0 && (a.WritesLost == nil || a.WritesLost.Box != p.Box || a.WritesLost.Session != p.Session))) {

+ return hostclaim.ErrLossRecord

+ }

+ if lossID == "" && a.LossID != "" {

+ return hostclaim.ErrLossRecord

+ }

+ current, err := hostclaim.Read(ctx, h.Evidence, admitted.Host)

+ if err != nil {

+ return err

+ }

+ if current.ETag != admitted.ETag {

+ return hostbox.ErrChanged

+ }

+ if h.Registry == nil {

+ return hostbox.ErrTypedInventory

+ }

+ journal, err := json.Marshal(claimRecoveryJournal{Prior: prior, Admitted: admitted})

+ if err != nil {

+ return err

+ }

+ if err = h.Registry.BeginRecoveryHandover(ctx, claimRecoveryJournalID(admitted), string(journal)); err != nil {

+ return err

+ }

+ // Activate retires the prior local stamp through the descriptor-based guard.

+ // The restore hook subsequently retires the source database and marker.

+ if err = h.Guard.Activate(ctx, admitted.Host, h.Box, nil); err != nil {

+ return err

+ }

+ return h.Guard.With(ctx, admitted.Host, hostguard.Captured{Incarnation: next.Incarnation, Stamp: next.Stamp}, func(c *hostclaim.Claim) error {

+ if c.Gen != ac.Gen || c.Holder.Session != next.Session || c.Holder.Generation != next.Generation || c.Holder.WriterBox != h.Box || c.Op.ID != op.ID || c.Op.Kind != op.Kind || c.Op.Phase != op.Phase || c.Op.Exec != h.Box {

+ return hostbox.ErrChanged

+ }

+ return nil

+ }, func() error { return h.installRecoveryInventory(ctx, prior, admitted) })

+}

+

+// The first return to the cut installation has not launched under this operation.

+// Later source sessions cannot reuse the cut's stop evidence.

+func claimInitialSourceRecovery(o hostclaim.Observed, box string) bool {

+ if !claimRecoveryObservation(o) || o.Claim.Holder == nil {

+ return false

+ }

+ c, h := o.Claim, o.Claim.Holder

+ cut, e := strconv.ParseUint(c.Op.Journal[hostclaim.JournalCutSession], 10, 64)

+ stamp, se := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64)

+ return e == nil && se == nil && stamp > 0 && h.Stamp == stamp && c.Op.ID != "" && c.Op.Phase == hostclaim.PhaseProbing && c.Op.Exec == box && c.Op.Intent["source"] == box && h.Session == cut && (h.WriterBox == box || h.Session == 0 && h.WriterBox == "" && len(h.Admissions) == 0) && h.Generation == c.Op.Journal[hostclaim.JournalSrcGen]

+}

+func (h claimCrossBoxHandover) sourceRecoveryEvidence(ctx context.Context, o hostclaim.Observed) error {

+ if !claimInitialSourceRecovery(o, h.Box) || h.Evidence == nil {

+ return hostbox.ErrChanged

+ }

+ c := o.Claim

+ key := moves.StoppedKey(moves.ReportPrefix(c.Holder.Account, c.Holder.App, o.Host, c.Op.ID), "source")

+ if c.Op.Journal[hostclaim.JournalEvidenceSource] != key {

+ return hostbox.ErrChanged

+ }

+ report, err := readStoppedReport(ctx, h.Evidence, key)

+ if err != nil {

+ return err

+ }

+ if report.Role != "source" || report.Box != h.Box || report.Host != o.Host || report.OpID != c.Op.ID || report.Incarnation != c.Holder.Incarnation {

+ return hostbox.ErrChanged

+ }

+ return nil

+}

+

+// The guard is already held. Preserve exact inventory entries and their physical

+// identities; only ownership moves to the admitted installation. The old inventory

+// remains available for retirement and replay. Never infer an empty installation.

+func (h claimCrossBoxHandover) installRecoveryInventory(ctx context.Context, prior, admitted hostclaim.Observed) error {

+ if h.Inventory == nil {

+ return hostbox.ErrTypedInventory

+ }

+ actor, err := h.Identity.Verify(ctx)

+ if err != nil {

+ return err

+ }

+ if actor.Box != h.Box {

+ return hostbox.ErrChanged

+ }

+ old, next := prior.Claim.Holder, admitted.Claim.Holder

+ stamp, err := strconv.ParseUint(prior.Claim.Op.Journal[hostclaim.JournalSourceStamp], 10, 64)

+ if err != nil || stamp == 0 || stamp >= next.Stamp {

+ return hostbox.ErrChanged

+ }

+ inv, err := h.Inventory.ReadInstallation(ctx, prior.Host, old.Incarnation, stamp)

+ if err != nil {

+ return err

+ }

+ if !inv.Complete || inv.Host != prior.Host || inv.Incarnation != old.Incarnation || inv.InstallationStamp != stamp || inv.Box != actor.Box || inv.Enrollment != actor.Enrollment {

+ return hostbox.ErrTypedInventory

+ }

+ current, err := h.Inventory.ReadInstallation(ctx, prior.Host, next.Incarnation, next.Stamp)

+ if err == nil {

+ if !current.Complete || current.Box != actor.Box || current.Enrollment != actor.Enrollment || current.DirectoryRevision != fmt.Sprint(actor.Revision) {

+ return hostbox.ErrTypedInventory

+ }

+ if h.Registry == nil {

+ return hostbox.ErrTypedInventory

+ }

+ row, e := h.Registry.Get(prior.Host)

+ if e != nil {

+ return e

+ }

+ return hostbox.ValidateAgentInventory(current, row, h.Guard.DataDir)

+ }

+ if !errors.Is(err, sql.ErrNoRows) {

+ return err

+ }

+ inv.InstallationStamp, inv.DirectoryRevision = next.Stamp, fmt.Sprint(actor.Revision)

+ sum := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%d\x00%s", prior.Host, next.Incarnation, next.Stamp, actor.Box)))

+ owner := hex.EncodeToString(sum[:])

+ if err := h.rebindRecoveryRows(ctx, &inv, stamp, owner); err != nil {

+ return err

+ }

+ for i := range inv.Entries {

+ inv.Entries[i].Ownership = owner

+ }

+ if h.afterRecoveryRows != nil {

+ if err := h.afterRecoveryRows(); err != nil {

+ return err

+ }

+ return h.Inventory.WriteStopInventory(ctx, inv)

+}

+

+func (h claimCrossBoxHandover) rebindRecoveryRows(ctx context.Context, inv *hostbox.StopInventory, oldStamp uint64, owner string) error {

+ if h.Registry == nil {

+ return hostbox.ErrTypedInventory

+ }

+ row, err := h.Registry.Get(inv.Host)

+ if err != nil {

+ return err

+ }

+ if row == nil || row.ClaimIncarnation != inv.Incarnation || (row.ClaimStamp != oldStamp && row.ClaimStamp != inv.InstallationStamp) || !row.ClaimOwnership.Valid() {

+ return hostbox.ErrTypedInventory

+ }

+ timers, err := h.Registry.TimersFor(inv.Host)

+ if err != nil {

+ return err

+ }

+ old := registry.ColdRows{Route: *row, Timers: append([]registry.Timer(nil), timers...)}

+ next := registry.ColdRows{Route: *row, Timers: append([]registry.Timer(nil), timers...)}

+ ownership := func(kind, name string) registry.ClaimOwnership {

+ sum := sha256.Sum256([]byte(owner + "\x00" + kind + "\x00" + name))

+ return registry.ClaimOwnership{ClaimIncarnation: inv.Incarnation, ClaimStamp: inv.InstallationStamp, ClaimOwner: owner, ClaimRowID: hex.EncodeToString(sum[:])}

+ }

+ next.Route.ClaimOwnership = ownership("route", inv.Host)

+ for i := range next.Timers {

+ next.Timers[i].ClaimOwnership = ownership("timer", next.Timers[i].Name)

+ }

+ // Retained source inventory recovers the old row identities after a lost reply.

+ routeFound := false

+ for i := range inv.Entries {

+ e := &inv.Entries[i]

+ if e.Kind == "route" {

+ if routeFound || e.RelativePath != inv.Host {

+ return hostbox.ErrTypedInventory

+ }

+ routeFound = true

+ old.Route.ClaimOwnership = registry.ClaimOwnership{ClaimIncarnation: inv.Incarnation, ClaimStamp: oldStamp, ClaimOwner: e.Ownership, ClaimRowID: e.RowIdentity}

+ e.RowIdentity = next.Route.ClaimRowID

+ }

+ if e.Kind == "timer" {

+ found := false

+ for j := range old.Timers {

+ if e.RelativePath == inv.Host+"/"+old.Timers[j].Name {

+ old.Timers[j].ClaimOwnership = registry.ClaimOwnership{ClaimIncarnation: inv.Incarnation, ClaimStamp: oldStamp, ClaimOwner: e.Ownership, ClaimRowID: e.RowIdentity}

+ e.RowIdentity = next.Timers[j].ClaimRowID

+ found = true

+ }

+ if !found {

+ return hostbox.ErrTypedInventory

+ }

+ if !routeFound {

+ return hostbox.ErrTypedInventory

+ }

+ source := *inv

+ source.InstallationStamp = oldStamp

+ source.Entries = append([]hostbox.StopInventoryEntry(nil), inv.Entries...)

+ for i := range source.Entries {

+ e := &source.Entries[i]

+ if e.Kind == "route" {

+ e.RowIdentity = old.Route.ClaimRowID

+ }

+ if e.Kind == "timer" {

+ for _, t := range old.Timers {

+ if e.RelativePath == inv.Host+"/"+t.Name {

+ e.RowIdentity = t.ClaimRowID

+ }

+ if err := hostbox.ValidateAgentInventory(source, &old.Route, h.Guard.DataDir); err != nil {

+ return err

+ }

+ return h.Registry.RebindClaimRows(ctx, old, next)

+}

diff --git a/internal/host/claimmovefence.go b/internal/host/claimmovefence.go

new file mode 100644

index 000000000..6a2014065

--- /dev/null

+++ b/internal/host/claimmovefence.go

@@ -0,0 +1,41 @@

+package host

+

+import (

+ "context"

+ "encoding/json"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostfence"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/release"

+ "time"

+)

+

+func (p ClaimProviders) VerifyMoveTargetFence(ctx context.Context, o hostclaim.Observed) (hostclaim.ReturnFence, error) {

+ if p.moveTargetFence == nil {

+ return hostclaim.ReturnFence{}, hostclaim.ErrFence

+ }

+ return p.moveTargetFence(ctx, o)

+}

+func claimBoxFenceVerifier(check func(context.Context) error, store release.Store, authority *hostops.ProductionAuthority) func(context.Context, hostclaim.Observed) (hostclaim.ReturnFence, error) {

+ return func(ctx context.Context, o hostclaim.Observed) (hostclaim.ReturnFence, error) {

+ if check == nil || store == nil || authority == nil || o.Claim == nil {

+ return hostclaim.ReturnFence{}, hostclaim.ErrFence

+ }

+ if err := check(ctx); err != nil {

+ return hostclaim.ReturnFence{}, err

+ }

+ fresh, err := hostclaim.Read(ctx, store, o.Host)

+ if err != nil {

+ return hostclaim.ReturnFence{}, err

+ }

+ if fresh.ETag != o.ETag {

+ return hostclaim.ReturnFence{}, hostbox.ErrChanged

+ }

+ var ref hostclaim.FenceRef

+ if json.Unmarshal([]byte(o.Claim.Op.Journal["fence."+o.Claim.Op.Intent["target"]]), &ref) != nil {

+ return hostclaim.ReturnFence{}, hostclaim.ErrFence

+ }

+ return hostclaim.VerifyMoveTargetFence(ctx, store, o, ref, hostfence.ClaimFenceAuthority{Directory: authority.Directory, Now: time.Now})

+ }

+}

diff --git a/internal/host/claimmoveparks.go b/internal/host/claimmoveparks.go

index 411363e92..00d37106b 100644

--- a/internal/host/claimmoveparks.go

+++ b/internal/host/claimmoveparks.go

@@ -16,21 +16,17 @@ import (

type claimMoveParkSources struct {

View func(context.Context, hostclaim.Observed) (park.View, error)

Evidence hostbox.RegionalEvidence

- Leases hostbox.LeaseSet

}

// RegisterProductionClaimMoveParks installs the regional contracts independently

// of the wake owners and C8 files manifest. Registration is atomic and one-shot.

-func RegisterProductionClaimMoveParks(view func(context.Context, hostclaim.Observed) (park.View, error), retirementEvidence hostbox.RegionalEvidence, leases hostbox.LeaseSet) error {

+func RegisterProductionClaimMoveParks(view func(context.Context, hostclaim.Observed) (park.View, error), retirementEvidence hostbox.RegionalEvidence) error {

var missing []string

if view == nil {

missing = append(missing, "moves Plan 5 exact-version generation and files-manifest view")

}

if retirementEvidence == nil {

- missing = append(missing, "moves Plan 5 archived-incarnation regional evidence")

- }

- if leases == nil {

- missing = append(missing, "moves Plan 5 complete replica and staging lease inventory")

+ missing = append(missing, "moves Plan 5 retired-move regional evidence")

}

if len(missing) != 0 {

return &ClaimDependencyError{Contracts: missing}

@@ -40,7 +36,7 @@ func RegisterProductionClaimMoveParks(view func(context.Context, hostclaim.Obser

if claimMoveSources.Parks != nil {

return hostwrite.ErrUnavailable

}

- claimMoveSources.Parks = &claimMoveParkSources{View: view, Evidence: retirementEvidence, Leases: leases}

+ claimMoveSources.Parks = &claimMoveParkSources{View: view, Evidence: retirementEvidence}

return nil

}

@@ -87,25 +83,38 @@ func claimBoundIdentity(verify func(context.Context) (hostbox.Actor, error), exp

}

+// ClaimLocalParkAuthorizer is the owner's local check before park.Local effects.

+type ClaimLocalParkAuthorizer func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error

+

type claimMoveParks struct {

+ local ClaimLocalParkAuthorizer

engine park.Engine

leases hostbox.LeaseSet

check func(context.Context) error

}

-// MoveParks returns the assembly's regional engine and complete lease inventory.

+// MoveParks returns the regional engine, lease inventory and local authorizer.

// Only Build can populate this capability, after Identity.Verify succeeds.

-func (p ClaimProviders) MoveParks(ctx context.Context) (park.Engine, hostbox.LeaseSet, error) {

+func (p ClaimProviders) MoveParks(ctx context.Context) (park.Engine, hostbox.LeaseSet, ClaimLocalParkAuthorizer, error) {

if p.moveParks == nil || p.moveParks.check == nil || p.moveParks.leases == nil {

- return park.Engine{}, nil, hostwrite.ErrUnavailable

+ return park.Engine{}, nil, nil, hostwrite.ErrUnavailable

}

if err := p.moveParks.check(ctx); err != nil {

- return park.Engine{}, nil, err

+ return park.Engine{}, nil, nil, err

}

- return p.moveParks.engine, p.moveParks.leases, nil

+ if p.moveParks.local == nil {

+ return park.Engine{}, nil, nil, hostwrite.ErrUnavailable

+ }

+ return p.moveParks.engine, p.moveParks.leases, p.moveParks.local, nil

}

func (p productionClaimRegional) VerifyMovesSweep(ctx context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (park.MoveSweepProof, error) {

+ if _, err := p.Identity.Verify(ctx); err != nil {

+ return park.MoveSweepProof{}, err

+ }

+ if o.Claim != nil && o.Claim.Free() {

+ return claimFreeMoveProof(o)

+ }

if p.Parks == nil {

return park.MoveSweepProof{}, &ClaimDependencyError{Contracts: []string{"moves Plan 5 newest retired move fence or release evidence"}}

}

diff --git a/internal/host/claimmoveparks_test.go b/internal/host/claimmoveparks_test.go

index 66c154ae0..96ffc1e5a 100644

--- a/internal/host/claimmoveparks_test.go

+++ b/internal/host/claimmoveparks_test.go

@@ -56,8 +56,8 @@ func TestMoveParksRegistrationIndependentAndAtomic(t *testing.T) {

claimMoveSources.Parks = oldParks

claimMoveSources.Unlock()

})

- err := RegisterProductionClaimMoveParks(nil, nil, nil)

- for _, name := range []string{"view", "regional evidence", "lease inventory"} {

+ err := RegisterProductionClaimMoveParks(nil, nil)

+ for _, name := range []string{"view", "regional evidence"} {

if err == nil || !strings.Contains(err.Error(), name) {

t.Fatalf("missing %s: %v", name, err)

}

@@ -66,13 +66,13 @@ func TestMoveParksRegistrationIndependentAndAtomic(t *testing.T) {

t.Fatal("partial registration")

}

source := &parkRegistrationWake{}

- if err := RegisterProductionClaimMoveParks(source.View, source, source.LeaseSet); err != nil {

+ if err := RegisterProductionClaimMoveParks(source.View, source); err != nil {

t.Fatal(err)

}

if source.opens != 0 || claimMoveSources.Source != nil {

t.Fatal("parks opened or registered wake")

}

- if err := RegisterProductionClaimMoveParks(source.View, source, source.LeaseSet); !errors.Is(err, hostwrite.ErrUnavailable) {

+ if err := RegisterProductionClaimMoveParks(source.View, source); !errors.Is(err, hostwrite.ErrUnavailable) {

t.Fatalf("replacement: %v", err)

}

parks := claimMoveSources.Parks

@@ -115,8 +115,8 @@ func TestMoveParksVerifiedIdentityAndRealEngine(t *testing.T) {

sentinel := errors.New("real view")

calls := 0

engine := park.Engine{Bucket: actor.Bucket, Region: actor.Region, View: func(context.Context, hostclaim.Observed) (park.View, error) { calls++; return park.View{}, sentinel }}

- providers := ClaimProviders{moveParks: &claimMoveParks{engine: engine, check: check, leases: func(context.Context, string, string) ([]string, error) { return []string{"real lease"}, nil }}}

- got, leases, err := providers.MoveParks(t.Context())

+ providers := ClaimProviders{moveParks: &claimMoveParks{local: func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error { return nil }, engine: engine, check: check, leases: func(context.Context, string, string) ([]string, error) { return []string{"real lease"}, nil }}}

+ got, leases, _, err := providers.MoveParks(t.Context())

if err != nil {

t.Fatal(err)

}

@@ -132,12 +132,12 @@ func TestMoveParksVerifiedIdentityAndRealEngine(t *testing.T) {

changed := actor

mutate(&changed)

providers.moveParks.check = claimBoundIdentity(func(context.Context) (hostbox.Actor, error) { return changed, nil }, actor)

- if _, _, err := providers.MoveParks(t.Context()); !errors.Is(err, hostbox.ErrChanged) {

+ if _, _, _, err := providers.MoveParks(t.Context()); !errors.Is(err, hostbox.ErrChanged) {

t.Fatalf("identity change accepted: %v", err)

}

})

}

- if _, _, err := (ClaimProviders{}).MoveParks(t.Context()); !errors.Is(err, hostwrite.ErrUnavailable) {

+ if _, _, _, err := (ClaimProviders{}).MoveParks(t.Context()); !errors.Is(err, hostwrite.ErrUnavailable) {

t.Fatal("unbuilt accessor accepted")

}

@@ -187,7 +187,13 @@ func TestMoveParksBuildWithFailedWakeUsesRegionalEngine(t *testing.T) {

if err != nil {

t.Fatal(err)

}

- engine, leases, err := providers.MoveParks(ctx)

+ engine, leases, local, err := providers.MoveParks(ctx)

+ if local == nil {

+ t.Fatal("missing local authorizer")

+ }

+ if err := local(ctx, &hostclaim.Claim{Host: "app.example"}, hostclaim.Obligation{Kind: "park-local"}); err != nil {

+ t.Fatal("local authorizer went through VerifyBound", err)

+ }

if err != nil {

t.Fatal(err)

}

diff --git a/internal/host/claimowners.go b/internal/host/claimowners.go

index 491a3e2ab..8b335f3c4 100644

--- a/internal/host/claimowners.go

+++ b/internal/host/claimowners.go

@@ -39,13 +39,15 @@ func (o ClaimKindOwners) complete() bool {

// with CARLOS_MOVES_PHASE1 on. ProbeReady replaces the claim's probe

// readiness check when set.

type ClaimMovesOwners struct {

+ LossRecovery hostclaim.LossRecoveryOwner

+ LossSources hostbox.LossAdmissionSources

ClaimKindOwners

Operations OperationAttempts

ProbeReady ClaimProbeReadiness

}

func (m ClaimMovesOwners) zero() bool {

- return m.Inputs == nil && m.AdmissionEvidence == nil && m.PublicationEvidence == nil && m.Operations == nil && m.ProbeReady == nil

+ return m.LossRecovery == nil && m.LossSources == nil && m.Inputs == nil && m.AdmissionEvidence == nil && m.PublicationEvidence == nil && m.Operations == nil && m.ProbeReady == nil

}

// ClaimRelocationOwners is region HA's owner set for the relocating kind

diff --git a/internal/host/claimproduction.go b/internal/host/claimproduction.go

index 86b8aabbc..8a2deeb1c 100644

--- a/internal/host/claimproduction.go

+++ b/internal/host/claimproduction.go

@@ -37,7 +37,6 @@ type ClaimProductionMoveSources interface {

Open(context.Context, release.Store, lease.VersionedStore, hostbox.Actor, ClaimMoveReadAccess) (ClaimMovesOwners, ClaimFilesManifest, error)

View(context.Context, hostclaim.Observed) (park.View, error)

VerifyRetirement(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error)

- LeaseSet(context.Context, string, string) ([]string, error)

Close(context.Context) error

}

@@ -54,7 +53,7 @@ func RegisterProductionClaimMoveSources(s ClaimProductionMoveSources) error {

return hostwrite.ErrUnavailable

}

if claimMoveSources.Parks == nil {

- claimMoveSources.Parks = &claimMoveParkSources{View: s.View, Evidence: s, Leases: s.LeaseSet}

+ claimMoveSources.Parks = &claimMoveParkSources{View: s.View, Evidence: s}

}

claimMoveSources.Source = s

return nil

@@ -346,6 +345,7 @@ func (p productionClaimRoster) Actor(ctx context.Context, box string) (hostbox.A

}

type productionClaimRegional struct {

+ Now func() time.Time

Identity hostbox.IdentityProvider

Parks *claimMoveParkSources

Local *productionClaimLocal

@@ -358,12 +358,21 @@ func (p productionClaimRegional) View(ctx context.Context, o hostclaim.Observed)

if _, err := p.Identity.Verify(ctx); err != nil {

return park.View{}, err

}

- return p.Parks.View(ctx, o)

+ v, err := p.Parks.View(ctx, o)

+ if err != nil {

+ return park.View{}, err

+ }

+ v.ArchivedIncarnations = claimArchivedLocations(o.Claim)

+ v.RefusedLocations = map[string]error{}

+ if _, err := claimLegacyIncarnation(o.Claim); err != nil {

+ v.RefusedLocations["instances/"+o.Host] = err

+ }

+ if o.Claim.Holder == nil && len(o.Claim.RetiredIncarnations) == 0 {

+ v.RefusedLocations["moves/"+o.Host] = claimRetirementDependency()

+ }

+ return v, nil

}

func (p productionClaimRegional) LeaseSet(ctx context.Context, bucket, scope string) ([]string, error) {

- if p.Parks == nil || p.Parks.Leases == nil {

- return nil, &ClaimDependencyError{Contracts: []string{"moves Plan 5 complete replica and staging lease inventory"}}

- }

a, err := p.Identity.Verify(ctx)

if err != nil {

return nil, err

@@ -371,16 +380,13 @@ func (p productionClaimRegional) LeaseSet(ctx context.Context, bucket, scope str

if a.Bucket != bucket {

return nil, hostbox.ErrChanged

}

- return p.Parks.Leases(ctx, bucket, scope)

+ return hostbox.ReplicaParkLeaseSet(ctx, bucket, scope)

}

func (p productionClaimRegional) VerifyRetirement(ctx context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) {

- if p.Parks == nil || p.Parks.Evidence == nil {

- return "", time.Time{}, &ClaimDependencyError{Contracts: []string{"moves Plan 5 archived-incarnation regional evidence"}}

- }

if _, err := p.Identity.Verify(ctx); err != nil {

return "", time.Time{}, err

}

- return p.Parks.Evidence.VerifyRetirement(ctx, o, v)

+ return p.retirement(ctx, o, v)

}

func (p productionClaimRegional) Authorize(ctx context.Context, o hostclaim.Observed, ob hostclaim.Obligation) error {

a, err := p.Identity.Verify(ctx)

diff --git a/internal/host/claimrecoveryjournal.go b/internal/host/claimrecoveryjournal.go

new file mode 100644

index 000000000..763b78a43

--- /dev/null

+++ b/internal/host/claimrecoveryjournal.go

@@ -0,0 +1,62 @@

+package host

+

+import (

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "fmt"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+)

+

+type claimRecoveryJournal struct{ Prior, Admitted hostclaim.Observed }

+

+func claimRecoveryJournalID(o hostclaim.Observed) string {

+ h := o.Claim.Holder

+ sum := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%d", o.Host, h.Incarnation, h.Stamp)))

+ return hex.EncodeToString(sum[:])

+}

+func (h claimCrossBoxHandover) pendingRecovery(ctx context.Context, o hostclaim.Observed) (hostclaim.Observed, bool, error) {

+ if !claimRecoveryObservation(o) || o.Claim.Holder == nil || h.Registry == nil {

+ return o, false, nil

+ }

+ body, err := h.Registry.PendingRecoveryHandover(ctx, claimRecoveryJournalID(o))

+ if err != nil {

+ return o, false, err

+ }

+ if body == "" {

+ return o, false, nil

+ }

+ var j claimRecoveryJournal

+ if json.Unmarshal([]byte(body), &j) != nil || j.Admitted.ETag != o.ETag || j.Admitted.Host != o.Host || j.Prior.Claim == nil || j.Prior.Claim.Holder == nil {

+ return o, false, hostbox.ErrChanged

+ }

+ if h.Launches == nil {

+ return o, false, hostbox.ErrTypedInventory

+ }

+ actor, err := h.Identity.Verify(ctx)

+ if err != nil {

+ return o, false, err

+ }

+ next := o.Claim.Holder

+ launches, err := h.Launches.SessionLaunches(ctx, o.Host, next.Incarnation, next.Stamp, next.Session, h.Box, actor.Enrollment)

+ if err != nil {

+ return o, false, err

+ }

+ if len(launches) > 0 {

+ // Once prelaunch exists, ordinary root stopping must settle that session.

+ return o, false, h.Registry.CompleteRecoveryHandover(ctx, claimRecoveryJournalID(o), body)

+ }

+ return j.Prior, true, nil

+}

+func (h claimCrossBoxHandover) completeRecovery(ctx context.Context, o hostclaim.Observed) error {

+ if !claimRecoveryObservation(o) || o.Claim.Holder == nil || h.Registry == nil {

+ return nil

+ }

+ body, err := h.Registry.PendingRecoveryHandover(ctx, claimRecoveryJournalID(o))

+ if err != nil || body == "" {

+ return err

+ }

+ return h.Registry.CompleteRecoveryHandover(ctx, claimRecoveryJournalID(o), body)

+}

diff --git a/internal/host/claimrecoveryreceipt.go b/internal/host/claimrecoveryreceipt.go

new file mode 100644

index 000000000..d9762f92e

--- /dev/null

+++ b/internal/host/claimrecoveryreceipt.go

@@ -0,0 +1,56 @@

+package host

+

+import (

+ "bytes"

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostevidence"

+ "strconv"

+)

+

+// A root-signed stop receipt is an alternative to the target's own release journal.

+func (h claimCrossBoxHandover) recoveryStopReceipt(ctx context.Context, o hostclaim.Observed, p hostclaim.AdmissionRecord, key, digest string) error {

+ raw, _, err := h.Evidence.Get(ctx, key)

+ if err != nil {

+ return err

+ }

+ sum := sha256.Sum256(raw)

+ if hex.EncodeToString(sum[:]) != digest {

+ return ErrClaimStopReceipt

+ }

+ var r StopReceipt

+ if json.Unmarshal(raw, &r) != nil {

+ return ErrClaimStopReceipt

+ }

+ canonical, err := json.Marshal(r)

+ if err != nil || !bytes.Equal(raw, canonical) {

+ return ErrClaimStopReceipt

+ }

+ c := o.Claim

+ stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalTargetStamp], 10, 64)

+ if err != nil || stamp == 0 || r.V != 1 || r.Kind != "stopped" || r.Host != o.Host || r.Box != p.Box || r.OperationID != c.Op.ID ||

+ r.Incarnation != c.Holder.Incarnation || r.WriterSession != p.Session || r.InstallationStamp != stamp ||

+ r.Region != c.Op.Intent["to"] || r.Enrollment != c.Op.Intent["target_enrollment"] || r.Phase != hostclaim.PhaseReleasing ||

+ key != stopEvidenceKey(o.Host, c.Op.ID, p.Box, r.ID) || r.FenceID == "" || r.ObligationID == "" || r.ClaimGen == 0 ||

+ r.EvidenceAt.IsZero() || r.RequestLimitNS != int64(hostbox.RequestLimit) || len(r.CgroupIdentities) == 0 ||

+ !stopDigest(r.InventorySHA256) || !stopDigest(r.LocalJournalSHA256) || !stopDigest(r.ProducerDrainDigest) {

+ return ErrClaimStopReceipt

+ }

+ roster, err := h.Directory.Read(ctx)

+ if err != nil {

+ return err

+ }

+ box, ok := roster.Roster.Boxes[p.Box]

+ if !ok || box.Enrollment != r.Enrollment || box.Region != r.Region {

+ return ErrClaimStopIdentity

+ }

+ sig, _, err := h.Evidence.Get(ctx, key+".sig")

+ if err != nil {

+ return err

+ }

+ return hostevidence.Verify(box.CensusPublicKey, key, raw, sig)

+}

diff --git a/internal/host/claimretire_local.go b/internal/host/claimretire_local.go

index 7fd73a3a8..3c1567cf7 100644

--- a/internal/host/claimretire_local.go

+++ b/internal/host/claimretire_local.go

@@ -4,8 +4,10 @@ import (

"context"

"errors"

"fmt"

+ "github.com/carlosframework/platform/internal/moves"

"os"

"path/filepath"

+ "strconv"

"github.com/carlosframework/platform/internal/hostbox"

"github.com/carlosframework/platform/internal/hostclaim"

@@ -13,21 +15,12 @@ import (

"github.com/carlosframework/platform/internal/park"

)

-// claimArtifactRetirement is the production ColdRestore retirement hook for a

-// landing writer (claim Plan 9 D4, moves ruling H15). ColdRestore calls it

-// when the local .gen names another generation: a restarted probe over the

-// landing's prepared database or an earlier probe's, or a cold wake after the

-// open over the probe's generation. It is retire-artifacts(S) of claim §6.4

-// and needs no chore (AdmitWriter queues none, and none is invented): it

-// parks only the database and its marker, both tagged with the stamp S of the

-// superseded local installation, through park.Local under the admitted

-// installation's guard, and moves no stamp. The files directory and

-// <host>.env stay where the landing placed them. Root's completed full stop

-// of the one writer session that ran S on this box must already be journalled

-// (the writer-session receipt the wake's Stop collected before admission).

-// Serving and every other kind keep main's refusal: their predecessor

-// installation is retired by the cold handover before restore. Continue is

-// bound only with the moves C1 integration; nil refuses.

+// claimArtifactRetirement parks a superseded database and marker under the

+// admitted installation's descriptor guard. Landing probes, restoring and

+// reverting share the same exact inventory checks. Initial source recovery

+// uses the source cut's journal and report; later sessions require root's

+// completed writer-stop receipt. Files and environment paths are unaffected.

+// Continue is bound only by the moves C1 integration; nil refuses.

type claimArtifactRetirement struct {

Local park.Local

Continue func(kind, from, to string) bool

@@ -35,7 +28,7 @@ type claimArtifactRetirement struct {

func (r claimArtifactRetirement) Retire(ctx context.Context, admitted hostclaim.Observed, in ClaimLaunchInputs, generation string) error {

box := r.Local.Box

- if r.Continue == nil || r.Local.Claims == nil || box == "" || !filepath.IsAbs(r.Local.DataDir) || admitted.Claim == nil || admitted.Claim.Holder == nil || admitted.Claim.Op.Kind != hostclaim.KindLanding {

+ if r.Continue == nil || r.Local.Claims == nil || box == "" || !filepath.IsAbs(r.Local.DataDir) || admitted.Claim == nil || admitted.Claim.Holder == nil || !claimArtifactKind(admitted.Claim.Op.Kind) {

return hostbox.ErrChanged

}

h := admitted.Claim.Holder

@@ -59,13 +52,16 @@ func (r claimArtifactRetirement) Retire(ctx context.Context, admitted hostclaim.

return hostbox.ErrChanged

}

c := fresh.Claim

- if _, ok := hostguard.LandingWriter(c, box); !ok {

+ if _, ok := claimArtifactWriter(c, box); !ok {

return hostbox.ErrChanged

}

// The superseded database is this op's: its prepared generation, named by

// the op id, or a generation an earlier session on this box was admitted

// into under it.

- known := generation == c.Op.ID

+ known := c.Op.Kind == hostclaim.KindLanding && generation == c.Op.ID

+ if claimRecoveryKind(c.Op.Kind) && generation == c.Op.Journal[hostclaim.JournalSrcGen] {

+ known = true

+ }

for _, a := range c.Holder.Admissions {

if a.Generation == generation && a.Box == box && a.Session < c.Holder.Session {

known = true

@@ -79,14 +75,14 @@ func (r claimArtifactRetirement) Retire(ctx context.Context, admitted hostclaim.

if stamp == 0 || stamp >= c.Holder.Stamp || paths[database] != stamp {

return hostbox.ErrChanged

}

- if err := r.stoppedAt(ctx, admitted.Host, c, box, stamp); err != nil {

+ if err := r.artifactsStopped(ctx, admitted.Host, c, box, stamp, generation); err != nil {

return err

}

check := func(now *hostclaim.Claim) error {

if !claimSameWriter(fresh, hostclaim.Observed{Host: admitted.Host, Claim: now}, r.Continue) {

return hostbox.ErrChanged

}

- if _, ok := hostguard.LandingWriter(now, box); !ok {

+ if _, ok := claimArtifactWriter(now, box); !ok {

return hostbox.ErrChanged

}

return nil

@@ -132,7 +128,7 @@ func (r claimArtifactRetirement) Recover(ctx context.Context, admitted hostclaim

if r.Continue == nil || r.Local.Claims == nil || admitted.Claim == nil || admitted.Claim.Holder == nil || !filepath.IsAbs(in.DBPath) {

return hostbox.ErrChanged

}

- if admitted.Claim.Op.Kind != hostclaim.KindLanding {

+ if !claimArtifactKind(admitted.Claim.Op.Kind) {

return nil

}

if _, err := os.Lstat(in.DBPath + ".gen"); err == nil {

@@ -148,7 +144,7 @@ func (r claimArtifactRetirement) Recover(ctx context.Context, admitted hostclaim

return hostbox.ErrChanged

}

c, box := fresh.Claim, r.Local.Box

- if _, ok := hostguard.LandingWriter(c, box); !ok {

+ if _, ok := claimArtifactWriter(c, box); !ok {

return hostbox.ErrChanged

}

db, err := filepath.Rel(r.Local.DataDir, in.DBPath)

@@ -167,6 +163,12 @@ func (r claimArtifactRetirement) Recover(ctx context.Context, admitted hostclaim

if stamp > c.Holder.Stamp {

return hostbox.ErrChanged

}

+ if claimRecoveryKind(c.Op.Kind) {

+ sourceStamp, e := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64)

+ if e == nil && stamp == sourceStamp {

+ return r.Retire(ctx, admitted, in, c.Op.Journal[hostclaim.JournalSrcGen])

+ }

receipts, err := r.Local.Guard.CompletedWriterStops(ctx, admitted.Host, hostguard.Captured{Incarnation: c.Holder.Incarnation, Stamp: stamp}, c.Op.ID, box)

if err != nil {

return err

@@ -184,3 +186,38 @@ func (r claimArtifactRetirement) Recover(ctx context.Context, admitted hostclaim

}

return hostbox.ErrChanged

}

+

+func claimArtifactKind(kind string) bool {

+ return kind == hostclaim.KindLanding || claimRecoveryKind(kind)

+}

+func claimArtifactWriter(c *hostclaim.Claim, box string) (hostclaim.AdmissionRecord, bool) {

+ if c != nil && claimRecoveryKind(c.Op.Kind) {

+ return hostguard.RecoveryWriter(c, box)

+ }

+ return hostguard.LandingWriter(c, box)

+}

+func (r claimArtifactRetirement) artifactsStopped(ctx context.Context, host string, c *hostclaim.Claim, box string, stamp uint64, generation string) error {

+ sourceStamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64)

+ if !claimRecoveryKind(c.Op.Kind) || err != nil || stamp != sourceStamp {

+ return r.stoppedAt(ctx, host, c, box, stamp)

+ }

+ if c.Op.Intent["source"] != box || generation != c.Op.Journal[hostclaim.JournalSrcGen] {

+ return hostbox.ErrChanged

+ }

+ session, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalCutSession], 10, 64)

+ if err != nil || session >= c.Holder.Session {

+ return hostbox.ErrChanged

+ }

+ key := moves.StoppedKey(moves.ReportPrefix(c.Holder.Account, c.Holder.App, host, c.Op.ID), "source")

+ if c.Op.Journal[hostclaim.JournalEvidenceSource] != key {

+ return hostbox.ErrChanged

+ }

+ report, err := readStoppedReport(ctx, r.Local.Claims, key)

+ if err != nil {

+ return err

+ }

+ if report.Box != box || report.Role != "source" || report.Host != host || report.OpID != c.Op.ID || report.Incarnation != c.Holder.Incarnation {

+ return hostbox.ErrChanged

+ }

+ return nil

+}

diff --git a/internal/host/claimretirementcensus.go b/internal/host/claimretirementcensus.go

new file mode 100644

index 000000000..08d9d993f

--- /dev/null

+++ b/internal/host/claimretirementcensus.go

@@ -0,0 +1,142 @@

+package host

+

+import (

+ "context"

+ "errors"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/park"

+ "strings"

+ "time"

+)

+

+// ClaimRetirementCensus supplies identity; the regional engine retains retirement

+// authority and region-specific live-set checks. Bare paths need complete history.

+func ClaimRetirementCensus(o hostclaim.Observed, versions []lease.ObjectVersion) (map[string]string, error) {

+ if o.Absent || o.Claim == nil || o.ETag == "" {

+ return nil, hostbox.ErrChanged

+ }

+ if err := o.Claim.Validate(); err != nil {

+ return nil, err

+ }

+ locations, err := park.ResidueLocations(o.Host, versions)

+ if err != nil {

+ return nil, err

+ }

+ labels := map[string]string{}

+ var dependency error

+ for _, loc := range locations {

+ if loc == "moves/"+o.Host {

+ continue

+ }

+ if loc != "instances/"+o.Host {

+ // Identity is encoded in this validated qualified path, even beyond the history cap.

+ labels[loc] = strings.TrimPrefix(loc, "instances/"+o.Host+"/")

+ continue

+ }

+ inc, err := claimLegacyIncarnation(o.Claim)

+ if err != nil {

+ dependency = err

+ continue

+ }

+ labels[loc] = inc

+ }

+ return labels, dependency

+}

+func claimRetirementDependency() error {

+ return &ClaimDependencyError{Contracts: []string{"claim archived incarnation and location evidence"}}

+}

+

+func claimLegacyIncarnation(c *hostclaim.Claim) (string, error) {

+ if c == nil || len(c.RetiredIncarnations) >= 32 {

+ return "", claimRetirementDependency()

+ }

+ loc, inc := "instances/"+c.Host, ""

+ if c.Holder != nil && c.Holder.Location == loc {

+ inc = c.Holder.Incarnation

+ }

+ for _, retired := range c.RetiredIncarnations {

+ where := c.RetiredLocations[retired]

+ if where == "" {

+ return "", claimRetirementDependency()

+ }

+ if where == loc {

+ if inc != "" && inc != retired {

+ return "", claimRetirementDependency()

+ }

+ inc = retired

+ }

+ if inc == "" {

+ return "", claimRetirementDependency()

+ }

+ return inc, nil

+}

+func claimArchivedLocations(c *hostclaim.Claim) map[string]string {

+ labels := map[string]string{}

+ if c == nil {

+ return labels

+ }

+ if inc, err := claimLegacyIncarnation(c); err == nil {

+ labels["instances/"+c.Host] = inc

+ }

+ return labels

+}

+

+func claimFreeMoveProof(o hostclaim.Observed) (park.MoveSweepProof, error) {

+ if o.Absent || o.Claim == nil || o.ETag == "" || !o.Claim.Free() || len(o.Claim.RetiredIncarnations) == 0 {

+ return park.MoveSweepProof{}, claimRetirementDependency()

+ }

+ var at time.Time

+ for _, bound := range o.Claim.QuietAfter {

+ if bound.After(at) {

+ at = bound

+ }

+ if at.IsZero() {

+ return park.MoveSweepProof{}, claimRetirementDependency()

+ }

+ return park.MoveSweepProof{Host: o.Host, ClaimETag: o.ETag, MoveID: "claim-free", Incarnation: o.Claim.RetiredIncarnations[0], RetiredAt: at}, nil

+}

+

+func (p productionClaimRegional) retirement(ctx context.Context, o hostclaim.Observed, v []lease.ObjectVersion) (string, time.Time, error) {

+ labels, err := ClaimRetirementCensus(o, v)

+ var dependency *ClaimDependencyError

+ if err != nil && !errors.As(err, &dependency) {

+ return "", time.Time{}, err

+ }

+ now := time.Now().UTC()

+ if p.Now != nil {

+ now = p.Now().UTC()

+ }

+ if now.IsZero() {

+ return "", time.Time{}, hostbox.ErrChanged

+ }

+ var filtered []lease.ObjectVersion

+ for _, x := range v {

+ if strings.HasPrefix(x.Key, "moves/"+o.Host+"/") {

+ filtered = append(filtered, x)

+ }

+ if len(filtered) == 0 {

+ return labels["instances/"+o.Host], now, nil

+ }

+ if proof, err := claimFreeMoveProof(o); err == nil {

+ return proof.Incarnation, now, nil

+ }

+ label := labels["instances/"+o.Host]

+ if o.Claim.Holder != nil {

+ label = o.Claim.Holder.Incarnation

+ } else if len(o.Claim.RetiredIncarnations) > 0 {

+ label = o.Claim.RetiredIncarnations[0]

+ }

+ if p.Parks != nil && p.Parks.Evidence != nil {

+ inc, at, err := p.Parks.Evidence.VerifyRetirement(ctx, o, filtered)

+ if err == nil && !at.IsZero() && !at.After(now) {

+ return inc, now, nil

+ }

+ // A refused lane is dropped by VerifyMovesSweep; it cannot block proven instance residue.

+ return label, now, nil

+}

diff --git a/internal/host/claimroleassembly.go b/internal/host/claimroleassembly.go

index 5946bcc55..984652467 100644

--- a/internal/host/claimroleassembly.go

+++ b/internal/host/claimroleassembly.go

@@ -21,6 +21,7 @@ import (

type ClaimRoleAssembly struct {

// Root is the reservation-bearing isolated root launch protocol (claim

// Plan 11, R10). Nil keeps every claimed launch refused at Preflight.

+ prepareGroup func(string, cgroup.Limit) (*cgroup.Prepared, hostbox.LaunchGroup, error)

Root activator.RootRoleLauncher

Manager *cgroup.Manager

Journal *hostbox.LaunchJournal

diff --git a/internal/host/claimroleassembly_linux.go b/internal/host/claimroleassembly_linux.go

index 293d12334..7e044989d 100644

--- a/internal/host/claimroleassembly_linux.go

+++ b/internal/host/claimroleassembly_linux.go

@@ -51,21 +51,19 @@ func (a *ClaimRoleAssembly) Prelaunch(ctx context.Context, o hostclaim.Observed,

return nil, err

}

parentName := claimContainmentName(target)

- parentPath, err := filepath.Rel("/sys/fs/cgroup", filepath.Join(a.Manager.Root(), cgroup.ExecName(parentName)))

- if err != nil || !filepath.IsLocal(parentPath) {

- return nil, hostwrite.ErrUnavailable

- }

var parent *cgroup.Prepared

+ var group hostbox.LaunchGroup

err = work.With(ctx, in.Identity, func() error {

var e error

- if parent, e = a.Manager.PrepareStrict(parentName, in.Limit); e != nil {

- return e

+ prepare := a.prepareGroup

+ if prepare == nil {

+ prepare = a.preparePrelaunchGroup

}

- id, e := parent.Identity()

+ parent, group, e = prepare(parentName, in.Limit)

if e != nil {

return e

}

- return a.Journal.Prelaunch(ctx, hostbox.LaunchInventory{ControllerPID: os.Getpid(), ProcessStart: start, Target: target, Groups: []hostbox.LaunchGroup{{RelativePath: parentPath, Device: id.Device, Inode: id.Inode}}, NoState: true})

+ return a.Journal.Prelaunch(ctx, hostbox.LaunchInventory{ControllerPID: os.Getpid(), ProcessStart: start, Target: target, Groups: []hostbox.LaunchGroup{group}, NoState: true})

})

if err != nil {

if parent != nil {

@@ -73,7 +71,7 @@ func (a *ClaimRoleAssembly) Prelaunch(ctx context.Context, o hostclaim.Observed,

}

return nil, err

}

- return &activator.RoleLaunch{Parent: parent, ParentPath: parentPath, CgroupRoot: "/sys/fs/cgroup", Target: target, Work: work, Identity: in.Identity, Journal: a.Journal}, nil

+ return &activator.RoleLaunch{Parent: parent, ParentPath: group.RelativePath, CgroupRoot: "/sys/fs/cgroup", Target: target, Work: work, Identity: in.Identity, Journal: a.Journal}, nil

}

// Launch completes the pre-launch record after restore and materialization.

@@ -377,3 +375,20 @@ func (a *ClaimRoleAssembly) PrepareState(ctx context.Context, o hostclaim.Observ

return a.Inventory.WriteStopInventory(ctx, inventory)

})

}

+

+func (a *ClaimRoleAssembly) preparePrelaunchGroup(name string, limit cgroup.Limit) (*cgroup.Prepared, hostbox.LaunchGroup, error) {

+ path, err := filepath.Rel("/sys/fs/cgroup", filepath.Join(a.Manager.Root(), cgroup.ExecName(name)))

+ if err != nil || !filepath.IsLocal(path) {

+ return nil, hostbox.LaunchGroup{}, hostwrite.ErrUnavailable

+ }

+ parent, err := a.Manager.PrepareStrict(name, limit)

+ if err != nil {

+ return nil, hostbox.LaunchGroup{}, err

+ }

+ id, err := parent.Identity()

+ if err != nil {

+ parent.Close()

+ return nil, hostbox.LaunchGroup{}, err

+ }

+ return parent, hostbox.LaunchGroup{RelativePath: path, Device: id.Device, Inode: id.Inode}, nil

+}

diff --git a/internal/host/claimwakeassembly.go b/internal/host/claimwakeassembly.go

index 6c64291ce..56aec0dac 100644

--- a/internal/host/claimwakeassembly.go

+++ b/internal/host/claimwakeassembly.go

@@ -90,18 +90,20 @@ type claimSyncer interface {

// ClaimWakeAssembly owns cold restore, lease transfer and failure cleanup.

// Root supplies Stop and Launch; moves supplies only its input/publication API.

type ClaimWakeAssembly struct {

- retained []*retainedClaimWork

- AfterAdmission func(context.Context, hostclaim.Observed, hostclaim.Observed) error

- BeforeAdmission func(context.Context, hostclaim.Observed) error

- Roles *ClaimRoleAssembly

- LeaseRegion string

- Producers func(context.Context, hostclaim.Observed) (*ClaimProducers, error)

- Inputs ClaimWakeInputs

- Manager *cgroup.Manager

- Retries activator.ServingRetries

- Stop func(context.Context, hostclaim.Observed) error

- RetireArtifacts func(context.Context, hostclaim.Observed, ClaimLaunchInputs, string) error

- RecoverArtifacts func(context.Context, hostclaim.Observed, ClaimLaunchInputs) error

+ recoveryPrior func(context.Context, hostclaim.Observed) (hostclaim.Observed, bool, error)

+ recoveryPrelaunched func(context.Context, hostclaim.Observed) error

+ retained []*retainedClaimWork

+ AfterAdmission func(context.Context, hostclaim.Observed, hostclaim.Observed) error

+ BeforeAdmission func(context.Context, hostclaim.Observed) error

+ Roles *ClaimRoleAssembly

+ LeaseRegion string

+ Producers func(context.Context, hostclaim.Observed) (*ClaimProducers, error)

+ Inputs ClaimWakeInputs

+ Manager *cgroup.Manager

+ Retries activator.ServingRetries

+ Stop func(context.Context, hostclaim.Observed) error

+ RetireArtifacts func(context.Context, hostclaim.Observed, ClaimLaunchInputs, string) error

+ RecoverArtifacts func(context.Context, hostclaim.Observed, ClaimLaunchInputs) error

// Prelaunch journals the admitted session's recoverable record before

// restore or materialization; Launch later completes that same record.

PrepareState func(context.Context, hostclaim.Observed, hostwrite.Work, ClaimLaunchInputs) error

@@ -125,17 +127,18 @@ type ClaimWakeAssembly struct {

pending map[string]*claimWakeAttempt

}

type claimWakeAttempt struct {

- releaseMu sync.Mutex

- prelaunch *activator.RoleLaunch

- launch *activator.RoleLaunch

- repDone chan struct{}

- repErr error

- released bool

- observed hostclaim.Observed

- inputs ClaimLaunchInputs

- held claimHeldLease

- rep *lease.Replicator

- syncer claimSyncer

+ resumedAdmission bool

+ releaseMu sync.Mutex

+ prelaunch *activator.RoleLaunch

+ launch *activator.RoleLaunch

+ repDone chan struct{}

+ repErr error

+ released bool

+ observed hostclaim.Observed

+ inputs ClaimLaunchInputs

+ held claimHeldLease

+ rep *lease.Replicator

+ syncer claimSyncer

// started is the attempt the latest operation start passed to Start.

started *hostclaim.WakeRetryAttempt

// publisher closes when the fenced publisher has joined; receipt and

@@ -202,6 +205,23 @@ func (w *ClaimWakeAssembly) Build(ctx context.Context, source hostbox.WorkSource

return hostclaim.Read(ctx, writer.Claims, h)

}

runtime := &activator.ClaimRuntime{Box: writer.Box, Read: read, RecoveryAttemptsReady: w.recoveryAttemptsReady}

+ runtime.ResumedAdmission = func(call context.Context, prior, admitted hostclaim.Observed) error {

+ a := w.get(admitted.Host)

+ if a == nil || prior.ETag == "" || prior.ETag != admitted.ETag || prior.Host != admitted.Host {

+ return hostbox.ErrChanged

+ }

+ current, err := read(call, admitted.Host)

+ if err != nil {

+ return err

+ }

+ a.mu.Lock()

+ defer a.mu.Unlock()

+ if !a.resumedAdmission || a.observed.ETag != admitted.ETag || current.ETag != admitted.ETag {

+ return hostbox.ErrChanged

+ }

+ a.resumedAdmission = false

+ return nil

+ }

runtime.StopForeign = func(ctx context.Context, h string) error {

o, err := read(ctx, h)

if err != nil {

@@ -225,7 +245,11 @@ func (w *ClaimWakeAssembly) Build(ctx context.Context, source hostbox.WorkSource

if rt.Seed != "" || rt.Backing != "" || rt.Unit != "" || rt.IsSidecar() || rt.Tunnel != "" {

return hostbox.ErrUnsupported

}

- in, err := w.Inputs.Select(call, rt, o, p)

+ prior, _, err := w.pendingRecovery(call, o)

+ if err != nil {

+ return err

+ }

+ in, err := w.Inputs.Select(call, rt, prior, p)

if err != nil {

return err

}

@@ -304,18 +328,27 @@ func (w *ClaimWakeAssembly) Build(ctx context.Context, source hostbox.WorkSource

}

// The retry reservation changed the ETag. Re-select the typed provider

// against that observation; never re-use authorization from the old read.

- in, err := w.Inputs.Select(call, rt, o, p)

+ prior, resuming, err := w.pendingRecovery(call, o)

if err != nil {

return hostclaim.Observed{}, err

}

- admitted, err := writer.Admit(call, o, in.Admission, in.Baseline)

+ in, err := w.Inputs.Select(call, rt, prior, p)

+ if err != nil {

+ return hostclaim.Observed{}, err

+ }

+ admitted := o

+ if !resuming {

+ admitted, err = writer.Admit(call, o, in.Admission, in.Baseline)

+ } else {

+ in.Admission.NewGeneration = o.Claim.Holder.Generation

+ }

if err != nil {

return admitted, err

}

a.mu.Lock()

a.observed, a.inputs = admitted, in

a.mu.Unlock()

- if err := w.AfterAdmission(call, o, admitted); err != nil {

+ if err := w.AfterAdmission(call, prior, admitted); err != nil {

return admitted, err

}

if err := source.Guard.Activate(call, rt.Host, writer.Box, nil); err != nil {

@@ -326,6 +359,14 @@ func (w *ClaimWakeAssembly) Build(ctx context.Context, source hostbox.WorkSource

if err := w.persistPrelaunch(call, a, source, admitted, in); err != nil {

return admitted, err

}

+ if w.recoveryPrelaunched != nil {

+ if err := w.recoveryPrelaunched(call, admitted); err != nil {

+ return admitted, err

+ }

+ a.mu.Lock()

+ a.resumedAdmission = resuming

+ a.mu.Unlock()

return admitted, nil

}

runtime.Prepare = func(call context.Context, rt registry.Route, o hostclaim.Observed, p activator.StartPolicy) (activator.ClaimStart, error) {

@@ -1009,7 +1050,20 @@ func (w *ClaimWakeAssembly) exactHealth(ctx context.Context, cmd *exec.Cmd, atte

// stopPredecessor uses sealed cross-box evidence only when production wired

// that path. The ordinary local stop remains the default.

+func (w *ClaimWakeAssembly) pendingRecovery(ctx context.Context, o hostclaim.Observed) (hostclaim.Observed, bool, error) {

+ if w.recoveryPrior == nil {

+ return o, false, nil

+ }

+ return w.recoveryPrior(ctx, o)

+}

func (w *ClaimWakeAssembly) stopPredecessor(ctx context.Context, o hostclaim.Observed) error {

+ prior, pending, err := w.pendingRecovery(ctx, o)

+ if err != nil {

+ return err

+ }

+ if pending {

+ return w.AfterAdmission(ctx, prior, o)

+ }

if w.BeforeAdmission != nil {

return w.BeforeAdmission(ctx, o)

}

diff --git a/internal/host/claimwakestop.go b/internal/host/claimwakestop.go

index 2b8fcdc44..cf3b8d883 100644

--- a/internal/host/claimwakestop.go

+++ b/internal/host/claimwakestop.go

@@ -175,6 +175,9 @@ func (a ClaimAssembly) stopCapturedSession(ctx context.Context, captured hostcla

if err != nil {

return err

}

+ if a.stopReaches() != nil && claimInitialSourceRecovery(captured, local.Actor.Box) {

+ return (claimCrossBoxHandover{Guard: a.Guard, Evidence: a.Claims, Box: local.Actor.Box}).beforeRecovery(ctx, captured)

+ }

if h.Session == 0 {

// No writer was ever admitted, so nothing was launched or journaled.

// Root's positive no-process witness (R4) is the stop: this

@@ -365,5 +368,8 @@ func (a ClaimAssembly) operationWriterStop(c *hostclaim.Claim, actor hostbox.Act

if c != nil && c.Op.Kind == hostclaim.KindRelocating {

return hostguard.RelocationWriterStop(c, actor.Box, actor.Region)

}

+ if c != nil && claimRecoveryKind(c.Op.Kind) {

+ return hostguard.RecoveryWriterStop(c, actor.Box, actor.Region)

+ }

return hostguard.LandingWriterStop(c, actor.Box, actor.Region)

}

diff --git a/internal/host/claimwiring.go b/internal/host/claimwiring.go

index 52a5488b9..8648e88d0 100644

--- a/internal/host/claimwiring.go

+++ b/internal/host/claimwiring.go

@@ -7,6 +7,7 @@ import (

"github.com/carlosframework/platform/internal/activator"

"github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

"github.com/carlosframework/platform/internal/hostwrite"

"github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/schedule"

@@ -19,14 +20,15 @@ type RegionalLoop interface {

}

type ClaimProviders struct {

- moveParks *claimMoveParks

- Delivery schedule.DeliveryAdmission

- Selector hostwrite.Selector

- Wake *activator.ClaimRuntime

- Regional RegionalLoop

- Stop *ClaimStopCoordinator

- Provision func(*provision.ReconcileConfig)

- Close func(context.Context) error

+ moveTargetFence func(context.Context, hostclaim.Observed) (hostclaim.ReturnFence, error)

+ moveParks *claimMoveParks

+ Delivery schedule.DeliveryAdmission

+ Selector hostwrite.Selector

+ Wake *activator.ClaimRuntime

+ Regional RegionalLoop

+ Stop *ClaimStopCoordinator

+ Provision func(*provision.ReconcileConfig)

+ Close func(context.Context) error

}

type ClaimFactory func(context.Context) (ClaimProviders, error)

diff --git a/internal/host/movessources.go b/internal/host/movessources.go

index e3caad3ed..72a23bda7 100644

--- a/internal/host/movessources.go

+++ b/internal/host/movessources.go

@@ -570,7 +570,7 @@ func newMovesParkSources(records MovesRecordReader, data genselect.Reader, bucke

// registerMovesParks registers the moves park providers with the claim

// (RegisterProductionClaimMoveParks), at agent start before newClaimWiring.

func registerMovesParks(src movesParkSources) error {

- return RegisterProductionClaimMoveParks(src.View, src.Evidence, src.Leases)

+ return RegisterProductionClaimMoveParks(src.View, src.Evidence)

}

// armMovesParks sets the moves arms' park hooks from the claim's MoveParks

@@ -582,7 +582,7 @@ func armMovesParks(ctx context.Context, cfg *moveCfg, providers ClaimProviders,

if ctx == nil || cfg == nil || plan12 == nil {

return hostwrite.ErrUnavailable

}

- engine, _, err := providers.MoveParks(ctx)

+ engine, _, local, err := providers.MoveParks(ctx)

if err != nil {

return fmt.Errorf("the claim's MoveParks: %w", err)

}

@@ -600,7 +600,10 @@ func armMovesParks(ctx context.Context, cfg *moveCfg, providers ClaimProviders,

return &ClaimDependencyError{Contracts: movesPlan12Contracts}

}

cfg.ParkLeases, cfg.ParkView, cfg.ParkAuthorize = engine.Leases, engine.View, engine.Authorize

- cfg.ParkRunHeld, cfg.ParkLocalAuthorize = p12.RunHeld, p12.AuthorizeLocal

+ cfg.ParkRunHeld = func(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error {

+ return e.RunHeld(ctx, w, held)

+ }

+ cfg.ParkLocalAuthorize = local

return nil

}

diff --git a/internal/host/movessources_test.go b/internal/host/movessources_test.go

index dcef3995a..b46336d09 100644

--- a/internal/host/movessources_test.go

+++ b/internal/host/movessources_test.go

@@ -330,8 +330,13 @@ func (p *fakePlan12) AuthorizeLocal(context.Context, *hostclaim.Claim, hostclaim

func TestArmMovesPassRefusesAMismatchedBucketOrRegion(t *testing.T) {

view := func(context.Context, hostclaim.Observed) (park.View, error) { return park.View{}, nil }

auth := func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }

+ locals := 0

providers := func(bucket, region string) ClaimProviders {

return ClaimProviders{moveParks: &claimMoveParks{

+ local: func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error {

+ locals++

+ return nil

+ },

engine: park.Engine{Bucket: bucket, Region: region, Leases: &fakeParkLeases{}, View: view, Authorize: auth},

leases: movesLeaseSet(bucket), check: func(context.Context) error { return nil }}}

}

@@ -381,7 +386,7 @@ func TestArmMovesPassRefusesAMismatchedBucketOrRegion(t *testing.T) {

if c.ParkLeases == nil || c.ParkView == nil || c.ParkAuthorize == nil || c.ParkRunHeld == nil || c.ParkLocalAuthorize == nil {

t.Fatalf("hooks unset: %+v", c)

}

- if err := c.ParkLocalAuthorize(t.Context(), nil, hostclaim.Obligation{}); err != nil || plan12.locals != 1 {

+ if err := c.ParkLocalAuthorize(t.Context(), nil, hostclaim.Obligation{}); err != nil || locals != 1 || plan12.locals != 0 {

t.Fatal("the local authorizer is not claim Plan 12's")

}

})

@@ -436,7 +441,7 @@ func TestMovesParksRefuseUntilClaimPlan12(t *testing.T) {

if err := registerMovesParks(src); err != nil {

t.Fatal(err)

}

- if p := claimMoveSources.Parks; p == nil || p.View == nil || p.Leases == nil || p.Evidence == nil {

+ if p := claimMoveSources.Parks; p == nil || p.View == nil || p.Evidence == nil {

t.Fatalf("registered %+v", p)

}

if err := registerMovesParks(src); !errors.Is(err, hostwrite.ErrUnavailable) {

@@ -568,6 +573,7 @@ func TestHomeBoxArms(t *testing.T) {

}

auth := func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }

providers := ClaimProviders{moveParks: &claimMoveParks{

+ local: func(context.Context, *hostclaim.Claim, hostclaim.Obligation) error { return nil },

engine: park.Engine{Bucket: "data-home", Region: "", Leases: &fakeParkLeases{}, View: view, Authorize: auth},

leases: movesLeaseSet("data-home"), check: func(context.Context) error { return nil }}}

c := moveCfg{OwnBucket: "data-home", Region: ""}

diff --git a/internal/host/plan10_j2_test.go b/internal/host/plan10_j2_test.go

index 6330f2b2e..d90eb31cd 100644

--- a/internal/host/plan10_j2_test.go

+++ b/internal/host/plan10_j2_test.go

@@ -303,7 +303,7 @@ func TestPlan10J2AdmissionFailureReleasesTheRecoveryLease(t *testing.T) {

// The recovery's local artifact retirement stays fail-closed: the landing's

// retire-artifacts is bound to the landing writer (hostguard.LandingWriter),

// and a recovery op is not one. Recover has nothing to re-drive for it.

-func TestPlan10J2ArtifactRetirementStaysLandingOnly(t *testing.T) {

+func TestPlan10J2ArtifactRetirementRefusesUnadmittedRecovery(t *testing.T) {

f := newRecoveryFixture(t)

dataDir := f.guard.DataDir

r := claimArtifactRetirement{Local: park.Local{Claims: f.claims, Box: "box-1", DataDir: dataDir}, Continue: hostclaim.SessionContinues}

@@ -314,42 +314,12 @@ func TestPlan10J2ArtifactRetirementStaysLandingOnly(t *testing.T) {

if err := r.Retire(t.Context(), o, in, "m1"); !errors.Is(err, hostbox.ErrChanged) {

t.Fatalf("%s retire: %v", kind, err)

}

- if err := r.Recover(t.Context(), o, in); err != nil {

+ if err := r.Recover(t.Context(), o, in); !errors.Is(err, hostbox.ErrChanged) {

t.Fatalf("%s recover: %v", kind, err)

}

-// Cross-box handover and predecessor evidence stay landing-only: a recovery

-// op whose previous writer ran on another box goes to the local path, which

-// needs this box's own completed fence receipt.

-func TestPlan10J2CrossBoxCodeStaysLandingOnly(t *testing.T) {

- var calls []string

- local := func(context.Context, hostclaim.Observed, hostclaim.Observed) error {

- calls = append(calls, "local")

- return nil

- }

- cross := func(context.Context, hostclaim.Observed, hostclaim.Observed) error {

- calls = append(calls, "cross")

- return nil

- }

- hook := claimAdmissionHandover("box-1", local, cross)

- for _, kind := range []string{hostclaim.KindRestoring, hostclaim.KindReverting} {

- calls = nil

- prior := plan10J2Observed(kind, "probing", "box-1")

- prior.Claim.Holder.WriterBox = "box-2"

- if err := hook(t.Context(), prior, hostclaim.Observed{}); err != nil || len(calls) != 1 || calls[0] != "local" {

- t.Fatalf("%s admission went to %v (%v)", kind, calls, err)

- }

- localStops, remote := 0, 0

- w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { localStops++; return nil }}

- w.BeforeAdmission = claimPredecessorStop(w, "box-1", func(context.Context, hostclaim.Observed) error { remote++; return nil })

- if err := w.stopPredecessor(t.Context(), prior); err != nil || localStops != 1 || remote != 0 {

- t.Fatalf("%s predecessor stop: local=%d remote=%d %v", kind, localStops, remote, err)

- }

-}

-

// A clean return records its current inputs at probing before the held

// render and launch. No configured phase is needed. The adapter must call

// recoveryProbeRequest.Record first; movesrecoveryprobe.go supplies the adapter.

diff --git a/internal/host/plan12_census_test.go b/internal/host/plan12_census_test.go

new file mode 100644

index 000000000..338f784a6

--- /dev/null

+++ b/internal/host/plan12_census_test.go

@@ -0,0 +1,164 @@

+package host

+

+import (

+ "errors"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/park"

+ "strings"

+ "testing"

+ "time"

+)

+

+func plan12Retired(t *testing.T) hostclaim.Observed {

+ t.Helper()

+ t.Setenv("CARLOS_HOST_CLAIM_RETIRED_LOCATIONS", "1")

+ f := newRecoveryFixture(t)

+ o := f.read()

+ var err error

+ o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {

+ c.Holder.Location = "instances/" + o.Host

+ c.Touched = hostclaim.Touched{Complete: true}

+ return hostclaim.BeginDelete("delete", "console", "record", f.now, nil, nil)(c)

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ o, err = hostclaim.Update(t.Context(), f.claims, o, hostclaim.ToParking("delete", nil, nil, f.newID))

+ if err != nil {

+ t.Fatal(err)

+ }

+ o, err = hostclaim.Update(t.Context(), f.claims, o, hostclaim.FreeAfterPark("delete"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ return o

+}

+func TestPlan12ArchivedIncarnationCensus(t *testing.T) {

+ for _, mode := range []string{"legacy", "old-incarnation", "missing-history", "qualified-beyond-history", "ambiguous-legacy"} {

+ t.Run(mode, func(t *testing.T) {

+ o := plan12Retired(t)

+ inc := o.Claim.RetiredIncarnations[0]

+ loc := "instances/" + o.Host

+ if mode == "old-incarnation" {

+ loc += "/" + inc

+ }

+ if mode == "missing-history" {

+ o.Claim.RetiredLocations = nil

+ }

+ if mode == "qualified-beyond-history" {

+ loc += "/" + strings.Repeat("b", 64)

+ }

+ if mode == "ambiguous-legacy" {

+ other := strings.Repeat("b", 64)

+ o.Claim.RetiredIncarnations = append(o.Claim.RetiredIncarnations, other)

+ o.Claim.RetiredLocations[other] = loc

+ }

+ versions := []lease.ObjectVersion{{Key: loc + "/0000/db", VersionID: "v1"}}

+ labels, err := ClaimRetirementCensus(o, versions)

+ if mode == "qualified-beyond-history" {

+ inc = strings.Repeat("b", 64)

+ }

+ if mode == "legacy" || mode == "old-incarnation" || mode == "qualified-beyond-history" {

+ if err != nil || labels[loc] != inc {

+ t.Fatal(labels, err)

+ }

+ } else {

+ var dependency *ClaimDependencyError

+ if !errors.As(err, &dependency) {

+ t.Fatal("not a typed dependency", err)

+ }

+ })

+ }

+}

+func TestPlan12LegacyAndMovesHaveIndependentLabels(t *testing.T) {

+ o := plan12Retired(t)

+ legacy := o.Claim.RetiredIncarnations[0]

+ move := strings.Repeat("c", 64)

+ versions := []lease.ObjectVersion{{Key: "instances/" + o.Host + "/0000/db", VersionID: "v1"}, {Key: "moves/" + o.Host + "/op/replica/db", VersionID: "v2"}}

+ view := park.View{ClaimETag: o.ETag, Region: "ie", ArchivedIncarnations: claimArchivedLocations(o.Claim)}

+ d, err := park.Decide(o, view, versions, move)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if len(d.Candidates) != 2 {

+ t.Fatal(d)

+ }

+ for _, c := range d.Candidates {

+ want := move

+ if strings.HasPrefix(c.Location, "instances/") {

+ want = legacy

+ }

+ if c.Incarnation != want {

+ t.Fatal(c)

+ }

+ p := productionClaimRegional{Now: func() time.Time { return time.Unix(100, 0) }}

+ label, at, err := p.retirement(t.Context(), o, versions[:1])

+ if err != nil || label != legacy || at.IsZero() {

+ t.Fatal(label, at, err)

+ }

+}

+

+func TestPlan12HolderResidueInOtherRegion(t *testing.T) {

+ f := newRecoveryFixture(t)

+ o := f.read()

+ c := o.Claim

+ c.Holder.Location = "instances/" + o.Host

+ c.Holder.Generation = "live"

+ loc := c.Holder.Location

+ versions := []lease.ObjectVersion{{Key: loc + "/live/db", VersionID: "v1"}}

+ labels, err := ClaimRetirementCensus(o, versions)

+ if err != nil || labels[loc] != c.Holder.Incarnation {

+ t.Fatal(labels, err)

+ }

+ view := park.View{ClaimETag: o.ETag, Region: "uk", AuthoritativeRegion: "ie", ArchivedIncarnations: labels}

+ d, err := park.Decide(o, view, versions, "")

+ if err != nil || len(d.Candidates) != 1 || d.Candidates[0].Incarnation != c.Holder.Incarnation {

+ t.Fatal(d, err)

+ }

+ view.Region = "ie"

+ view.MarkerKnown = true

+ view.MarkerGeneration = "live"

+ view.GenerationOf = func(park.Ref) (string, bool) { return "live", true }

+ d, err = park.Decide(o, view, versions, "")

+ if err != nil || len(d.Candidates) != 0 {

+ t.Fatal("live generation swept", d, err)

+ }

+}

+func TestPlan12CensusNamespacesAndFreeLanes(t *testing.T) {

+ o := plan12Retired(t)

+ o.Claim.QuietAfter = map[string]time.Time{"ie": time.Unix(10, 0), "uk": time.Unix(20, 0)}

+ proof, err := claimFreeMoveProof(o)

+ if err != nil || proof.Check(o, o.Claim.RetiredIncarnations[0], time.Unix(21, 0)) != nil {

+ t.Fatal(proof, err)

+ }

+ if proof.Check(o, proof.Incarnation, time.Unix(20, 0)) == nil {

+ t.Fatal("accepted listing before retirement")

+ }

+ delete(o.Claim.RetiredLocations, o.Claim.RetiredIncarnations[0])

+ v := []lease.ObjectVersion{{Key: "instances/" + o.Host + "/0000/db", VersionID: "one"}, {Key: "moves/" + o.Host + "/move/db", VersionID: "two"}}

+ p := productionClaimRegional{Now: func() time.Time { return time.Unix(30, 0) }}

+ label, _, err := p.retirement(t.Context(), o, v)

+ if err != nil || label != proof.Incarnation {

+ t.Fatal("legacy refusal blocked lanes", err)

+ }

+ view := park.View{ClaimETag: o.ETag, Region: "ie", RefusedLocations: map[string]error{"instances/" + o.Host: claimRetirementDependency()}}

+ d, err := park.Decide(o, view, v, label)

+ if err != nil || len(d.Candidates) != 1 || d.Candidates[0].Location != "moves/"+o.Host {

+ t.Fatal(d, err)

+ }

+ o.Claim.QuietAfter = nil

+ if _, err := claimFreeMoveProof(o); err == nil {

+ t.Fatal("invented retirement bound")

+ }

+}

+func TestPlan12QualifiedIdentityBeyondHistoryCap(t *testing.T) {

+ o := plan12Retired(t)

+ loc := "instances/" + o.Host + "/" + strings.Repeat("e", 64)

+ labels, err := ClaimRetirementCensus(o, []lease.ObjectVersion{{Key: loc + "/g/db", VersionID: "v"}})

+ if err != nil || labels[loc] != strings.Repeat("e", 64) {

+ t.Fatal(labels, err)

+ }

+}

diff --git a/internal/host/plan12_handover_test.go b/internal/host/plan12_handover_test.go

new file mode 100644

index 000000000..2572762ee

--- /dev/null

+++ b/internal/host/plan12_handover_test.go

@@ -0,0 +1,244 @@

+package host

+

+import (

+ "context"

+ "crypto/ed25519"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "fmt"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/termfence"

+ "os/exec"

+ "path/filepath"

+ "strings"

+ "testing"

+ "time"

+)

+

+type plan12FenceAuthority struct{ reject bool }

+

+func (a plan12FenceAuthority) ValidateFence(context.Context, termfence.Record) (hostclaim.FenceBinding, error) {

+ if a.reject {

+ return hostclaim.FenceBinding{}, hostclaim.ErrFence

+ }

+ return hostclaim.FenceBinding{Region: "uk", RosterRevision: 1, RosterETag: "roster"}, nil

+}

+func plan12Handover(t *testing.T, kind, mode string) (*recoveryFixture, hostclaim.Observed, claimCrossBoxHandover) {

+ t.Helper()

+ f := newRecoveryFixture(t)

+ o := f.read()

+ key := moves.StoppedKey(moves.ReportPrefix("bab", "jam", o.Host, "m1"), "target")

+ journal := map[string]string{hostclaim.JournalEvidenceTarget: key, hostclaim.JournalTargetSession: "2", hostclaim.JournalTargetGeneration: "target-gen"}

+ if mode == "wrong-session" {

+ journal[hostclaim.JournalTargetSession] = "3"

+ }

+ pub := f.signer.Public().(ed25519.PublicKey)

+ h := claimCrossBoxHandover{Guard: f.guard, Evidence: f.claims, Box: "box-1", PublicKey: pub, FenceAuthority: plan12FenceAuthority{reject: mode == "unverified-fence"}}

+ if mode == "fence" || mode == "unverified-fence" {

+ record := termfence.Record{V: 1, Box: "box-2", Enrollment: strings.Repeat("a", 32), State: "terminated", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)}

+ raw, err := json.Marshal(record)

+ if err != nil {

+ t.Fatal(err)

+ }

+ sum := sha256.Sum256(raw)

+ ref := hostclaim.FenceRef{Enrollment: record.Enrollment, Key: "control/box-fences/" + record.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])}

+ if _, err = f.claims.Put(t.Context(), ref.Key, raw, ""); err != nil {

+ t.Fatal(err)

+ }

+ refRaw, _ := json.Marshal(ref)

+ journal = map[string]string{"fence.box-2": string(refRaw)}

+ }

+ if mode == "loss" || mode == "wrong-loss-op" {

+ journal = map[string]string{"loss.loss_1": "{}"}

+ }

+ var err error

+ o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {

+ c.Op = plan10J2Op(kind, hostclaim.PhaseProbing)

+ c.Op.Journal = journal

+ c.Op.Intent["target_enrollment"] = strings.Repeat("a", 32)

+ c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 2, "target-gen", "box-2"

+ if mode == "loss" || mode == "wrong-loss-op" {

+ c.LossRefs = []string{"loss_1"}

+ }

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ if err = f.guard.Activate(t.Context(), o.Host, "box-1", nil); err != nil {

+ t.Fatal(err)

+ }

+ report := plan12Stopped(o, "target", "box-2")

+ if mode == "wrong-op" {

+ report.OpID = "another"

+ }

+ raw, err := json.Marshal(report)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err = f.claims.Put(t.Context(), key, raw, ""); err != nil {

+ t.Fatal(err)

+ }

+

+ if mode == "loss" || mode == "wrong-loss-op" {

+ now := time.Now().UTC()

+ r := hostclaim.LossRecord{V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: o.Host, HolderIncarnation: o.Claim.Holder.Incarnation, HolderStamp: o.Claim.Holder.Stamp, HolderSession: 2, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: "m1", OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "box-2", Enrollment: "enroll", QuarantineRevision: 1,

+ CloudObservation: hostclaim.LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now,

+ ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"move-target-lost"}`)}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, WriteSources: []hostclaim.WriteSource{{Reason: "target", Box: "box-2", Session: 2, Generation: "target-gen", Region: "uk", Baseline: "target-snapshot"}}}

+ if mode == "wrong-loss-op" {

+ r.OpID = "other"

+ }

+ made, err := hostclaim.CreateLossRecord(t.Context(), f.claims, release.RootSigner(f.signer), r)

+ if err != nil {

+ t.Fatal(err)

+ }

+ r.State, r.ResultClaimGen = "committed", o.Claim.Gen

+ if _, err = hostclaim.ReplaceLossRecord(t.Context(), f.claims, release.RootSigner(f.signer), made, r); err != nil {

+ t.Fatal(err)

+ }

+ h.Identity = handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Region: "ie"})

+ h.Inventory, err = hostbox.OpenStopInventoryDB(filepath.Join(t.TempDir(), "inventory.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { h.Inventory.Close() })

+ h.Registry, err = registry.Open(filepath.Join(t.TempDir(), "registry.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { h.Registry.Close() })

+ row := registry.Route{Host: o.Host, Kind: "instance", DBPath: provision.NestedDBPath(f.guard.DataDir, o.Host), ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: o.Claim.Holder.Incarnation, ClaimStamp: o.Claim.Holder.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-row"}}

+ if err = h.Registry.Upsert(row); err != nil {

+ t.Fatal(err)

+ }

+ if err = h.Inventory.WriteStopInventory(t.Context(), hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: row.ClaimIncarnation, InstallationStamp: row.ClaimStamp, Box: "box-1", Enrollment: "enrollment-1", DirectoryRevision: "7", Entries: []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: o.Host, RowIdentity: "old-row", Ownership: "old-owner", Materialization: "materialized"}}}); err != nil {

+ t.Fatal(err)

+ }

+ o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {

+ c.Op.Journal[hostclaim.JournalSourceStamp] = fmt.Sprint(row.ClaimStamp)

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ return f, o, h

+}

+func TestPlan12RecoveryPredecessorEvidence(t *testing.T) {

+ for _, kind := range []string{hostclaim.KindRestoring} {

+ for _, mode := range []string{"stop", "loss", "fence", "wrong-session", "wrong-op", "wrong-loss-op", "unverified-fence"} {

+ t.Run(kind+"/"+mode, func(t *testing.T) {

+ _, o, h := plan12Handover(t, kind, mode)

+ if !claimRemotePredecessor(o, "box-1") {

+ t.Fatal("local path selected")

+ }

+ err := h.Before(t.Context(), o)

+ valid := mode == "stop" || mode == "loss" || mode == "fence"

+ if valid && err != nil || !valid && err == nil {

+ t.Fatalf("%s: %v", mode, err)

+ }

+ })

+ }

+}

+func TestPlan12RecoveryAdmissionAfterTargetSession(t *testing.T) {

+ for _, kind := range []string{hostclaim.KindRestoring} {

+ t.Run(kind, func(t *testing.T) {

+ f, o, h := plan12Handover(t, kind, "stop")

+ c := f.controller("boot-1", "controller-1", 4101, "start-1")

+ c.wake.Inputs = plan9D2Inputs{recoveryInputs{f}}

+ c.wake.Operations = &plan9D2Attempts{}

+ c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }

+ wireClaimAdmission(c.wake, "box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error {

+ t.Fatal("local predecessor handover")

+ return nil

+ }, h, nil)

+ if err := c.wake.stopPredecessor(t.Context(), o); err != nil {

+ t.Fatal(err)

+ }

+ live, cancel := context.WithCancel(t.Context())

+ defer cancel()

+ done := make(chan struct{})

+ go func() { <-live.Done(); close(done) }()

+ c.wake.mu.Lock()

+ c.wake.pending[o.Host] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done}

+ c.wake.mu.Unlock()

+ got, err := c.runtime.Admit(t.Context(), registry.Route{Host: o.Host}, o, activator.StartPolicy{Probe: true})

+ if err != nil {

+ t.Fatal(err)

+ }

+ if got.Claim.Holder.Session <= 2 || got.Claim.Holder.Stamp != got.Claim.Holder.Session || got.Claim.Holder.WriterBox != "box-1" {

+ t.Fatal(got)

+ }

+ })

+ }

+}

+

+func plan12Stopped(o hostclaim.Observed, role, box string) moves.StoppedReport {

+ return moves.StoppedReport{V: 1, Role: role, Box: box, Host: o.Host, OpID: o.Claim.Op.ID, Incarnation: o.Claim.Holder.Incarnation, ClaimGen: o.Claim.Gen, CheckedAt: time.Now().UTC(), Steps: map[string]moves.Check{moves.StepTree: {OK: true}, moves.StepStray: {OK: true}, moves.StepFDs: {OK: true}, moves.StepLease: {OK: true}, moves.StepReplica: {OK: true}}}

+}

+func TestPlan12RecoveryLocalRouting(t *testing.T) {

+ for _, kind := range []string{hostclaim.KindRestoring, hostclaim.KindReverting} {

+ for _, box := range []string{"", "box-1"} {

+ o := plan10J2Observed(kind, hostclaim.PhaseProbing, "box-1")

+ o.Claim.Holder = &hostclaim.Holder{WriterBox: box}

+ if claimRemotePredecessor(o, "box-1") {

+ t.Fatal(kind, box)

+ }

+ called := false

+ hook := claimAdmissionHandover("box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error { called = true; return nil }, func(context.Context, hostclaim.Observed, hostclaim.Observed) error { t.Fatal("remote"); return nil })

+ if err := hook(t.Context(), o, o); err != nil || !called {

+ t.Fatal(err)

+ }

+}

+

+func TestPlan12ConsumedFenceSurvivesRosterChange(t *testing.T) {

+ f, o, h := plan12Handover(t, hostclaim.KindRestoring, "fence")

+ rb := "rb_abcdefghijklmnopqrstuvwxyz"

+ var ref hostclaim.FenceRef

+ if err := json.Unmarshal([]byte(o.Claim.Op.Journal["fence.box-2"]), &ref); err != nil {

+ t.Fatal(err)

+ }

+ o, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {

+ c.Op.Exec = "box-2"

+ c.Op.Phase = hostclaim.PhaseReleasing

+ c.Op.Intent["rollback"] = rb

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ fence, err := hostclaim.VerifyMoveTargetFence(t.Context(), f.claims, o, ref, h.FenceAuthority)

+ if err != nil {

+ t.Fatal(err)

+ }

+ n := 0

+ scope := func(loc string) hostclaim.ParkScope {

+ return hostclaim.ParkScope{Location: loc, Incarnation: o.Claim.Holder.Incarnation, EvidenceAt: fence.ObservedAt(), RequestLimit: 2 * time.Minute}

+ }

+ o, err = hostclaim.Update(t.Context(), f.claims, o, hostclaim.ReleaseFencedTarget(hostclaim.FencedTargetRelease{OpID: o.Claim.Op.ID, RollbackID: rb, Target: "box-2", TargetRegion: "uk", SourceRegion: "ie", Fence: fence, Location: scope(o.Claim.Holder.Location), Lane: scope("moves/" + o.Host + "/" + o.Claim.Op.ID), NewID: func() string { n++; return fmt.Sprint("park-", n) }}))

+ if err != nil {

+ t.Fatal(err)

+ }

+ o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Exec = "box-1"; c.Op.Phase = hostclaim.PhaseProbing; return nil })

+ if err != nil {

+ t.Fatal(err)

+ }

+ h.FenceAuthority = plan12FenceAuthority{reject: true}

+ if err := h.Before(t.Context(), o); err != nil {

+ t.Fatal("consumed evidence depended on roster", err)

+ }

+ o.Claim.EvidenceRefs = nil

+ if err := h.Before(t.Context(), o); err == nil {

+ t.Fatal("journal alone accepted as durable loss")

+ }

+}

diff --git a/internal/host/plan12_local_test.go b/internal/host/plan12_local_test.go

new file mode 100644

index 000000000..b02f522c9

--- /dev/null

+++ b/internal/host/plan12_local_test.go

@@ -0,0 +1,30 @@

+package host

+

+import (

+ "context"

+ "errors"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "testing"

+)

+

+func TestPlan12LocalParkUsesOwnerAndRechecksIdentity(t *testing.T) {

+ identityErr := error(nil)

+ calls := 0

+ sentinel := errors.New("owner")

+ local := claimMoveLocalAuthorizer(func(context.Context) error { return identityErr }, func(_ context.Context, o hostclaim.Observed, ob hostclaim.Obligation) error {

+ calls++

+ if o.Host != "app.example" || ob.Park != nil || ob.Kind != "park-local" {

+ t.Fatal("wrong owner input")

+ }

+ return sentinel

+ })

+ c := &hostclaim.Claim{Host: "app.example"}

+ ob := hostclaim.Obligation{Kind: "park-local"}

+ if err := local(t.Context(), c, ob); !errors.Is(err, sentinel) || calls != 1 {

+ t.Fatal(err, calls)

+ }

+ identityErr = errors.New("enrollment changed")

+ if err := local(t.Context(), c, ob); !errors.Is(err, identityErr) || calls != 1 {

+ t.Fatal(err, calls)

+ }

+}

diff --git a/internal/host/plan12_receipt_test.go b/internal/host/plan12_receipt_test.go

new file mode 100644

index 000000000..d34226554

--- /dev/null

+++ b/internal/host/plan12_receipt_test.go

@@ -0,0 +1,67 @@

+package host

+

+import (

+ "crypto/ed25519"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "github.com/carlosframework/platform/internal/hostactors"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostevidence"

+ "github.com/carlosframework/platform/internal/release"

+ "strings"

+ "testing"

+ "time"

+)

+

+func TestPlan12RecoveryRootReceiptAlternative(t *testing.T) {

+ for _, bad := range []string{"", "session", "stamp", "enrollment", "signature", "operation", "region", "digest"} {

+ t.Run(bad, func(t *testing.T) {

+ f, o, h := plan12Handover(t, hostclaim.KindRestoring, "stop")

+ key := ed25519.NewKeyFromSeed([]byte(strings.Repeat("b", 32)))

+ en := strings.Repeat("a", 32)

+ d := hostactors.Directory{Store: f.claims, PublicKey: f.signer.Public().(ed25519.PublicKey), Bucket: "bucket"}

+ roster := hostactors.Roster{V: 1, Revision: 1, DeploymentBucket: "bucket", Boxes: map[string]hostactors.Box{"box-2": {Enrollment: en, BoxID: "box-2", Label: "box-2", Namespace: "platform", Region: "uk", Pool: "default", State: "active", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-2", InstanceID: "i-target"}, EvidenceSHA256: strings.Repeat("d", 64), CensusPublicKey: key.Public().(ed25519.PublicKey)}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}

+ wire, _ := release.SignRootEnvelope(release.RootSigner(f.signer), roster)

+ if _, err := f.claims.Put(t.Context(), hostactors.Key, wire, ""); err != nil {

+ t.Fatal(err)

+ }

+ r := StopReceipt{V: 1, ID: "stop", Kind: "stopped", Host: o.Host, Incarnation: o.Claim.Holder.Incarnation, InstallationStamp: 2, WriterSession: 2, OperationID: "m1", Phase: "releasing", ObligationID: "writer-session-2", Box: "box-2", Region: "uk", Enrollment: en, EvidenceAt: time.Now(), RequestLimitNS: int64(2 * time.Minute), ClaimGen: 1, FenceID: "fence", InventorySHA256: strings.Repeat("e", 64), LocalJournalSHA256: strings.Repeat("f", 64), ProducerDrainDigest: strings.Repeat("a", 64), CgroupIdentities: []string{"root-role"}}

+ switch bad {

+ case "session":

+ r.WriterSession++

+ case "stamp":

+ r.InstallationStamp++

+ case "enrollment":

+ r.Enrollment = strings.Repeat("e", 32)

+ case "operation":

+ r.OperationID = "other"

+ case "region":

+ r.Region = "ie"

+ }

+ body, _ := json.Marshal(r)

+ receiptKey := stopEvidenceKey(o.Host, "m1", "box-2", "stop")

+ sum := sha256.Sum256(body)

+ signer := key

+ if bad == "signature" {

+ signer = f.signer

+ }

+ if err := hostevidence.Publish(t.Context(), f.claims, signer, receiptKey, body); err != nil {

+ t.Fatal(err)

+ }

+ h.Directory = d

+ o.Claim.Op.Journal["target_stop_receipt"] = receiptKey

+ o.Claim.Op.Journal["target_stop_sha256"] = hex.EncodeToString(sum[:])

+ o.Claim.Op.Journal[hostclaim.JournalTargetStamp] = "2"

+ // Remove the unsigned report path: the receipt alone must suffice.

+ delete(o.Claim.Op.Journal, hostclaim.JournalEvidenceTarget)

+ if bad == "digest" {

+ o.Claim.Op.Journal["target_stop_sha256"] = strings.Repeat("0", 64)

+ }

+ err := h.recoveryEvidence(t.Context(), o)

+ if (err == nil) != (bad == "") {

+ t.Fatal(bad, err)

+ }

+ })

+ }

+}

diff --git a/internal/host/plan12_restore_linux_test.go b/internal/host/plan12_restore_linux_test.go

new file mode 100644

index 000000000..52c9e7e67

--- /dev/null

+++ b/internal/host/plan12_restore_linux_test.go

@@ -0,0 +1,260 @@

+//go:build linux

+

+package host

+

+import (

+ "context"

+ "database/sql"

+ "encoding/json"

+ "errors"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/cgroup"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostguard"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/park"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/registry"

+ "golang.org/x/sys/unix"

+ "os"

+ "os/exec"

+ "path/filepath"

+ "strconv"

+ "testing"

+)

+

+func TestPlan12RecoveryRestoreSourceDatabase(t *testing.T) {

+ for _, name := range []string{hostclaim.KindRestoring, hostclaim.KindReverting, "clean-return", "never-restoring", "never-reverting", "never-clean-return", "missing-inventory", "wrong-inventory", "missing-registry", "interrupted"} {

+ t.Run(name, func(t *testing.T) {

+ kind, clean := name, name == "clean-return" || name == "never-clean-return"

+ bad := name == "missing-inventory" || name == "wrong-inventory" || name == "missing-registry"

+ if bad || name == "interrupted" {

+ kind = hostclaim.KindRestoring

+ }

+ if name == "never-restoring" {

+ kind = hostclaim.KindRestoring

+ }

+ if name == "never-reverting" {

+ kind = hostclaim.KindReverting

+ }

+ never := name == "never-restoring" || name == "never-reverting" || name == "never-clean-return"

+ if clean {

+ kind = hostclaim.KindRestoring

+ }

+ f, o, h := plan12Handover(t, hostclaim.KindRestoring, "stop")

+ stamp := o.Claim.Holder.Stamp

+ db := provision.NestedDBPath(f.guard.DataDir, o.Host)

+ dbRel, _ := filepath.Rel(f.guard.DataDir, db)

+ dirRel, _ := filepath.Rel(f.guard.DataDir, filepath.Dir(db))

+ if err := os.MkdirAll(filepath.Dir(db), 0700); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.Mkdir(filepath.Join(f.guard.DataDir, ".parked"), 0770); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(db, []byte("source database"), 0600); err != nil {

+ t.Fatal(err)

+ }

+ if err := hostbox.WriteGenerationMarker(db+".gen", "src"); err != nil {

+ t.Fatal(err)

+ }

+ key := moves.StoppedKey(moves.ReportPrefix("bab", "jam", o.Host, "m1"), "source")

+ report := plan12Stopped(o, "source", "box-1")

+ raw, _ := json.Marshal(report)

+ if _, err := f.claims.Put(t.Context(), key, raw, ""); err != nil {

+ t.Fatal(err)

+ }

+ var err error

+ o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {

+ c.Op.Kind = kind

+ c.Op.Journal[hostclaim.JournalSourceStamp] = strconv.FormatUint(stamp, 10)

+ c.Op.Journal[hostclaim.JournalSrcGen] = "src"

+ c.Op.Journal[hostclaim.JournalCutSession] = "2"

+ c.Op.Journal[hostclaim.JournalEvidenceSource] = key

+ if kind == hostclaim.KindRestoring {

+ c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: 2, Box: "box-2", Generation: "target-gen", Mode: "cold", Region: "uk", Predecessor: "src", PredecessorRegion: "ie"}}

+ if clean {

+ c.Holder.Generation = "src"

+ }

+ if kind == hostclaim.KindReverting {

+ c.Holder.WriterBox = "box-1"

+ c.Holder.Session = 2

+ c.Holder.Generation = "src"

+ c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: 2, Box: "box-1", Generation: "src", Mode: "cold", Region: "ie"}}

+ }

+ if never {

+ c.Holder.WriterBox = ""

+ c.Holder.Session = 0

+ c.Holder.Generation = ""

+ c.Holder.Admissions = nil

+ c.Op.Journal[hostclaim.JournalCutSession] = "0"

+ c.Op.Journal[hostclaim.JournalSrcGen] = ""

+ }

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ if never {

+ if err = os.Remove(db); err != nil {

+ t.Fatal(err)

+ }

+ if err = os.Remove(db + ".gen"); err != nil {

+ t.Fatal(err)

+ }

+ local := park.Local{Guard: f.guard, Claims: f.claims, DataDir: f.guard.DataDir, Box: "box-1"}

+ if err = local.RecordInventory(t.Context(), o.Host, hostguard.Captured{Incarnation: o.Claim.Holder.Incarnation, Stamp: stamp}, "flat", []string{dbRel, dbRel + ".gen"}, func(*hostclaim.Claim) error { return nil }); err != nil {

+ t.Fatal(err)

+ }

+ o = f.read()

+ c := f.controller("boot-1", "controller-1", 4101, "start-1")

+ c.wake.Inputs = plan12ReturnInputs{plan9D2Inputs: plan9D2Inputs{recoveryInputs{f}}, clean: clean}

+ c.wake.Continue = hostclaim.SessionContinues

+ c.wake.Operations = &plan9D2Attempts{}

+ c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }

+ retirement := claimArtifactRetirement{Local: local, Continue: hostclaim.SessionContinues}

+ c.wake.RetireArtifacts, c.wake.RecoverArtifacts = retirement.Retire, retirement.Recover

+ c.assembly.Guard = f.guard

+ c.assembly.bindClaimWakeStop()

+ inv, e := h.Inventory.ReadInstallation(t.Context(), o.Host, o.Claim.Holder.Incarnation, stamp)

+ if e != nil {

+ t.Fatal(e)

+ }

+ var stat unix.Stat_t

+ if e = unix.Lstat(filepath.Dir(db), &stat); e != nil {

+ t.Fatal(e)

+ }

+ inv.Entries = append(inv.Entries, hostbox.StopInventoryEntry{Kind: "path", ArtifactID: "state", ManagedRoot: "data", RelativePath: dirRel, Ownership: "old-owner", Device: uint64(stat.Dev), Inode: stat.Ino, Type: unix.S_IFDIR, Materialization: "materialized"})

+ if e = h.Inventory.WriteStopInventory(t.Context(), inv); e != nil {

+ t.Fatal(e)

+ }

+ roles := &ClaimRoleAssembly{Manager: c.wake.Manager, Journal: f.journal, Inventory: h.Inventory, ManagedRoots: map[string]string{"data": f.guard.DataDir}, Identity: h.Identity, BootID: "boot-1", ControllerID: "controller-1"}

+ roles.prepareGroup = func(name string, _ cgroup.Limit) (*cgroup.Prepared, hostbox.LaunchGroup, error) {

+ return nil, hostbox.LaunchGroup{RelativePath: "carlos/" + name, Device: 1, Inode: 1}, nil

+ }

+ c.wake.Prelaunch, c.wake.PrepareState = roles.Prelaunch, roles.PrepareState

+ if name == "missing-inventory" {

+ replacement, e := hostbox.OpenStopInventoryDB(filepath.Join(t.TempDir(), "empty.db"))

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer replacement.Close()

+ h.Inventory = replacement

+ }

+ if name == "wrong-inventory" {

+ inv.Enrollment = "other"

+ if e = h.Inventory.WriteStopInventory(t.Context(), inv); e != nil {

+ t.Fatal(e)

+ }

+ if name == "missing-registry" {

+ h.Registry = nil

+ }

+ h.Launches = f.journal

+ interrupted := false

+ if name == "interrupted" {

+ h.afterRecoveryRows = func() error { interrupted = true; return hostbox.ErrChanged }

+ if _, e := h.Registry.UpsertTimerRegistration(registry.Timer{Host: o.Host, Name: "uncensused", Kind: "cron", Spec: "* * * * *", ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: o.Claim.Holder.Incarnation, ClaimStamp: stamp, ClaimOwner: "old-owner", ClaimRowID: "timer-row"}}, ""); e != nil {

+ t.Fatal(e)

+ }

+ wireClaimAdmission(c.wake, "box-1", c.wake.AfterAdmission, h, nil)

+ if err := c.wake.stopPredecessor(t.Context(), o); err != nil {

+ t.Fatal(err)

+ }

+ live, cancel := context.WithCancel(t.Context())

+ defer cancel()

+ done := make(chan struct{})

+ go func() { <-live.Done(); close(done) }()

+ c.wake.mu.Lock()

+ c.wake.pending[o.Host] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done}

+ c.wake.mu.Unlock()

+ admitted, err := c.runtime.Admit(t.Context(), registry.Route{Host: o.Host}, o, activator.StartPolicy{Probe: true})

+ if name == "interrupted" {

+ if err == nil || !interrupted {

+ t.Fatal("did not interrupt after registry commit", err)

+ }

+ first := f.read()

+ row, e := h.Registry.Get(o.Host)

+ if e != nil || row.ClaimStamp != first.Claim.Holder.Stamp {

+ t.Fatal("registry did not commit", e)

+ }

+ if _, e = h.Inventory.ReadInstallation(t.Context(), o.Host, first.Claim.Holder.Incarnation, first.Claim.Holder.Stamp); !errors.Is(e, sql.ErrNoRows) {

+ t.Fatal("successor inventory already written", e)

+ }

+ c = f.controller("boot-2", "controller-2", 4102, "start-2")

+ c.wake.Inputs = plan12ReturnInputs{plan9D2Inputs: plan9D2Inputs{recoveryInputs{f}}}

+ c.wake.Continue = hostclaim.SessionContinues

+ c.wake.Operations = &plan9D2Attempts{}

+ c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }

+ c.wake.RetireArtifacts, c.wake.RecoverArtifacts = retirement.Retire, retirement.Recover

+ c.wake.Prelaunch, c.wake.PrepareState = roles.Prelaunch, roles.PrepareState

+ c.assembly.Guard = f.guard

+ c.assembly.bindClaimWakeStop()

+ h.afterRecoveryRows = nil

+ wireClaimAdmission(c.wake, "box-1", c.wake.AfterAdmission, h, nil)

+ if e := c.wake.stopPredecessor(t.Context(), first); e != nil {

+ t.Fatal("retry chose predecessor stop before handover", e)

+ }

+ retryCtx, retryCancel := context.WithCancel(t.Context())

+ defer retryCancel()

+ retryDone := make(chan struct{})

+ go func() { <-retryCtx.Done(); close(retryDone) }()

+ c.wake.pending[o.Host] = &claimWakeAttempt{observed: first, held: recoveryHeld{}, ctx: retryCtx, cancel: retryCancel, done: retryDone}

+ admitted, err = c.runtime.Admit(t.Context(), registry.Route{Host: o.Host}, first, activator.StartPolicy{Probe: true})

+ if err != nil || admitted.Claim.Holder.Session != first.Claim.Holder.Session {

+ t.Fatal("retry did not resume admitted session", err)

+ }

+ if err = c.runtime.ResumedAdmission(t.Context(), first, admitted); err != nil {

+ t.Fatal("activator could not verify resumed admission", err)

+ }

+ if err := c.runtime.ResumedAdmission(t.Context(), first, admitted); !errors.Is(err, hostbox.ErrChanged) {

+ t.Fatal("resume verification was not single use", err)

+ }

+ if bad {

+ if err == nil {

+ t.Fatal("unproved inventory admitted")

+ }

+ return

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ launch, err := c.runtime.Prepare(t.Context(), registry.Route{Host: o.Host}, admitted, activator.StartPolicy{Probe: true})

+ if err != nil {

+ t.Fatal(err)

+ }

+ if never {

+ return

+ }

+ if err := launch.Restore(t.Context()); err != nil {

+ t.Fatal(err)

+ }

+ if got, err := os.ReadFile(db + ".gen"); err != nil || string(got) == "src" {

+ t.Fatal(string(got), err)

+ }

+ if tag := f.read().Claim.Inventory["box-1"].Paths[dbRel]; tag != admitted.Claim.Holder.Stamp {

+ t.Fatal("old ownership", tag)

+ }

+ })

+ }

+}

+

+type plan12ReturnInputs struct {

+ plan9D2Inputs

+ clean bool

+}

+

+func (in plan12ReturnInputs) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, p activator.StartPolicy) (ClaimLaunchInputs, error) {

+ selected, err := in.plan9D2Inputs.Select(ctx, rt, o, p)

+ selected.Admission.KeepStamp = false

+ selected.DBPath = provision.NestedDBPath(in.f.guard.DataDir, o.Host)

+ if in.clean {

+ selected.Admission.Coverage = "clean-return"

+ }

+ return selected, err

+}

diff --git a/internal/hostbox/aggregateleases.go b/internal/hostbox/aggregateleases.go

index 87f135329..ab3c078c1 100644

--- a/internal/hostbox/aggregateleases.go

+++ b/internal/hostbox/aggregateleases.go

@@ -8,9 +8,11 @@ import (

"github.com/carlosframework/platform/internal/park"

)

-// LeaseSet resolves every lock which can write a retirement scope. Production

-// must include the replica location and any moves staging location able to

-// write it; returning an empty set is a refusal, not permission.

+// LeaseSet names locks in the engine's data bucket. A moves/<host>[/<op>]

+// scope needs its own lock; staging writers are bounded by Park.EvidenceAt.

+// An instances/<host>[/<inc>] scope needs its replica lease and scope lock

+// (the same location, deduplicated). Deployment-bucket staging leases must

+// never be expressed as data-bucket names. An empty set refuses.

type LeaseSet func(context.Context, string, string) ([]string, error)

// AggregateLeases adapts a renewing lease implementation to park.Leases.

@@ -22,6 +24,23 @@ type AggregateLeases struct {

}

func (a AggregateLeases) Acquire(ctx context.Context, bucket, scope string) (park.HeldLease, error) {

+ return a.acquire(ctx, bucket, scope, nil)

+}

+func (a AggregateLeases) AcquireHeld(ctx context.Context, bucket, scope string, own park.HeldLease) (park.HeldLease, error) {

+ bound, ok := own.(interface{ LeaseScope() (string, string) })

+ if !ok || own == nil {

+ return nil, ErrUnsupported

+ }

+ b, s := bound.LeaseScope()

+ if b != bucket || s != scope {

+ return nil, ErrChanged

+ }

+ if err := own.Check(ctx); err != nil {

+ return nil, err

+ }

+ return a.acquire(ctx, bucket, scope, own)

+}

+func (a AggregateLeases) acquire(ctx context.Context, bucket, scope string, own park.HeldLease) (park.HeldLease, error) {

if ctx == nil || a.Leases == nil || a.Resolve == nil || bucket == "" || scope == "" {

return nil, ErrUnsupported

}

@@ -29,6 +48,7 @@ func (a AggregateLeases) Acquire(ctx context.Context, bucket, scope string) (par

if err != nil {

return nil, err

}

+ names = append([]string(nil), names...)

sort.Strings(names)

uniq := names[:0]

hasScope := false

@@ -46,9 +66,15 @@ func (a AggregateLeases) Acquire(ctx context.Context, bucket, scope string) (par

if len(uniq) == 0 || !hasScope {

return nil, ErrUnsupported

}

- held := &aggregateHeld{}

+ held := &aggregateHeld{bucket: bucket, scope: scope}

for _, name := range uniq {

- h, acquireErr := a.Leases.Acquire(ctx, bucket, name)

+ var h park.HeldLease

+ var acquireErr error

+ if own != nil && name == scope {

+ h = borrowedLease{own}

+ } else {

+ h, acquireErr = a.Leases.Acquire(ctx, bucket, name)

+ }

if acquireErr != nil {

return nil, errors.Join(acquireErr, held.releaseAndClose(ctx))

}

@@ -61,8 +87,9 @@ func (a AggregateLeases) Acquire(ctx context.Context, bucket, scope string) (par

}

type aggregateHeld struct {

- leases []park.HeldLease

- closed bool

+ bucket, scope string

+ leases []park.HeldLease

+ closed bool

}

func (h *aggregateHeld) Check(ctx context.Context) error {

@@ -106,3 +133,18 @@ func (h *aggregateHeld) releaseAndClose(ctx context.Context) error {

h.closed = true

return errors.Join(errs...)

}

+

+func (h *aggregateHeld) LeaseScope() (string, string) { return h.bucket, h.scope }

+

+type borrowedLease struct{ park.HeldLease }

+

+func (b borrowedLease) Release(ctx context.Context) error { return b.Check(ctx) }

+func (b borrowedLease) Close() {}

+func (b borrowedLease) Context(ctx context.Context) context.Context {

+ if bound, ok := b.HeldLease.(interface {

+ Context(context.Context) context.Context

+ }); ok {

+ return bound.Context(ctx)

+ }

+ return ctx

+}

diff --git a/internal/hostbox/parkleases.go b/internal/hostbox/parkleases.go

new file mode 100644

index 000000000..a2488f687

--- /dev/null

+++ b/internal/hostbox/parkleases.go

@@ -0,0 +1,42 @@

+package hostbox

+

+import (

+ "context"

+ "encoding/hex"

+ "github.com/carlosframework/platform/internal/controlname"

+ "strings"

+)

+

+// ReplicaParkLeaseSet implements the data-bucket locks in claim section 6.2a.

+// Generation parks also use their location's replica lock; the generation

+// selector is in the chore, not an extra lock namespace.

+func ReplicaParkLeaseSet(ctx context.Context, bucket, scope string) ([]string, error) {

+ if ctx == nil || bucket == "" {

+ return nil, ErrUnsupported

+ }

+ if err := ctx.Err(); err != nil {

+ return nil, err

+ }

+ p := strings.Split(scope, "/")

+ if len(p) < 2 || len(p) > 3 || !controlname.ValidHostShape(p[1]) {

+ return nil, ErrUnsupported

+ }

+ switch p[0] {

+ case "instances":

+ if len(p) == 3 {

+ if len(p[2]) != 64 || strings.ToLower(p[2]) != p[2] {

+ return nil, ErrUnsupported

+ }

+ if _, err := hex.DecodeString(p[2]); err != nil {

+ return nil, ErrUnsupported

+ }

+ case "moves":

+ if len(p) == 3 && (p[2] == "" || p[2] == "." || p[2] == ".." || strings.ContainsAny(p[2], "\\\x00:")) {

+ return nil, ErrUnsupported

+ }

+ default:

+ return nil, ErrUnsupported

+ }

+ return []string{scope}, nil

+}

diff --git a/internal/hostbox/plan12_fence_loss_test.go b/internal/hostbox/plan12_fence_loss_test.go

new file mode 100644

index 000000000..93d03a293

--- /dev/null

+++ b/internal/hostbox/plan12_fence_loss_test.go

@@ -0,0 +1,105 @@

+//go:build unix

+

+package hostbox

+

+import (

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "encoding/json"

+ "fmt"

+ "github.com/carlosframework/platform/internal/genselect"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/termfence"

+ "testing"

+ "time"

+)

+

+type plan12FenceLossAuthority struct{}

+

+func (plan12FenceLossAuthority) ValidateFence(context.Context, termfence.Record) (hostclaim.FenceBinding, error) {

+ return hostclaim.FenceBinding{Region: "uk", RosterRevision: 1, RosterETag: "roster"}, nil

+}

+func TestPlan12FencedTargetLossMaximum(t *testing.T) {

+ for _, mode := range []string{"ok", "missing-validator", "wrong-source", "bypass"} {

+ t.Run(mode, func(t *testing.T) {

+ _, s, o := workFixture(t)

+ now := time.Now().UTC()

+ record := termfence.Record{V: 1, Box: "target", Enrollment: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", State: "terminated", ObservedAt: now.Format(time.RFC3339Nano)}

+ raw, _ := json.Marshal(record)

+ sum := sha256.Sum256(raw)

+ ref := hostclaim.FenceRef{Enrollment: record.Enrollment, Key: "control/box-fences/" + record.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])}

+ if _, err := s.Put(t.Context(), ref.Key, raw, ""); err != nil {

+ t.Fatal(err)

+ }

+ var err error

+ o, err = hostclaim.Update(t.Context(), s, o, func(c *hostclaim.Claim) error {

+ c.Op = hostclaim.Op{Kind: hostclaim.KindLanding, ID: "move", Phase: hostclaim.PhaseProbing, Exec: "target", Participants: []string{"box", "target"}, Intent: map[string]string{"source": "box", "target": "target", "to": "uk", "target_enrollment": record.Enrollment}, Journal: map[string]string{hostclaim.JournalSourceStamp: "1", hostclaim.JournalSrcGen: "src"}}

+ c.Holder.Session, c.Holder.Stamp, c.Holder.WriterBox, c.Holder.Generation = 2, 2, "target", "target-gen"

+ c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: 2, Box: "target", Generation: "target-gen", Region: "uk", Mode: "cold"}}

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ fence, err := hostclaim.VerifyMoveTargetFence(t.Context(), s, o, ref, plan12FenceLossAuthority{})

+ if err != nil {

+ t.Fatal(err)

+ }

+ scope := func(loc string) hostclaim.ParkScope {

+ return hostclaim.ParkScope{Location: loc, Incarnation: o.Claim.Holder.Incarnation, EvidenceAt: now, RequestLimit: 2 * time.Minute}

+ }

+ n := 0

+ o, err = hostclaim.Update(t.Context(), s, o, hostclaim.ReleaseFencedTarget(hostclaim.FencedTargetRelease{OpID: "move", RollbackID: "rb_abcdefghijklmnopqrstuvwxyz", Target: "target", TargetRegion: "uk", SourceRegion: "ie", Location: scope(o.Claim.Holder.Location), Lane: scope("moves/" + o.Host + "/move"), NewID: func() string { n++; return fmt.Sprint("park-", n) }, Fence: fence}))

+ if err != nil {

+ t.Fatal(err)

+ }

+ o, err = hostclaim.Update(t.Context(), s, o, hostclaim.ReturnToSource("move", "box", false, hostclaim.ReturnFence{}))

+ if err != nil {

+ t.Fatal(err)

+ }

+ o, err = hostclaim.Update(t.Context(), s, o, hostclaim.SetRestoringPhase("move", hostclaim.PhaseClaimed, hostclaim.PhaseProbing))

+ if err != nil {

+ t.Fatal(err)

+ }

+ in := hostclaim.Admission{ObservedOpID: "move", ObservedPhase: o.Claim.Op.Phase, ObservedSession: 2, ObservedGeneration: o.Claim.Holder.Generation, Box: "box", Mode: "cold", NewGeneration: "return", Region: "ie", PredecessorRegion: "uk", RemoteMaxTXID: 3}

+ w := Writer{Claims: s, Box: "box", Region: "ie", AdmissionEvidence: func(_ context.Context, _ hostclaim.Observed, a hostclaim.Admission, p *genselect.Plan) error {

+ if a.RemoteMaxTXID != 7 || p.MaxTXID != 3 {

+ t.Fatalf("loss maximum %d restore maximum %d", a.RemoteMaxTXID, p.MaxTXID)

+ }

+ return nil

+ }, Loss: &LossAdmission{Sources: func(_ context.Context, _ hostclaim.Observed, l hostclaim.LossRecord, _ hostclaim.Admission, _ *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error) {

+ if len(l.WriteSources) != 1 {

+ t.Fatalf("frozen sources: %+v", l)

+ }

+ source := l.WriteSources[0]

+ if mode == "wrong-source" {

+ source.Generation = "other"

+ }

+ return []hostclaim.ValidatedWriteSource{{Reason: source.Reason, Box: source.Box, Session: source.Session, Generation: source.Generation, Region: source.Region, Baseline: source.Baseline, RemoteMaxTXID: 7}}, nil

+ }}}

+ if mode == "missing-validator" {

+ w.Loss = nil

+ }

+ var got hostclaim.Observed

+ if mode == "bypass" {

+ got, err = hostclaim.Update(t.Context(), s, o, hostclaim.AdmitWriter(in))

+ } else {

+ got, err = w.Admit(t.Context(), o, in, &genselect.Plan{MaxTXID: 3})

+ }

+ if mode != "ok" {

+ if err == nil {

+ t.Fatal("accepted", mode)

+ }

+ return

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ a := got.Claim.Holder.Admissions[len(got.Claim.Holder.Admissions)-1]

+ if a.WritesLost == nil || a.WritesLost.RemoteMaxTXID != 7 || a.WritesLost.Generation != "target-gen" {

+ t.Fatalf("lost target maximum: %+v", a)

+ }

+ })

+ }

+}

diff --git a/internal/hostbox/plan12_held_test.go b/internal/hostbox/plan12_held_test.go

new file mode 100644

index 000000000..9cb005894

--- /dev/null

+++ b/internal/hostbox/plan12_held_test.go

@@ -0,0 +1,116 @@

+package hostbox

+

+import (

+ "context"

+ "errors"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease/leasetest"

+ "github.com/carlosframework/platform/internal/park"

+ "github.com/carlosframework/platform/internal/release"

+ "strings"

+ "testing"

+ "time"

+)

+

+type plan12Renewable struct {

+ lost chan struct{}

+ err error

+ releases int

+}

+

+func (r *plan12Renewable) Err() error { return r.err }

+func (r *plan12Renewable) Lost() <-chan struct{} { return r.lost }

+func (r *plan12Renewable) Release(context.Context) error { r.releases++; return nil }

+func TestPlan12LandingResidueParkUnderItsOwnLease(t *testing.T) {

+ for _, mode := range []string{"ok", "wrong-host", "wrong-location", "wrong-bucket", "lost", "authorization"} {

+ t.Run(mode, func(t *testing.T) {

+ ctx := t.Context()

+ host := "app.example"

+ inc := strings.Repeat("a", 64)

+ loc := "instances/" + host + "/" + inc

+ now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)

+ claims := release.NewDirStore(t.TempDir())

+ _, err := hostclaim.Update(ctx, claims, hostclaim.Observed{Host: host, Absent: true}, func(c *hostclaim.Claim) error {

+ c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "op", Intent: map[string]string{"park_sweep": "v1"}}

+ c.Obligations = []hostclaim.Obligation{{ID: "chore", Op: "op", Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: "ret", Park: &hostclaim.ParkScope{Location: loc, Incarnation: inc, EvidenceAt: now.Add(-time.Hour), RequestLimit: RequestLimit}}}

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ data := leasetest.NewVersionedBucket()

+ data.AddBucket("data")

+ data.Put("data", loc+"/db", []byte("residue"))

+ r := &plan12Renewable{lost: make(chan struct{})}

+ acquires := 0

+ leases := RenewingLeases{Region: "eu-west-1", Open: func(context.Context, string, string, string) (Renewable, error) {

+ acquires++

+ if acquires > 1 {

+ return nil, errors.New("already held")

+ }

+ return r, nil

+ }}

+ bucket, scope := "data", loc

+ switch mode {

+ case "wrong-host":

+ scope = "instances/other.example/" + inc

+ case "wrong-location":

+ scope = "instances/" + host

+ case "wrong-bucket":

+ bucket = "other"

+ }

+ own, err := leases.Acquire(ctx, bucket, scope)

+ if err != nil {

+ t.Fatal(err)

+ }

+ defer own.Close()

+ if mode == "lost" {

+ r.err = errors.New("lost")

+ }

+ authorized := 0

+ engine := park.Engine{Claims: claims, Store: data, Bucket: "data", Region: "ie", Leases: AggregateLeases{Leases: leases, Resolve: ReplicaParkLeaseSet}, Now: func() time.Time { return now }, NewID: func() string { return "next" }, View: func(_ context.Context, o hostclaim.Observed) (park.View, error) {

+ return park.View{ClaimETag: o.ETag, Region: "ie"}, nil

+ }, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error {

+ authorized++

+ if mode == "authorization" {

+ return park.ErrUnsafe

+ }

+ return nil

+ }}

+ err = engine.RunHeld(ctx, park.Work{Host: host, ID: "chore", Op: "op"}, own)

+ if mode == "ok" {

+ if err != nil && !errors.Is(err, park.ErrPending) {

+ t.Fatal(err)

+ }

+ if authorized == 0 {

+ t.Fatal("authorizer bypassed")

+ }

+ versions, e := data.ListVersions(ctx, "data", loc+"/")

+ if e != nil || len(versions) != 0 {

+ t.Fatal("residue not parked", e, versions)

+ }

+ } else if err == nil {

+ t.Fatal("accepted", mode)

+ }

+ if acquires != 1 || r.releases != 0 {

+ t.Fatalf("acquires=%d releases=%d", acquires, r.releases)

+ }

+ if mode != "lost" && own.Check(ctx) != nil {

+ t.Fatal("caller lease lost")

+ }

+ })

+ }

+}

+func TestPlan12LeaseSetSemantics(t *testing.T) {

+ for _, scope := range []string{"moves/app.example/op", "moves/app.example", "instances/app.example", "instances/app.example/" + strings.Repeat("a", 64)} {

+ got, err := ReplicaParkLeaseSet(t.Context(), "data", scope)

+ if err != nil || len(got) != 1 || got[0] != scope {

+ t.Fatalf("%s: %v %v", scope, got, err)

+ }

+ for _, scope := range []string{"control/staging-leases/ie/app.example", "s3://deployment/moves/app.example/op", "instances/app.example/not-an-incarnation", "moves/other/../op"} {

+ if _, err := ReplicaParkLeaseSet(t.Context(), "data", scope); err == nil {

+ t.Fatal(scope)

+ }

+}

diff --git a/internal/hostbox/plan12_loss_test.go b/internal/hostbox/plan12_loss_test.go

new file mode 100644

index 000000000..3e3e9dc2a

--- /dev/null

+++ b/internal/hostbox/plan12_loss_test.go

@@ -0,0 +1,175 @@

+//go:build unix

+

+package hostbox

+

+import (

+ "context"

+ "crypto/ed25519"

+ "encoding/json"

+ "errors"

+ "github.com/carlosframework/platform/internal/genselect"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/release"

+ "testing"

+ "time"

+)

+

+type plan12LossOwner struct{ calls int }

+

+func (p *plan12LossOwner) ValidateLossRecovery(c hostclaim.Claim, l hostclaim.LossRecord, a hostclaim.Admission) error {

+ p.calls++

+ if c.Op.Kind != hostclaim.KindRestoring || l.OpID != c.Op.ID || a.Box != "box" {

+ return hostclaim.ErrLossRecord

+ }

+ return nil

+}

+func TestPlan12LossAdmission(t *testing.T) {

+ plan12LossAdmissionCases(t, []string{"ok", "source-cut", "covered-source", "no-source", "clean-return", "sealed", "missing-owner", "missing-source", "wrong-source", "wrong-op", "wrong-signature", "stale", "restore-mismatch", "bypass"})

+}

+func TestLossBeforeTargetAdmissionRecovers(t *testing.T) {

+ plan12LossAdmissionCases(t, []string{"source-cut", "covered-source", "no-source"})

+}

+func plan12LossAdmissionCases(t *testing.T, modes []string) {

+ for _, mode := range modes {

+ t.Run(mode, func(t *testing.T) {

+ _, store, o := workFixture(t)

+ var err error

+ o, err = hostclaim.Update(t.Context(), store, o, func(c *hostclaim.Claim) error {

+ c.Op = hostclaim.Op{Kind: hostclaim.KindRestoring, ID: "op", Phase: hostclaim.PhaseProbing, Exec: "box", Participants: []string{"box"}, Journal: map[string]string{"loss.loss_1": "{}"}}

+ c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 2, "target-gen", "target"

+ if mode == "source-cut" || mode == "covered-source" || mode == "no-source" {

+ c.Holder.WriterBox = "box"

+ }

+ if mode == "no-source" {

+ c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 0, "", ""

+ }

+ c.LossRefs = []string{"loss_1"}

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ pub, priv, err := ed25519.GenerateKey(nil)

+ if err != nil {

+ t.Fatal(err)

+ }

+ now := time.Now().UTC()

+ r := hostclaim.LossRecord{V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: o.Host, HolderIncarnation: o.Claim.Holder.Incarnation, HolderStamp: o.Claim.Holder.Stamp, HolderSession: 2, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: "op", OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "target", Enrollment: "enroll", QuarantineRevision: 1,

+ CloudObservation: hostclaim.LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now,

+ ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"move-target-lost"}`)}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, WriteSources: []hostclaim.WriteSource{{Reason: "target", Box: "target", Session: 2, Generation: "target-gen", Region: "uk", Baseline: "target-snapshot"}}}

+ if mode == "source-cut" {

+ r.WriteSources[0].Reason, r.WriteSources[0].Box = "source-cut", "box"

+ }

+ if mode == "no-source" || mode == "covered-source" {

+ r.WriteSources = nil

+ r.HolderSession = 0

+ }

+ if mode == "wrong-op" {

+ r.OpID = "different"

+ }

+ created, err := hostclaim.CreateLossRecord(t.Context(), store, release.RootSigner(priv), r)

+ if err != nil {

+ t.Fatal(err)

+ }

+ r.State, r.ResultClaimGen = "committed", o.Claim.Gen

+ if _, err = hostclaim.ReplaceLossRecord(t.Context(), store, release.RootSigner(priv), created, r); err != nil {

+ t.Fatal(err)

+ }

+ owner := &plan12LossOwner{}

+ w := Writer{Claims: store, Box: "box", Region: "ie", AdmissionEvidence: func(_ context.Context, obs hostclaim.Observed, admission hostclaim.Admission, baseline *genselect.Plan) error {

+ if baseline == nil || baseline.MaxTXID != 3 {

+ t.Fatal("restore baseline changed", baseline)

+ }

+ want := uint64(7)

+ if mode == "no-source" || mode == "covered-source" || mode == "wrong-source" || hostclaim.OperationLossConsumed(obs.Claim, "loss_1") {

+ want = 3

+ }

+ if admission.RemoteMaxTXID != want {

+ t.Fatalf("consumer maximum=%d want=%d", admission.RemoteMaxTXID, want)

+ }

+ return nil

+ }, Loss: &LossAdmission{PublicKey: pub, Owner: owner}}

+ w.Loss.Sources = func(_ context.Context, _ hostclaim.Observed, r hostclaim.LossRecord, _ hostclaim.Admission, _ *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error) {

+ if mode == "no-source" || mode == "covered-source" {

+ return nil, nil

+ }

+ f := r.WriteSources[0]

+ v := hostclaim.ValidatedWriteSource{Reason: f.Reason, Box: f.Box, Session: f.Session, Generation: f.Generation, Region: f.Region, Baseline: f.Baseline, RemoteMaxTXID: 7}

+ if mode == "wrong-source" {

+ v.Session++

+ }

+ return []hostclaim.ValidatedWriteSource{v}, nil

+ }

+ if mode == "missing-owner" {

+ w.Loss.Owner = nil

+ }

+ if mode == "missing-source" {

+ w.Loss.Sources = nil

+ }

+ if mode == "wrong-signature" {

+ w.Loss.PublicKey, _, _ = ed25519.GenerateKey(nil)

+ }

+ in := hostclaim.Admission{ObservedOpID: "op", ObservedPhase: hostclaim.PhaseProbing, ObservedSession: 2, ObservedGeneration: "target-gen", Box: "box", Mode: "cold", NewGeneration: "returned", Region: "ie", PredecessorRegion: "uk", RemoteMaxTXID: 3}

+ if mode == "no-source" {

+ in.ObservedSession, in.ObservedGeneration = 0, ""

+ }

+ if mode == "clean-return" || mode == "sealed" {

+ in.Coverage = mode

+ }

+ if mode == "stale" {

+ _, err = hostclaim.Update(t.Context(), store, o, func(c *hostclaim.Claim) error { return nil })

+ if err != nil {

+ t.Fatal(err)

+ }

+ if mode == "restore-mismatch" {

+ in.RemoteMaxTXID = 4

+ }

+ var admitted hostclaim.Observed

+ if mode == "bypass" {

+ admitted, err = hostclaim.Update(t.Context(), store, o, hostclaim.AdmitWriter(in))

+ } else {

+ admitted, err = w.Admit(t.Context(), o, in, &genselect.Plan{MaxTXID: 3})

+ }

+ if mode != "ok" && mode != "source-cut" && mode != "covered-source" && mode != "no-source" {

+ if err == nil {

+ t.Fatal("accepted", mode)

+ }

+ if mode == "stale" && !errors.Is(err, hostclaim.ErrConflict) {

+ t.Fatal(err)

+ }

+ return

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ a := admitted.Claim.Holder.Admissions[0]

+ if a.LossID != "loss_1" || a.Coverage != "" || (mode != "no-source" && mode != "covered-source" && (a.WritesLost == nil || a.WritesLost.Session != 2 || a.WritesLost.RemoteMaxTXID != 7)) || owner.calls != 1 {

+ t.Fatalf("%+v owner=%d", a, owner.calls)

+ }

+ if mode == "covered-source" && (a.WritesLost == nil || a.WritesLost.RemoteMaxTXID != 3) {

+ t.Fatalf("restore maximum lost: %+v", a)

+ }

+ if mode == "no-source" && a.WritesLost != nil {

+ t.Fatalf("invented predecessor: %+v", a)

+ }

+ // Restart loses the writer object, but not the claim's consumption marker.

+ restarted := Writer{Claims: store, Box: "box", Region: "ie", AdmissionEvidence: w.AdmissionEvidence}

+ fresh, err := hostclaim.Read(t.Context(), store, o.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ in.ObservedSession, in.ObservedGeneration, in.NewGeneration = fresh.Claim.Holder.Session, fresh.Claim.Holder.Generation, "second-cold"

+ in.PredecessorRegion = "ie"

+ in.LossID = "loss_1"

+ again, err := restarted.Admit(t.Context(), fresh, in, &genselect.Plan{MaxTXID: 3})

+ if err != nil {

+ t.Fatal("second cold admission after restart", err)

+ }

+ if again.Claim.Holder.Admissions[0].LossID != "loss_1" || again.Claim.Holder.Admissions[len(again.Claim.Holder.Admissions)-1].LossID != "" {

+ t.Fatal("loss consumed again")

+ }

+

+ })

+ }

+}

diff --git a/internal/hostbox/renewing.go b/internal/hostbox/renewing.go

index fca8e9825..4c5ddbf0d 100644

--- a/internal/hostbox/renewing.go

+++ b/internal/hostbox/renewing.go

@@ -37,7 +37,7 @@ func (l RenewingLeases) Acquire(ctx context.Context, bucket, scope string) (park

return nil, ErrUnsupported

}

cctx, stop := context.WithCancel(ctx)

- held := &renewingHeld{h: h, ctx: cctx, cancel: stop, done: make(chan struct{})}

+ held := &renewingHeld{bucket: bucket, scope: scope, h: h, ctx: cctx, cancel: stop, done: make(chan struct{})}

go func() {

defer close(held.done)

select {

@@ -54,12 +54,13 @@ func (l RenewingLeases) Acquire(ctx context.Context, bucket, scope string) (park

}

type renewingHeld struct {

- h Renewable

- ctx context.Context

- cancel context.CancelFunc

- done chan struct{}

- once sync.Once

- releaseErr error

+ bucket, scope string

+ h Renewable

+ ctx context.Context

+ cancel context.CancelFunc

+ done chan struct{}

+ once sync.Once

+ releaseErr error

}

func (h *renewingHeld) Check(ctx context.Context) error {

@@ -108,3 +109,5 @@ func LeasedCheck(ctx context.Context, health func() error, authority func(contex

}

return errors.Join(ctx.Err(), health(), authority(ctx))

}

+

+func (h *renewingHeld) LeaseScope() (string, string) { return h.bucket, h.scope }

diff --git a/internal/hostbox/writer.go b/internal/hostbox/writer.go

index 1a388184a..7a1409c47 100644

--- a/internal/hostbox/writer.go

+++ b/internal/hostbox/writer.go

@@ -12,6 +12,7 @@ import (

// Writer supplies transition adapters, not stop, certificate or marker evidence.

type Writer struct {

+ Loss *LossAdmission

Claims release.Store

Data genselect.Reader

Bucket, Region, Box string

@@ -34,6 +35,22 @@ func (w Writer) Admit(ctx context.Context, observed hostclaim.Observed, in hostc

if w.Claims == nil || w.AdmissionEvidence == nil || w.Box == "" || w.Region == "" || observed.Absent || observed.Claim == nil || observed.Claim.Holder == nil || in.Box != w.Box || in.Region != w.Region {

return hostclaim.Observed{}, ErrUnsupported

}

+ if record, pending := hostclaim.PendingMoveFenceLoss(observed.Claim); pending {

+ return w.admitFencedLoss(ctx, observed, in, baseline, record)

+ }

+ lossID, err := hostclaim.OperationLossID(observed.Claim)

+ if err != nil {

+ return hostclaim.Observed{}, err

+ }

+ if lossID != "" && !hostclaim.OperationLossConsumed(observed.Claim, lossID) {

+ return w.admitLoss(ctx, observed, in, baseline, lossID)

+ }

+ if in.LossID != "" {

+ if lossID == "" || in.LossID != lossID || !hostclaim.OperationLossConsumed(observed.Claim, lossID) {

+ return hostclaim.Observed{}, hostclaim.ErrLossRecord

+ }

+ in.LossID = ""

+ }

if baseline != nil && baseline.MaxTXID != in.RemoteMaxTXID {

return hostclaim.Observed{}, ErrChanged

}

diff --git a/internal/hostbox/writer_loss.go b/internal/hostbox/writer_loss.go

new file mode 100644

index 000000000..5e6317fc1

--- /dev/null

+++ b/internal/hostbox/writer_loss.go

@@ -0,0 +1,80 @@

+package hostbox

+

+import (

+ "context"

+ "crypto/ed25519"

+ "github.com/carlosframework/platform/internal/genselect"

+ "github.com/carlosframework/platform/internal/hostclaim"

+)

+

+// LossAdmissionSources validates every frozen source against its remote baseline.

+// The writer verifies a signed declaration or the claim's consumed automatic fence.

+type LossAdmissionSources func(context.Context, hostclaim.Observed, hostclaim.LossRecord, hostclaim.Admission, *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error)

+

+type LossAdmission struct {

+ PublicKey ed25519.PublicKey

+ Owner hostclaim.LossRecoveryOwner

+ Sources LossAdmissionSources

+}

+

+func (w Writer) admitLoss(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, baseline *genselect.Plan, id string) (hostclaim.Observed, error) {

+ if w.Loss == nil || w.Loss.Owner == nil || w.Loss.Sources == nil || len(w.Loss.PublicKey) != ed25519.PublicKeySize {

+ return hostclaim.Observed{}, hostclaim.ErrLossOwnerRequired

+ }

+ if o.Claim.Op.Kind != hostclaim.KindRestoring || o.Claim.Op.Phase != hostclaim.PhaseProbing || in.Coverage != "" || in.KeepStamp || (in.LossID != "" && in.LossID != id) {

+ return hostclaim.Observed{}, hostclaim.ErrLossRecord

+ }

+ loss, err := hostclaim.ReadLossRecord(ctx, w.Claims, w.Loss.PublicKey, id)

+ if err != nil {

+ return hostclaim.Observed{}, err

+ }

+ r := loss.Record

+ if r.State != "committed" || r.Host != o.Host || r.OpID != o.Claim.Op.ID || r.HolderIncarnation != o.Claim.Holder.Incarnation {

+ return hostclaim.Observed{}, hostclaim.ErrLossRecord

+ }

+ sources, err := w.Loss.Sources(ctx, o, r, in, baseline)

+ if err != nil {

+ return hostclaim.Observed{}, err

+ }

+ p := hostclaim.AdmissionRecord{Box: o.Claim.Holder.WriterBox, Session: o.Claim.Holder.Session, Generation: o.Claim.Holder.Generation}

+ if a := o.Claim.Holder.Admissions; len(a) > 0 {

+ p = a[len(a)-1]

+ }

+ // Select validates the restore point; loss sources describe independent baselines.

+ if baseline == nil || baseline.MaxTXID != in.RemoteMaxTXID {

+ return hostclaim.Observed{}, ErrChanged

+ }

+ in.AdditionalWritesLost = nil

+ for _, s := range sources {

+ if s.Box == p.Box && s.Session == p.Session && s.Generation == p.Generation {

+ in.RemoteMaxTXID = s.RemoteMaxTXID

+ } else {

+ in.AdditionalWritesLost = append(in.AdditionalWritesLost, hostclaim.WritesLost{Box: s.Box, Session: s.Session, Generation: s.Generation, RemoteMaxTXID: s.RemoteMaxTXID})

+ }

+ if err := w.AdmissionEvidence(ctx, o, in, baseline); err != nil {

+ return hostclaim.Observed{}, err

+ }

+ return hostclaim.AdmitWriterWithLoss(ctx, w.Claims, w.Loss.PublicKey, o, in, id, sources, w.Loss.Owner)

+}

+

+func (w Writer) admitFencedLoss(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, baseline *genselect.Plan, record hostclaim.LossRecord) (hostclaim.Observed, error) {

+ if w.Loss == nil || w.Loss.Sources == nil {

+ return hostclaim.Observed{}, hostclaim.ErrLossOwnerRequired

+ }

+ if baseline == nil || baseline.MaxTXID != in.RemoteMaxTXID {

+ return hostclaim.Observed{}, ErrChanged

+ }

+ sources, err := w.Loss.Sources(ctx, o, record, in, baseline)

+ if err != nil {

+ return hostclaim.Observed{}, err

+ }

+ if len(sources) != 1 {

+ return hostclaim.Observed{}, hostclaim.ErrFence

+ }

+ in.RemoteMaxTXID = sources[0].RemoteMaxTXID

+ if err = w.AdmissionEvidence(ctx, o, in, baseline); err != nil {

+ return hostclaim.Observed{}, err

+ }

+ return hostclaim.AdmitWriterWithFencedTargetLoss(ctx, w.Claims, o, in, sources)

+}

diff --git a/internal/hostclaim/claim.go b/internal/hostclaim/claim.go

index db1daf3bb..86fc8b95e 100644

--- a/internal/hostclaim/claim.go

+++ b/internal/hostclaim/claim.go

@@ -155,6 +155,7 @@ type CancelledCreate struct {

}

type Claim struct {

+ RetiredLocations map[string]string `json:"retired_locations,omitempty"`

EvidenceRefs []AutomaticLoss `json:"evidence_refs,omitempty"`

V int `json:"v"`

Host string `json:"host"`

@@ -297,6 +298,11 @@ func (c *Claim) Validate() error {

if err := validateAutomaticLosses(c, seenLossRefs); err != nil {

return err

}

+ for inc, location := range c.RetiredLocations {

+ if !isHex64(inc) || !stringPresent(c.RetiredIncarnations, inc) || (location != "instances/"+c.Host && location != "instances/"+c.Host+"/"+inc) {

+ return ErrMalformed

+ }

if len(c.RetiredIncarnations) > retiredCap {

return fmt.Errorf("%w: retired_incarnations over cap", ErrMalformed)

}

diff --git a/internal/hostclaim/create.go b/internal/hostclaim/create.go

index 8079e8629..7ad706f65 100644

--- a/internal/hostclaim/create.go

+++ b/internal/hostclaim/create.go

@@ -4,6 +4,7 @@ import (

"encoding/json"

"errors"

"fmt"

+ "os"

"time"

)

@@ -42,9 +43,20 @@ func recordMatches(nh NewHolder, body []byte) bool {

}

func retire(c *Claim, inc string) {

+ if os.Getenv("CARLOS_HOST_CLAIM_RETIRED_LOCATIONS") == "1" && c.Holder != nil && c.Holder.Incarnation == inc && isHex64(inc) {

+ if c.RetiredLocations == nil {

+ c.RetiredLocations = map[string]string{}

+ }

+ c.RetiredLocations[inc] = c.Holder.Location

+ }

c.RetiredIncarnations = append([]string{inc}, c.RetiredIncarnations...)

if len(c.RetiredIncarnations) > retiredCap {

c.RetiredIncarnations = c.RetiredIncarnations[:retiredCap]

+ for retired := range c.RetiredLocations {

+ if !stringPresent(c.RetiredIncarnations, retired) {

+ delete(c.RetiredLocations, retired)

+ }

}

diff --git a/internal/hostclaim/generation.go b/internal/hostclaim/generation.go

index 0d69b1442..8c7e71022 100644

--- a/internal/hostclaim/generation.go

+++ b/internal/hostclaim/generation.go

@@ -3,6 +3,7 @@ package hostclaim

import (

"fmt"

"reflect"

+ "strconv"

)

const RootGeneration = "@root"

@@ -23,6 +24,7 @@ type Admission struct {

AdditionalWritesLost []WritesLost

LossID string

lossVerified bool

+ fenceVerified bool

}

// AdmitWriter is §6.4a's writer admission CAS within the current operation.

@@ -48,6 +50,20 @@ func AdmitWriter(in Admission) func(*Claim) error {

return err

}

+ id, err := OperationLossID(c)

+ if err != nil {

+ return err

+ }

+ if id != "" && (in.Coverage != "" || (!OperationLossConsumed(c, id) && (!in.lossVerified || in.LossID != id))) {

+ return ErrLossRecord

+ }

+

+ if c.Op.Kind == KindRestoring && c.Op.Journal["fence."+c.Op.Intent["target"]] != "" && in.Coverage != "" {

+ return ErrFence

+ }

+ if _, pending := PendingMoveFenceLoss(c); pending && !in.fenceVerified {

+ return ErrFence

+ }

h := c.Holder

if in.LossID != "" && !in.lossVerified {

return fmt.Errorf("%w: declaration-bound admission requires verified durable declaration", ErrWrongState)

@@ -151,6 +167,9 @@ func AdmitWriter(in Admission) func(*Claim) error {

h.Admissions = h.Admissions[1:]

}

h.Session, h.Generation, h.WriterBox = entry.Session, entry.Generation, in.Box

+ if in.lossVerified && id != "" {

+ setJournal(c, "loss_admitted."+id, strconv.FormatUint(entry.Session, 10))

+ }

return nil

}

@@ -171,6 +190,7 @@ func PublishGeneration(observedOpID, observedPhase string, observedSession uint6

return ErrWrongState

}

+

h := c.Holder

admitted := false

for _, record := range h.Admissions {

diff --git a/internal/hostclaim/lossrecord.go b/internal/hostclaim/lossrecord.go

index 3e496aa6b..6825222af 100644

--- a/internal/hostclaim/lossrecord.go

+++ b/internal/hostclaim/lossrecord.go

@@ -95,6 +95,8 @@ type ArchivedAdmissionLoss struct {

}

type LossRecord struct {

+ AlertsPending bool `json:"alerts_pending,omitempty"`

+ Automatic *AutomaticLoss `json:"automatic,omitempty"`

V int `json:"v"`

ID string `json:"id"`

State string `json:"state"` // prepared | committed | conflicted

@@ -147,6 +149,9 @@ func safeLossID(id string) bool {

}

func (r LossRecord) Validate() error {

+ if r.Automatic != nil {

+ return r.validateAutomatic()

+ }

if r.V != lossRecordVersion || !safeLossID(r.ID) || (r.State != "prepared" && r.State != "committed" && r.State != "conflicted") ||

r.Actor == "" || r.AcceptedAt.IsZero() || r.Host == "" || !isHex64(r.HolderIncarnation) || r.HolderStamp == 0 || r.ExpectedClaimGen == 0 ||

r.ExpectedClaimETag == "" || !knownKinds[r.OpKind] || r.Box == "" || r.Enrollment == "" ||

@@ -393,3 +398,22 @@ func ValidateLossRecordReplacement(before ObservedLossRecord, next LossRecord) e

}

return nil

}

+

+// Automatic records retain the exact claim evidence, without fabricating operator authority.

+func (r LossRecord) validateAutomatic() error {

+ l := r.Automatic

+ if l == nil || l.Reported || !r.AlertsPending || r.V != lossRecordVersion || r.Actor != "automatic" || r.ID != l.ID || r.Host == "" || r.HolderIncarnation != l.Incarnation || r.HolderStamp != l.Stamp || r.HolderSession != l.Session || r.OpID != l.Op || r.OpKind != l.Kind || r.OpPhase != l.Phase || r.Box != l.Box || r.Enrollment != l.Enrollment || r.QuarantineRevision != 0 || r.ExpectedClaimGen == 0 || r.ExpectedClaimETag == "" || (r.State != "prepared" && r.State != "committed") || (r.State == "prepared" && r.ResultClaimGen != 0) || (r.State == "committed" && r.ResultClaimGen == 0) {

+ return ErrLossRecord

+ }

+ at, err := time.Parse(time.RFC3339Nano, l.ObservedAt)

+ if err != nil || !r.AcceptedAt.Equal(at) || !r.FencedAt.Equal(at) || r.OperatorAttestation.MaintainComputeFence || r.OperatorAttestation.ColdQuarantineStartOnly || len(r.OperatorAttestation.DestroyedStorage) != 0 || r.CloudObservation != (LossCloudObservation{}) || r.Timers.Knowledge != l.Timers || r.Timers.Source != nil || len(r.Timers.Entries) != 0 || len(r.WriteSources) != 0 {

+ return ErrLossRecord

+ }

+ if _, err = Key(r.Host); err != nil {

+ return ErrLossRecord

+ }

+ if err = validateAutomaticLosses(&Claim{EvidenceRefs: []AutomaticLoss{*l}}, map[string]bool{l.ID: true}); err != nil {

+ return ErrLossRecord

+ }

+ return nil

+}

diff --git a/internal/hostclaim/lost.go b/internal/hostclaim/lost.go

index 0a7f3497b..ebe3eb993 100644

--- a/internal/hostclaim/lost.go

+++ b/internal/hostclaim/lost.go

@@ -152,14 +152,17 @@ func AdmitWriterWithLoss(ctx context.Context, s release.Store, pub ed25519.Publi

want[key] = source

}

targetKey := fmt.Sprintf("%s\x00%s\x00%d", predecessor.Box, predecessor.Generation, predecessor.Session)

- if want[targetKey].Reason != "target" {

+ if len(want) != 0 && want[targetKey].Reason != "target" && want[targetKey].Reason != "source-cut" {

return Observed{}, fmt.Errorf("%w: predecessor is not the declared lost target", ErrLossRecord)

}

primary := WritesLost{Box: predecessor.Box, Generation: predecessor.Generation, Session: predecessor.Session, RemoteMaxTXID: in.RemoteMaxTXID}

- if !validWriteLosses(&primary, in.AdditionalWritesLost) {

+ if len(want) != 0 && !validWriteLosses(&primary, in.AdditionalWritesLost) {

return Observed{}, ErrLossRecord

}

- declared := map[string]WritesLost{targetKey: primary}

+ declared := map[string]WritesLost{}

+ if len(want) != 0 {

+ declared[targetKey] = primary

+ }

for _, additional := range in.AdditionalWritesLost {

key := fmt.Sprintf("%s\x00%s\x00%d", additional.Box, additional.Generation, additional.Session)

if _, ok := want[key]; !ok {

diff --git a/internal/hostclaim/move.go b/internal/hostclaim/move.go

index 66718d41c..81899281e 100644

--- a/internal/hostclaim/move.go

+++ b/internal/hostclaim/move.go

@@ -416,6 +416,8 @@ func Revert(opID, sourceBox, targetRegion string, stagingScope *ParkScope, newID

if stagingScope.Location != "moves/"+c.Host+"/"+opID || stagingScope.Incarnation != c.Holder.Incarnation {

return fmt.Errorf("%w: the revert's park must be this move's own lane and this holder's incarnation", ErrMalformed)

}

+ setJournal(c, JournalSourceStamp, strconv.FormatUint(c.Holder.Stamp, 10))

+ setJournal(c, JournalSrcGen, c.Holder.Generation)

c.Op.Kind, c.Op.Exec = KindReverting, sourceBox

if c.Op.Journal[JournalRevPhase] == "" {

setJournal(c, JournalRevPhase, RevPhaseEvidence)

@@ -890,6 +892,9 @@ func RollBackFlipped(opID, rbID, targetBox string, sourceStamp uint64, srcGen st

// A replay that finds the release written with the same evidence (released

// or any later phase) is a no-op and queues nothing, even after a chore has

// completed. A refused release leaves the claim as found.

+const JournalTargetSession = "target_session"

+const JournalTargetGeneration = "target_generation"

+

func TargetReleased(opID, targetBox, targetRegion, evidenceKey string, scope ParkScope, newID func() string) func(*Claim) error {

return func(c *Claim) error {

if err := RequireOp(opID)(c); err != nil {

@@ -921,6 +926,13 @@ func TargetReleased(opID, targetBox, targetRegion, evidenceKey string, scope Par

parkBox = targetBox

}

return inOneWrite(c,

+ func(c *Claim) error {

+ if c.Holder != nil && c.Holder.WriterBox == targetBox {

+ setJournal(c, JournalTargetSession, strconv.FormatUint(c.Holder.Session, 10))

+ setJournal(c, JournalTargetGeneration, c.Holder.Generation)

+ }

+ return nil

+ },

ExecToNone(opID, targetBox, evidenceKey, JournalEvidenceTarget),

func(c *Claim) error { c.Op.Phase = PhaseReleased; return nil },

QueueLocationPark(opID, targetRegion, parkBox, scope, newID),

@@ -933,7 +945,7 @@ func TargetReleased(opID, targetBox, targetRegion, evidenceKey string, scope Par

// verified before Update (VerifyReturnFence, J5): the op, the box and the

// exact fence.<box> journal entry it verified. The zero value vouches for

// nothing.

-type ReturnFence struct{ op, box, ref string }

+type ReturnFence struct{ op, box, ref, observedAt string }

// VerifyReturnFence re-reads and verifies the fence the op journals for box

// (VerifyOperationFence) and returns what ReturnToSource checks inside its

@@ -959,6 +971,9 @@ func targetGaveUp(c *Claim, opID string, fence ReturnFence) bool {

return true

}

if !stringPresent(c.Op.Participants, target) {

+ if _, consumed := ConsumedMoveTargetFence(c); consumed {

+ return true

+ }

for _, id := range c.LossRefs {

if _, ok := c.Op.Journal["loss."+id]; ok {

return true

diff --git a/internal/hostclaim/move_fence_admission.go b/internal/hostclaim/move_fence_admission.go

new file mode 100644

index 000000000..d31322abb

--- /dev/null

+++ b/internal/hostclaim/move_fence_admission.go

@@ -0,0 +1,50 @@

+package hostclaim

+

+import (

+ "context"

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// PendingMoveFenceLoss selects the frozen target writer until its first recovery admission.

+func PendingMoveFenceLoss(c *Claim) (LossRecord, bool) {

+ ref, ok := ConsumedMoveTargetFence(c)

+ if !ok {

+ return LossRecord{}, false

+ }

+ for _, l := range c.EvidenceRefs {

+ if l.Op != c.Op.ID || l.Box != c.Op.Intent["target"] || l.Fence != ref || l.Session == 0 || c.Op.Journal["fence_admitted."+l.ID] != "" {

+ continue

+ }

+ region := c.Op.Intent["to"]

+ return LossRecord{ID: l.ID, Host: c.Host, OpID: l.Op, OpKind: l.Kind, OpPhase: l.Phase, HolderIncarnation: l.Incarnation, HolderStamp: l.Stamp, HolderSession: l.Session, Box: l.Box, Enrollment: l.Enrollment, Automatic: &l, WriteSources: []WriteSource{{Reason: "target", Box: l.Box, Session: l.Session, Generation: l.Generation, Region: region, Baseline: "replica:" + region + "/" + l.Generation}}}, true

+ }

+ return LossRecord{}, false

+}

+

+// AdmitWriterWithFencedTargetLoss binds the frozen target maximum independently of restore.

+func AdmitWriterWithFencedTargetLoss(ctx context.Context, s release.Store, o Observed, in Admission, sources []ValidatedWriteSource) (Observed, error) {

+ record, ok := PendingMoveFenceLoss(o.Claim)

+ if !ok || o.Claim.Op.Phase != PhaseProbing || len(sources) != 1 || in.Coverage != "" || in.Mode != "cold" || in.KeepStamp || in.PredecessorCertified || in.LossID != "" || len(in.AdditionalWritesLost) != 0 {

+ return Observed{}, ErrFence

+ }

+ want, got := record.WriteSources[0], sources[0]

+ if got.Reason != want.Reason || got.Box != want.Box || got.Session != want.Session || got.Generation != want.Generation || got.Region != want.Region || got.Baseline != want.Baseline || in.RemoteMaxTXID != got.RemoteMaxTXID {

+ return Observed{}, ErrFence

+ }

+ h := o.Claim.Holder

+ p := AdmissionRecord{Box: h.WriterBox, Session: h.Session, Generation: h.Generation}

+ if len(h.Admissions) > 0 {

+ p = h.Admissions[len(h.Admissions)-1]

+ }

+ if p.Box != want.Box || p.Session != want.Session || p.Generation != want.Generation {

+ return Observed{}, ErrFence

+ }

+ in.fenceVerified = true

+ return Update(ctx, s, o, func(c *Claim) error {

+ if err := AdmitWriter(in)(c); err != nil {

+ return err

+ }

+ setJournal(c, "fence_admitted."+record.ID, "true")

+ return nil

+ })

+}

diff --git a/internal/hostclaim/move_loss.go b/internal/hostclaim/move_loss.go

index 221941a6a..7a59749d7 100644

--- a/internal/hostclaim/move_loss.go

+++ b/internal/hostclaim/move_loss.go

@@ -68,88 +68,7 @@ func ReleaseLostTarget(r LostTargetRelease) func(*Claim) error {

stringPresent(c.Op.Participants, r.Target) || c.Op.Exec == r.Target {

return fmt.Errorf("%w: %q is not this move's declared-lost target", ErrWrongState, r.Target)

}

- location := r.Location

- if err := location.Validate(c.Host); err != nil {

- return err

- }

- lane, err := parkQueueGuard(c, r.OpID, r.TargetRegion, r.Lane, r.NewID)

- if err != nil {

- return err

- }

- // The location park's scope is checked against holder.location by

- // QueueLocationPark; the lane is this move's own.

- if err := requireHolderLocation(c, r.Location); err != nil {

- return err

- }

- if lane.Location != "moves/"+c.Host+"/"+r.OpID {

- return fmt.Errorf("%w: a lost target's lane park is the move's own lane", ErrMalformed)

- }

- var returnLane *ParkScope

- if r.ReturnLane != nil {

- s, err := parkQueueGuard(c, r.OpID, r.SourceRegion, *r.ReturnLane, r.NewID)

- if err != nil {

- return err

- }

- returnLane = &s

- if s.Location != "moves/"+c.Host+"/"+r.RollbackID {

- return fmt.Errorf("%w: the return lane is this rollback's own lane", ErrMalformed)

- }

- steps := []func(*Claim) error{}

- switch {

- case c.Op.Kind == KindLanding && c.Op.Phase != PhaseCommitted:

- steps = append(steps, RollBack(r.OpID, r.RollbackID))

- case c.Op.Kind == KindRestoring && c.Op.Intent["rollback"] == r.RollbackID:

- default:

- return fmt.Errorf("%w: op is %s/%q, a lost target is released from a landing before its commit or from its own rollback", ErrWrongState, c.Op.Kind, c.Op.Phase)

- }

- steps = append(steps, func(c *Claim) error {

- switch c.Op.Phase {

- case PhaseReleasing:

- if c.Op.Exec != "" && c.Op.Exec != "none" {

- return fmt.Errorf("%w: exec is %q", ErrWrongState, c.Op.Exec)

- }

- c.Op.Phase = PhaseReleased

- parkBox := ""

- if stamp, err := strconv.ParseUint(c.Op.Journal[JournalTargetStamp], 10, 64); err == nil && InventoryHasStamp(c, r.Target, stamp) {

- parkBox = r.Target

- }

- if err := QueueLocationPark(r.OpID, r.TargetRegion, parkBox, r.Location, r.NewID)(c); err != nil {

- return err

- }

- return CoverRegionPredecessors(r.OpID, PhaseReleased, r.TargetRegion)(c)

- case PhaseReleased:

- // The target released itself before it was lost: its release

- // queued the location park and covered the region.

- if c.Op.Journal[JournalEvidenceTarget] == "" {

- return fmt.Errorf("%w: released without the target's evidence", ErrWrongState)

- }

- if returnLane == nil {

- return fmt.Errorf("%w: a dirty release that died before its return owes the return lane's park", ErrMalformed)

- }

- return nil

- }

- return fmt.Errorf("%w: op is restoring/%q, the lost target's release precedes the source's claim", ErrWrongState, c.Op.Phase)

- }, func(c *Claim) error {

- if owesStagingPark(c, r.OpID, r.TargetRegion) {

- return nil

- }

- _, err := queueReplicaPark(c, r.OpID, r.TargetRegion, lane, r.NewID)

- return err

- }, func(c *Claim) error {

- if returnLane == nil {

- return nil

- }

- if _, err := queueReplicaPark(c, r.OpID, r.SourceRegion, *returnLane, r.NewID); err != nil {

- return err

- }

- setJournal(c, journalReturnLane, returnLane.Location)

- return nil

- }, func(c *Claim) error {

- setJournal(c, journalLostTargetReleased, r.LossID)

- return nil

- })

- return inOneWrite(c, steps...)

+ return inOneWrite(c, func(c *Claim) error { return releaseTargetParks(c, r) }, func(c *Claim) error { setJournal(c, journalLostTargetReleased, r.LossID); return nil })

}

@@ -171,3 +90,85 @@ func owesStagingPark(c *Claim, opID, region string) bool {

func LostTargetReleased(c *Claim, lossID string) bool {

return c != nil && lossID != "" && c.Op.Kind == KindRestoring && c.Op.Journal[journalLostTargetReleased] == lossID

}

+

+func releaseTargetParks(c *Claim, r LostTargetRelease) error {

+ location := r.Location

+ if err := location.Validate(c.Host); err != nil {

+ return err

+ }

+ lane, err := parkQueueGuard(c, r.OpID, r.TargetRegion, r.Lane, r.NewID)

+ if err != nil {

+ return err

+ }

+ // The location park's scope is checked against holder.location by

+ // QueueLocationPark; the lane is this move's own.

+ if err := requireHolderLocation(c, r.Location); err != nil {

+ return err

+ }

+ if lane.Location != "moves/"+c.Host+"/"+r.OpID {

+ return fmt.Errorf("%w: a lost target's lane park is the move's own lane", ErrMalformed)

+ }

+ var returnLane *ParkScope

+ if r.ReturnLane != nil {

+ s, err := parkQueueGuard(c, r.OpID, r.SourceRegion, *r.ReturnLane, r.NewID)

+ if err != nil {

+ return err

+ }

+ returnLane = &s

+ if s.Location != "moves/"+c.Host+"/"+r.RollbackID {

+ return fmt.Errorf("%w: the return lane is this rollback's own lane", ErrMalformed)

+ }

+ steps := []func(*Claim) error{}

+ switch {

+ case c.Op.Kind == KindLanding && c.Op.Phase != PhaseCommitted:

+ steps = append(steps, RollBack(r.OpID, r.RollbackID))

+ case c.Op.Kind == KindRestoring && c.Op.Intent["rollback"] == r.RollbackID:

+ default:

+ return fmt.Errorf("%w: op is %s/%q, a lost target is released from a landing before its commit or from its own rollback", ErrWrongState, c.Op.Kind, c.Op.Phase)

+ }

+ steps = append(steps, func(c *Claim) error {

+ switch c.Op.Phase {

+ case PhaseReleasing:

+ if c.Op.Exec != "" && c.Op.Exec != "none" {

+ return fmt.Errorf("%w: exec is %q", ErrWrongState, c.Op.Exec)

+ }

+ c.Op.Phase = PhaseReleased

+ parkBox := ""

+ if stamp, err := strconv.ParseUint(c.Op.Journal[JournalTargetStamp], 10, 64); err == nil && InventoryHasStamp(c, r.Target, stamp) {

+ parkBox = r.Target

+ }

+ if err := QueueLocationPark(r.OpID, r.TargetRegion, parkBox, r.Location, r.NewID)(c); err != nil {

+ return err

+ }

+ return CoverRegionPredecessors(r.OpID, PhaseReleased, r.TargetRegion)(c)

+ case PhaseReleased:

+ // The target released itself before it was lost: its release

+ // queued the location park and covered the region.

+ if c.Op.Journal[JournalEvidenceTarget] == "" {

+ return fmt.Errorf("%w: released without the target's evidence", ErrWrongState)

+ }

+ if returnLane == nil {

+ return fmt.Errorf("%w: a dirty release that died before its return owes the return lane's park", ErrMalformed)

+ }

+ return nil

+ }

+ return fmt.Errorf("%w: op is restoring/%q, the lost target's release precedes the source's claim", ErrWrongState, c.Op.Phase)

+ }, func(c *Claim) error {

+ if owesStagingPark(c, r.OpID, r.TargetRegion) {

+ return nil

+ }

+ _, err := queueReplicaPark(c, r.OpID, r.TargetRegion, lane, r.NewID)

+ return err

+ }, func(c *Claim) error {

+ if returnLane == nil {

+ return nil

+ }

+ if _, err := queueReplicaPark(c, r.OpID, r.SourceRegion, *returnLane, r.NewID); err != nil {

+ return err

+ }

+ setJournal(c, journalReturnLane, returnLane.Location)

+ return nil

+ })

+ return inOneWrite(c, steps...)

+}

diff --git a/internal/hostclaim/move_target_fence.go b/internal/hostclaim/move_target_fence.go

new file mode 100644

index 000000000..33e14e51d

--- /dev/null

+++ b/internal/hostclaim/move_target_fence.go

@@ -0,0 +1,161 @@

+package hostclaim

+

+import (

+ "context"

+ "encoding/json"

+ "github.com/carlosframework/platform/internal/release"

+ "time"

+)

+

+const journalFencedTargetReleased = "fenced_target_released"

+

+// VerifyMoveTargetFence supplies a target-bound capability without writing the claim.

+func VerifyMoveTargetFence(ctx context.Context, s release.Store, o Observed, ref FenceRef, authority FenceAuthority) (ReturnFence, error) {

+ if o.Absent || o.Claim == nil || o.Claim.Holder == nil || o.ETag == "" {

+ return ReturnFence{}, ErrStaleOp

+ }

+ c := o.Claim

+ if (c.Op.Kind != KindLanding && c.Op.Kind != KindRestoring) || c.Op.Phase == PhaseCommitted || c.Op.ID == "" || c.Op.Intent["target"] == c.Op.Intent["source"] || c.Op.Intent["target_enrollment"] != ref.Enrollment {

+ return ReturnFence{}, ErrWrongState

+ }

+ f, err := VerifyFence(ctx, s, authority, ref)

+ if err != nil {

+ return ReturnFence{}, err

+ }

+ if f.Box() != c.Op.Intent["target"] || f.Region() != c.Op.Intent["to"] {

+ return ReturnFence{}, ErrFence

+ }

+ raw, err := json.Marshal(ref)

+ if err != nil {

+ return ReturnFence{}, err

+ }

+ at, err := time.Parse(time.RFC3339Nano, f.record.ObservedAt)

+ if err != nil || at.IsZero() {

+ return ReturnFence{}, ErrFence

+ }

+ return ReturnFence{op: c.Op.ID, box: f.Box(), ref: string(raw), observedAt: f.record.ObservedAt}, nil

+}

+

+// FencedTargetRelease has the same scopes as loss release, with verified fence authority.

+type FencedTargetRelease struct {

+ OpID, RollbackID, Target, TargetRegion, SourceRegion string

+ Location, Lane ParkScope

+ ReturnLane *ParkScope

+ NewID func() string

+ Fence ReturnFence

+}

+

+// ReleaseFencedTarget declares and consumes the fence in one CAS, including all parks.

+func ReleaseFencedTarget(r FencedTargetRelease) func(*Claim) error {

+ return func(c *Claim) error {

+ if err := RequireOp(r.OpID)(c); err != nil {

+ return err

+ }

+ if c.Holder == nil || !rollbackIDShape.MatchString(r.RollbackID) || r.Target == "" || r.NewID == nil || r.SourceRegion == "" || r.SourceRegion == r.TargetRegion {

+ return ErrMalformed

+ }

+ f := r.Fence

+ if f.op != r.OpID || f.box != r.Target || f.ref == "" || c.Op.Intent["target"] != r.Target || c.Op.Intent["source"] == r.Target || c.Op.Intent["to"] != r.TargetRegion {

+ return ErrFence

+ }

+ var ref FenceRef

+ if json.Unmarshal([]byte(f.ref), &ref) != nil || c.Op.Intent["target_enrollment"] != ref.Enrollment {

+ return ErrFence

+ }

+ if old := c.Op.Journal["fence."+r.Target]; old != "" && old != f.ref {

+ return ErrFence

+ }

+ if c.Op.Kind == KindRestoring && c.Op.Intent["rollback"] == r.RollbackID && c.Op.Journal[journalFencedTargetReleased] == f.ref {

+ return nil

+ }

+ if c.Op.Exec != r.Target && c.Op.Exec != "" && c.Op.Exec != "none" {

+ return ErrWrongState

+ }

+ at, err := time.Parse(time.RFC3339Nano, f.observedAt)

+ if err != nil || at.IsZero() || !r.Location.EvidenceAt.Equal(at) || !r.Lane.EvidenceAt.Equal(at) || (r.ReturnLane != nil && !r.ReturnLane.EvidenceAt.Equal(at)) {

+ return ErrFence

+ }

+ loss := AutomaticLoss{ID: automaticLossID(c.Op.ID, r.Target, ref), Box: r.Target, Enrollment: ref.Enrollment, Fence: ref, ObservedAt: f.observedAt, Op: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase, Role: "target", Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp, Timers: TimersUnknown}

+ if c.Holder.WriterBox == r.Target {

+ loss.Session, loss.Generation = c.Holder.Session, c.Holder.Generation

+ }

+ for i := len(c.Holder.Admissions) - 1; i >= 0; i-- {

+ a := c.Holder.Admissions[i]

+ if a.Box == r.Target {

+ loss.Session, loss.Generation, loss.LastAdmission = a.Session, a.Generation, &a

+ break

+ }

+ return inOneWrite(c, func(c *Claim) error {

+ setJournal(c, "fence."+r.Target, f.ref)

+ c.Op.Exec = "none"

+ c.Op.Participants = removeString(c.Op.Participants, r.Target)

+ return releaseTargetParks(c, LostTargetRelease{OpID: r.OpID, RollbackID: r.RollbackID, Target: r.Target, TargetRegion: r.TargetRegion, SourceRegion: r.SourceRegion, Location: r.Location, Lane: r.Lane, ReturnLane: r.ReturnLane, NewID: r.NewID})

+ }, func(c *Claim) error {

+ for i := range c.Obligations {

+ o := &c.Obligations[i]

+ if o.Op == r.OpID && (o.Kind == "park-generation" || o.Kind == "park-replica") && (o.Region == r.TargetRegion || o.Owner == "box:"+r.Target) {

+ o.Owner = "region:" + o.Region

+ loss.Reassigned = append(loss.Reassigned, ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement})

+ }

+ if !stringPresent(c.Lost, r.Target) {

+ c.Lost = append(c.Lost, r.Target)

+ }

+ if !lossRefPresent(c, loss.ID) {

+ c.LossRefs = append(c.LossRefs, loss.ID)

+ c.EvidenceRefs = append(c.EvidenceRefs, loss)

+ }

+ setJournal(c, journalFencedTargetReleased, f.ref)

+ return nil

+ })

+ }

+}

+

+// FencedTargetReleased is the durable marker, checked without creating another CAS.

+func FencedTargetReleased(c *Claim, fence ReturnFence) bool {

+ ref, ok := ConsumedMoveTargetFence(c)

+ var expected FenceRef

+ return ok && c.Op.ID == fence.op && c.Op.Intent["target"] == fence.box && json.Unmarshal([]byte(fence.ref), &expected) == nil && ref == expected

+}

+

+// ConsumedMoveTargetFence reads the claim's completed revocation, not live roster authority.

+func ConsumedMoveTargetFence(c *Claim) (FenceRef, bool) {

+ if c == nil || c.Holder == nil || c.Op.Kind != KindRestoring {

+ return FenceRef{}, false

+ }

+ target := c.Op.Intent["target"]

+ raw := c.Op.Journal["fence."+target]

+ var ref FenceRef

+ if raw == "" || raw != c.Op.Journal[journalFencedTargetReleased] || json.Unmarshal([]byte(raw), &ref) != nil || !validFenceRef(ref) || ref.Enrollment != c.Op.Intent["target_enrollment"] || c.Op.Exec == target || stringPresent(c.Op.Participants, target) || !stringPresent(c.Lost, target) {

+ return FenceRef{}, false

+ }

+ for _, l := range c.EvidenceRefs {

+ if l.ID == automaticLossID(c.Op.ID, target, ref) && l.Op == c.Op.ID && l.Box == target && l.Fence == ref && l.Incarnation == c.Holder.Incarnation && lossRefPresent(c, l.ID) {

+ return ref, true

+ }

+ return FenceRef{}, false

+}

+

+// LostMoveTargetParked requires revocation and completion of every target chore.

+func LostMoveTargetParked(c *Claim, region string) bool {

+ if c == nil || c.Holder == nil || c.Op.Kind != KindRestoring || region == "" || region != c.Op.Intent["to"] {

+ return false

+ }

+ target := c.Op.Intent["target"]

+ if target == "" || !stringPresent(c.Lost, target) || c.Op.Exec == target || stringPresent(c.Op.Participants, target) {

+ return false

+ }

+ for _, o := range c.Obligations {

+ if o.Op == c.Op.ID && (o.Box == target || o.Owner == "box:"+target || o.Region == region || o.Owner == "region:"+region) {

+ return false

+ }

+ return true

+}

+

+func (f ReturnFence) ObservedAt() time.Time {

+ at, _ := time.Parse(time.RFC3339Nano, f.observedAt)

+ return at

+}

diff --git a/internal/hostclaim/operation_loss.go b/internal/hostclaim/operation_loss.go

new file mode 100644

index 000000000..6962ec13d

--- /dev/null

+++ b/internal/hostclaim/operation_loss.go

@@ -0,0 +1,36 @@

+package hostclaim

+

+import (

+ "strconv"

+ "strings"

+)

+

+// OperationLossID selects only a declaration journalled under this operation.

+// Old claim-level LossRefs alone do not turn a later move into a loss recovery.

+func OperationLossID(c *Claim) (string, error) {

+ if c == nil {

+ return "", ErrLossRecord

+ }

+ id := ""

+ for key := range c.Op.Journal {

+ if !strings.HasPrefix(key, "loss.") {

+ continue

+ }

+ next := strings.TrimPrefix(key, "loss.")

+ if c.Op.ID == "" || !safeLossID(next) || !lossRefPresent(c, next) || id != "" {

+ return "", ErrLossRecord

+ }

+ id = next

+ }

+ return id, nil

+}

+

+// OperationLossConsumed survives admission-history trimming and process restart.

+// Only the verified admission CAS records this operation-local marker.

+func OperationLossConsumed(c *Claim, id string) bool {

+ if c == nil || c.Holder == nil || id == "" {

+ return false

+ }

+ n, err := strconv.ParseUint(c.Op.Journal["loss_admitted."+id], 10, 64)

+ return err == nil && n > 0 && n <= c.Holder.Session

+}

diff --git a/internal/hostclaim/plan12_fence_test.go b/internal/hostclaim/plan12_fence_test.go

new file mode 100644

index 000000000..51b3955f0

--- /dev/null

+++ b/internal/hostclaim/plan12_fence_test.go

@@ -0,0 +1,113 @@

+package hostclaim

+

+import (

+ "errors"

+ "reflect"

+ "strconv"

+ "testing"

+ "time"

+)

+

+func TestPlan12TargetFenceRelease(t *testing.T) {

+ for _, kind := range []string{KindLanding, KindRestoring} {

+ for _, mode := range []string{"ok", "wrong-op", "wrong-target", "wrong-enrollment", "wrong-region", "unverified", "zero-proof", "stale", "committed", "partial-return"} {

+ t.Run(kind+"/"+mode, func(t *testing.T) {

+ s, o, ref, a := c7Fixture(t)

+ var err error

+ o, err = Update(t.Context(), s, o, func(c *Claim) error {

+ c.Obligations = nil

+ c.Op.Kind, c.Op.Phase, c.Op.Exec = kind, PhaseProbing, "adi-knitting"

+ c.Op.Participants = []string{"bri-knitting", "adi-knitting"}

+ c.Op.Intent["target"], c.Op.Intent["source"], c.Op.Intent["to"], c.Op.Intent["target_enrollment"] = "adi-knitting", "bri-knitting", "ie", ref.Enrollment

+ setJournal(c, JournalSourceStamp, "1")

+ setJournal(c, JournalSrcGen, "src")

+ if kind == KindRestoring {

+ c.Op.Phase = PhaseReleasing

+ c.Op.Intent["rollback"] = lostRB

+ setJournal(c, JournalTargetStamp, strconv.FormatUint(c.Holder.Stamp, 10))

+ }

+ if mode == "wrong-target" {

+ c.Op.Intent["target"] = "other"

+ }

+ if mode == "wrong-enrollment" {

+ c.Op.Intent["target_enrollment"] = "bad"

+ }

+ if mode == "wrong-region" {

+ c.Op.Intent["to"] = "uk"

+ }

+ if mode == "committed" {

+ c.Op.Phase = PhaseCommitted

+ }

+ return nil

+ })

+ if err != nil {

+ t.Fatal(err)

+ }

+ if mode == "unverified" {

+ a.err = ErrFence

+ }

+ f, err := VerifyMoveTargetFence(t.Context(), s, o, ref, a)

+ n := 0

+ scope := func(loc string) ParkScope {

+ return ParkScope{Location: loc, Incarnation: o.Claim.Holder.Incarnation, EvidenceAt: f.ObservedAt(), RequestLimit: 2 * time.Minute}

+ }

+ r := FencedTargetRelease{OpID: o.Claim.Op.ID, RollbackID: lostRB, Target: "adi-knitting", TargetRegion: "ie", SourceRegion: "uk", Location: scope(o.Claim.Holder.Location), Lane: scope("moves/" + o.Host + "/" + o.Claim.Op.ID), Fence: f, NewID: func() string { n++; return "fence-" + strconv.Itoa(n) }}

+ if mode == "wrong-op" {

+ r.OpID = "other"

+ }

+ if mode == "zero-proof" {

+ r.Fence = ReturnFence{}

+ }

+ if mode == "partial-return" {

+ lane := scope("moves/" + o.Host + "/" + lostRB)

+ r.ReturnLane = &lane

+ }

+ if mode == "stale" {

+ if _, e := Update(t.Context(), s, o, func(*Claim) error { return nil }); e != nil {

+ t.Fatal(e)

+ }

+ s.puts = 0

+ var got Observed

+ if err == nil {

+ got, err = Update(t.Context(), s, o, ReleaseFencedTarget(r))

+ }

+ if mode != "ok" && mode != "partial-return" {

+ if err == nil {

+ t.Fatal("accepted", mode)

+ }

+ if mode == "stale" && !errors.Is(err, ErrConflict) {

+ t.Fatal(err)

+ }

+ return

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ if s.puts != 1 || got.Claim.Op.Phase != PhaseReleased || got.Claim.Op.Exec != "none" || stringPresent(got.Claim.Op.Participants, r.Target) || len(got.Claim.Obligations) < 2 {

+ t.Fatal(got.Claim)

+ }

+ clone, _ := cloneClaim(got.Claim)

+ if err := ReleaseFencedTarget(r)(clone); err != nil || !reflect.DeepEqual(clone, got.Claim) {

+ t.Fatal("replay", err)

+ }

+ if err := ReturnToSource(r.OpID, "bri-knitting", false, ReturnFence{})(clone); err != nil {

+ t.Fatal("source needs fence authority", err)

+ }

+ })

+ }

+}

+

+func TestPlan12FenceRefusesCleanCoverage(t *testing.T) {

+ c := lostTargetClaim(t, KindRestoring, PhaseProbing)

+ c.LossRefs = nil

+ delete(c.Op.Journal, "loss."+lostID)

+ c.Op.Exec = "ie-box"

+ c.Op.Journal["fence.uk-box"] = "verified-in-release"

+ h := c.Holder

+ in := Admission{ObservedOpID: c.Op.ID, ObservedPhase: c.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "ie-box", Mode: "cold", NewGeneration: "return", Region: "ie", PredecessorRegion: "uk", Coverage: "clean-return"}

+ if err := AdmitWriter(in)(c); !errors.Is(err, ErrFence) {

+ t.Fatal(err)

+ }

+}

diff --git a/internal/hostclaim/plan12_history_test.go b/internal/hostclaim/plan12_history_test.go

new file mode 100644

index 000000000..a055dae82

--- /dev/null

+++ b/internal/hostclaim/plan12_history_test.go

@@ -0,0 +1,18 @@

+package hostclaim

+

+import (

+ "strings"

+ "testing"

+)

+

+func TestPlan12RetiredLocationWriteGate(t *testing.T) {

+ for _, enabled := range []string{"", "1"} {

+ t.Setenv("CARLOS_HOST_CLAIM_RETIRED_LOCATIONS", enabled)

+ inc := strings.Repeat("a", 64)

+ c := &Claim{Host: "shop.example", Holder: &Holder{Incarnation: inc, Location: "instances/shop.example"}}

+ retire(c, inc)

+ if (c.RetiredLocations[inc] != "") != (enabled == "1") {

+ t.Fatal(enabled, c.RetiredLocations)

+ }

+}

diff --git a/internal/hostclaim/plan12_target_release_test.go b/internal/hostclaim/plan12_target_release_test.go

new file mode 100644

index 000000000..27157c76f

--- /dev/null

+++ b/internal/hostclaim/plan12_target_release_test.go

@@ -0,0 +1,30 @@

+package hostclaim

+

+import (

+ "strconv"

+ "testing"

+)

+

+func TestPlan12TargetReleasedCapturesWriter(t *testing.T) {

+ for _, admitted := range []bool{false, true} {

+ c := lostTargetClaim(t, KindRestoring, PhaseReleasing)

+ c.Lost, c.LossRefs, c.LostRevisions = nil, nil, nil

+ delete(c.Op.Journal, "loss."+lostID)

+ c.Op.Participants = []string{"ie-box", "uk-box"}

+ c.Op.Exec = "uk-box"

+ if !admitted {

+ c.Holder.WriterBox = "ie-box"

+ }

+ r := lostRelease(c, false)

+ if err := TargetReleased(lostOp, "uk-box", "uk", "evidence", r.Location, r.NewID)(c); err != nil {

+ t.Fatal(err)

+ }

+ if admitted {

+ if c.Op.Journal[JournalTargetSession] != strconv.FormatUint(c.Holder.Session, 10) || c.Op.Journal[JournalTargetGeneration] != c.Holder.Generation {

+ t.Fatal(c.Op.Journal)

+ }

+ } else if _, ok := c.Op.Journal[JournalTargetSession]; ok {

+ t.Fatal("invented target session")

+ }

+}

diff --git a/internal/hostguard/fence.go b/internal/hostguard/fence.go

index dee8d1b05..7e9f3baf3 100644

--- a/internal/hostguard/fence.go

+++ b/internal/hostguard/fence.go

@@ -268,6 +268,9 @@ func StopAuthorityMatches(c *hostclaim.Claim, t FenceTarget) bool {

// landing's own current writer on this box (claim Plan 9 D4).

if writerStop(t.ObligationID) {

id, ok := LandingWriterStop(c, t.Box, t.Region)

+ if c.Op.Kind == hostclaim.KindRestoring || c.Op.Kind == hostclaim.KindReverting {

+ id, ok = RecoveryWriterStop(c, t.Box, t.Region)

+ }

if c.Op.Kind == hostclaim.KindRelocating {

id, ok = RelocationWriterStop(c, t.Box, t.Region)

}

diff --git a/internal/hostguard/writerstop.go b/internal/hostguard/writerstop.go

index eb19d45c2..d64fb558f 100644

--- a/internal/hostguard/writerstop.go

+++ b/internal/hostguard/writerstop.go

@@ -110,3 +110,16 @@ func (g Guard) CompletedWriterStops(ctx context.Context, host string, captured C

})

return out, err

}

+

+func RecoveryWriter(c *hostclaim.Claim, box string) (hostclaim.AdmissionRecord, bool) {

+ if c == nil || (c.Op.Kind != hostclaim.KindRestoring && c.Op.Kind != hostclaim.KindReverting) || c.Op.Intent["source"] != box {

+ return hostclaim.AdmissionRecord{}, false

+ }

+ return operationWriter(c, box, c.Op.Kind)

+}

+func RecoveryWriterStop(c *hostclaim.Claim, box, region string) (string, bool) {

+ if _, ok := RecoveryWriter(c, box); !ok {

+ return "", false

+ }

+ return operationWriterStop(c, box, region, c.Op.Kind)

+}

diff --git a/internal/hostops/automatic_loss.go b/internal/hostops/automatic_loss.go

new file mode 100644

index 000000000..3d72b8da0

--- /dev/null

+++ b/internal/hostops/automatic_loss.go

@@ -0,0 +1,76 @@

+package hostops

+

+import (

+ "context"

+ "errors"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/release"

+ "reflect"

+ "time"

+)

+

+// ReportAutomaticLosses materializes retained consequences even after operation

+// or holder replacement. The signed record is durable before reporting is marked.

+func (w *ControlWriter) ReportAutomaticLosses(ctx context.Context, signer release.Signer, host string) error {

+ if recoveryWriter(ctx) != w {

+ return ErrContract

+ }

+ if err := w.lossSigner(signer); err != nil {

+ return err

+ }

+ o, err := hostclaim.Read(ctx, w.raw, host)

+ if err != nil {

+ return err

+ }

+ if o.Absent {

+ return nil

+ }

+ for _, l := range o.Claim.EvidenceRefs {

+ if l.Reported {

+ continue

+ }

+ record, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, l.ID)

+ if errors.Is(err, hostclaim.ErrLossRecordAbsent) {

+ at, e := time.Parse(time.RFC3339Nano, l.ObservedAt)

+ if e != nil {

+ return e

+ }

+ r := hostclaim.LossRecord{AlertsPending: true, V: 1, ID: l.ID, State: "prepared", Actor: "automatic", AcceptedAt: at, Host: host, HolderIncarnation: l.Incarnation, HolderStamp: l.Stamp, HolderSession: l.Session, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: l.Op, OpKind: l.Kind, OpPhase: l.Phase, Box: l.Box, Enrollment: l.Enrollment, FencedAt: at, Timers: hostclaim.TimerLoss{Knowledge: l.Timers}, Automatic: &l}

+ record, err = hostclaim.CreateLossRecord(ctx, w.raw, signer, r)

+ }

+ if err != nil {

+ return err

+ }

+ if !reflect.DeepEqual(record.Record.Automatic, &l) || record.Record.Host != host {

+ return hostclaim.ErrLossRecord

+ }

+ if record.Record.State == "prepared" {

+ next := record.Record

+ next.State = "committed"

+ next.ResultClaimGen = next.ExpectedClaimGen

+ record, err = hostclaim.ReplaceLossRecord(ctx, w.raw, signer, record, next)

+ if err != nil {

+ return err

+ }

+ if record.Record.State != "committed" {

+ return hostclaim.ErrLossRecord

+ }

+ o, err = hostclaim.Update(ctx, w.raw, o, func(c *hostclaim.Claim) error {

+ for i := range c.EvidenceRefs {

+ if c.EvidenceRefs[i].ID == l.ID {

+ if !reflect.DeepEqual(c.EvidenceRefs[i], l) {

+ return hostclaim.ErrLossRecord

+ }

+ c.EvidenceRefs[i].Reported = true

+ return nil

+ }

+ return hostclaim.ErrLossRecord

+ })

+ if err != nil {

+ return err

+ }

+ return nil

+}

diff --git a/internal/hostops/move_fence_release.go b/internal/hostops/move_fence_release.go

new file mode 100644

index 000000000..5935b032d

--- /dev/null

+++ b/internal/hostops/move_fence_release.go

@@ -0,0 +1,174 @@

+package hostops

+

+import (

+ "context"

+ "crypto/rand"

+ "encoding/json"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// ReleaseFencedTarget advances one record-first recovery step per grant:

+// record decision, claim release with parks, then record release. Each pass

+// verifies unconsumed authority; completion uses the durable claim evidence.

+func (g *MoveRecoveryGrant) ReleaseFencedTarget(ctx context.Context, ref hostclaim.FenceRef) error {

+ if g == nil || g.state == nil {

+ return ErrPending

+ }

+ g.state.mu.Lock()

+ defer g.state.mu.Unlock()

+ o, record, rec, err := g.current(ctx)

+ if err != nil {

+ return err

+ }

+

+ m := rec.Move

+ if m == nil || m.ID != g.o.OpID || m.Incarnation != g.o.Incarnation {

+ return ErrIdentity

+ }

+ guard := provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation}

+ writeRecord := func() error {

+ body, err := g.prepareRecord(ctx, rec)

+ if err != nil {

+ return err

+ }

+ _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, record.Holder.Key, record.Body, body, record.ETag)

+ return err

+ }

+ if consumed, ok := hostclaim.ConsumedMoveTargetFence(o.Claim); ok {

+ if consumed != ref || m.Rollback == nil || m.State != provision.MoveRollingBack || rec.Region != m.From {

+ return ErrIdentity

+ }

+ rb := m.Rollback

+ if (rb.TargetPhase == provision.RbTargetReleased || rb.TargetPhase == provision.RbTargetCleaned && rb.TargetParkedAt != nil) && !rb.Dirty {

+ if rb.LossRef == ref.Key && rb.Evidence == ref.Key {

+ return nil

+ }

+ return g.verifyReleasedTarget(ctx, o, rec)

+ }

+ if rb.LossRef != ref.Key {

+ return ErrIdentity

+ }

+ if err := provision.MoveRollbackLossCommitted(guard)(&rec); err != nil {

+ return err

+ }

+ return writeRecord()

+ }

+ fence, err := hostclaim.VerifyMoveTargetFence(ctx, g.w.raw, o, ref, g.authority)

+ if err != nil {

+ return err

+ }

+ if m.State == provision.MoveLanding {

+ if rec.Region != m.To {

+ return ErrIdentity

+ }

+ id, err := moves.NewRollbackID(rand.Reader)

+ if err != nil {

+ return err

+ }

+ if err := provision.MoveRollBack(guard, id, provision.RollbackTargetFenced, g.w.now())(&rec); err != nil {

+ return err

+ }

+ } else if m.State != provision.MoveRollingBack || m.Rollback == nil || rec.Region != m.From {

+ return ErrIdentity

+ }

+ rb := m.Rollback

+ if rb.Dirty {

+ return ErrPending

+ } // J4: a dead dirty target cannot build a return lane.

+ released := rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty

+ if released {

+ if err := g.verifyReleasedTarget(ctx, o, rec); err != nil {

+ return err

+ }

+ if o.Claim.Op.Kind != hostclaim.KindRestoring || o.Claim.Op.Phase != hostclaim.PhaseReleasing || o.Claim.Op.Exec != o.Claim.Op.Intent["target"] {

+ return ErrPending

+ }

+ } else if rb.TargetPhase != provision.RbTargetLost || rb.LossRef != ref.Key {

+ if err := provision.MoveRollbackTargetLost(guard, ref.Key, g.w.now())(&rec); err != nil {

+ return err

+ }

+ return writeRecord()

+ }

+ h := o.Claim.Holder

+ r := hostclaim.FencedTargetRelease{OpID: m.ID, RollbackID: rb.ID, Target: o.Claim.Op.Intent["target"], TargetRegion: m.To, SourceRegion: m.From,

+ Location: moves.TargetLocationScope(*h, fence.ObservedAt()), Lane: moves.StagingScope(o.Host, m.ID, h.Incarnation, fence.ObservedAt()), NewID: moveLossNewID(ref.SHA256), Fence: fence}

+ if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil {

+ return err

+ }

+ // Fence and roster reads may have raced a new record. Bind both ETag and sequence again.

+ if _, _, _, err := g.current(ctx); err != nil {

+ return err

+ }

+ g.state.active = false

+ _, err = hostclaim.Update(ctx, g.w.raw, o, func(c *hostclaim.Claim) error {

+ record, err := ReadRecord(ctx, g.w.raw, h.Key)

+ if err != nil {

+ return err

+ }

+ _, seq, err := recordSequence(record.Body)

+ if err != nil {

+ return err

+ }

+ if record.ETag != g.o.RecordETag || seq != g.o.RecordSeq {

+ return release.ErrCASConflict

+ }

+ if err := moveAdmission(ctx, g.w.raw, o.Host, c); err != nil {

+ return err

+ }

+ return hostclaim.ReleaseFencedTarget(r)(c)

+ })

+ return err

+}

+

+// The record-first release can outlive its target. Require the canonical complete report.

+func (g *MoveRecoveryGrant) verifyReleasedTarget(ctx context.Context, o hostclaim.Observed, rec provision.Record) error {

+ m, h := rec.Move, o.Claim.Holder

+ if m == nil || m.Rollback == nil || m.Rollback.Dirty || (m.Rollback.TargetPhase != provision.RbTargetReleased && (m.Rollback.TargetPhase != provision.RbTargetCleaned || m.Rollback.TargetParkedAt == nil)) {

+ return ErrIdentity

+ }

+ key := moves.StoppedKey(moves.ReportPrefix(h.Account, h.App, o.Host, m.ID), "target")

+ if m.Rollback.Evidence != key {

+ return ErrIdentity

+ }

+ raw, _, err := g.w.raw.Get(ctx, key)

+ if err != nil {

+ return err

+ }

+ var report moves.StoppedReport

+ if json.Unmarshal(raw, &report) != nil || !report.Complete() || report.Role != "target" || report.Box != o.Claim.Op.Intent["target"] || report.Host != o.Host || report.OpID != m.ID || report.Incarnation != h.Incarnation {

+ return ErrIdentity

+ }

+ return nil

+}

+

+// StampLostMoveTargetParked uses the same expiring, record-sequenced grant as closure.

+func (g *MoveRecoveryGrant) StampLostMoveTargetParked(ctx context.Context) error {

+ if g == nil || g.state == nil {

+ return ErrPending

+ }

+ g.state.mu.Lock()

+ defer g.state.mu.Unlock()

+ o, r, rec, err := g.current(ctx)

+ if err != nil {

+ return err

+ }

+ m := rec.Move

+ if m == nil || m.ID != g.o.OpID || m.Incarnation != g.o.Incarnation || m.Rollback == nil || !hostclaim.LostMoveTargetParked(o.Claim, m.To) {

+ return ErrPending

+ }

+ if m.Rollback.TargetParkedAt != nil {

+ return nil

+ }

+ if err := provision.MoveLostTargetParked(provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation}, m.Rollback.ID, g.w.now())(&rec); err != nil {

+ return err

+ }

+ body, err := g.prepareRecord(ctx, rec)

+ if err != nil {

+ return err

+ }

+ _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, r.Holder.Key, r.Body, body, r.ETag)

+ return err

+}

diff --git a/internal/hostops/move_loss_admission.go b/internal/hostops/move_loss_admission.go

new file mode 100644

index 000000000..a2b798571

--- /dev/null

+++ b/internal/hostops/move_loss_admission.go

@@ -0,0 +1,49 @@

+package hostops

+

+import (

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "strconv"

+ "strings"

+)

+

+const MoveLossReplicaBaseline = "replica:"

+const MoveLossSealBaseline = "seal:"

+

+// ParseMoveLossBaseline is shared by the frozen-source writer and validators.

+func ParseMoveLossBaseline(value string) (kind, region, identity string, err error) {

+ if strings.HasPrefix(value, MoveLossReplicaBaseline) {

+ r, g, ok := strings.Cut(strings.TrimPrefix(value, MoveLossReplicaBaseline), "/")

+ if ok && r != "" && g != "" && !strings.Contains(g, "/") {

+ return "replica", r, g, nil

+ }

+ if strings.HasPrefix(value, MoveLossSealBaseline) {

+ id := strings.TrimPrefix(value, MoveLossSealBaseline)

+ if id != "" && !strings.Contains(id, "/") {

+ return "seal", "", id, nil

+ }

+ return "", "", "", hostclaim.ErrLossRecord

+}

+

+// MoveLossRecoveryOwner shares the resolve owner's signed-intent policy with boxes.

+// This policy is pure; it exposes neither the console store nor mutation grants.

+func MoveLossRecoveryOwner() hostclaim.LossRecoveryOwner { return &moveLossOwner{} }

+

+func (m *moveLossOwner) ValidateLossRecovery(c hostclaim.Claim, loss hostclaim.LossRecord, a hostclaim.Admission) error {

+ in, _, err := lossIntentOf(loss)

+ if err != nil {

+ return err

+ }

+ stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64)

+ if err != nil || stamp == 0 || c.Holder == nil || c.Holder.Stamp != stamp ||

+ c.Holder.Incarnation != in.Incarnation || c.Holder.Key != in.RecordKey ||

+ c.Op.ID != in.Move || c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseProbing ||

+ c.Op.Intent["rollback"] != in.Rollback || c.Op.Exec != in.Source || c.Op.Intent["source"] != in.Source ||

+ c.Op.Intent["target"] != in.Target || c.Op.Intent["to"] != in.TargetRegion ||

+ !hostclaim.LostTargetReleased(&c, loss.ID) || a.Box != in.Source || a.Region != in.SourceRegion ||

+ a.Mode != "cold" || a.Coverage != "" || a.KeepStamp || a.PredecessorCertified {

+ return hostclaim.ErrLossRecord

+ }

+ return nil

+}

diff --git a/internal/hostops/move_loss_owner.go b/internal/hostops/move_loss_owner.go

index 2967230e6..8204632b3 100644

--- a/internal/hostops/move_loss_owner.go

+++ b/internal/hostops/move_loss_owner.go

@@ -546,17 +546,17 @@ func (m *moveLossOwner) LossWriteSources(_ context.Context, o hostclaim.Observed

if a.Box != in.Target || a.Session <= in.CutSession {

continue

}

- if err := add(hostclaim.WriteSource{Reason: "target", Box: a.Box, Generation: a.Generation, Session: a.Session, Region: a.Region, Baseline: "replica:" + a.Region + "/" + a.Generation}); err != nil {

+ if err := add(hostclaim.WriteSource{Reason: "target", Box: a.Box, Generation: a.Generation, Session: a.Session, Region: a.Region, Baseline: MoveLossReplicaBaseline + a.Region + "/" + a.Generation}); err != nil {

return nil, err

}

if h.WriterBox == in.Target && h.Session > in.CutSession {

- if err := add(hostclaim.WriteSource{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: in.TargetRegion, Baseline: "replica:" + in.TargetRegion + "/" + h.Generation}); err != nil {

+ if err := add(hostclaim.WriteSource{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: in.TargetRegion, Baseline: MoveLossReplicaBaseline + in.TargetRegion + "/" + h.Generation}); err != nil {

return nil, err

}

if !in.SrcCovered {

- if err := add(hostclaim.WriteSource{Reason: "source-cut", Box: in.Source, Generation: in.SrcGen, Session: in.CutSession, Region: in.SourceRegion, Baseline: "seal:" + in.Move}); err != nil {

+ if err := add(hostclaim.WriteSource{Reason: "source-cut", Box: in.Source, Generation: in.SrcGen, Session: in.CutSession, Region: in.SourceRegion, Baseline: MoveLossSealBaseline + in.Move}); err != nil {

return nil, err

}

diff --git a/internal/hostops/move_recovery_bridge.go b/internal/hostops/move_recovery_bridge.go

index 59539e0d1..52e1fe31b 100644

--- a/internal/hostops/move_recovery_bridge.go

+++ b/internal/hostops/move_recovery_bridge.go

@@ -5,7 +5,6 @@ import (

"crypto/sha256"

"encoding/hex"

"encoding/json"

- "slices"

"strconv"

"time"

@@ -304,36 +303,3 @@ func RecoverTerminatedMovingHost(ctx context.Context, s release.Store, mh *Start

_, err = hostclaim.ConvertMovingToRelocating(mh.Lifecycle(ctx), s, fresh, c.Op.ID, ref, authority, guard)

return err

}

-

-// StampLostMoveTargetParked completes the missing console half for a lost

-// target. It neither acknowledges chores nor treats termination as disk loss.

-func StampLostMoveTargetParked(ctx context.Context, s release.Store, host string, now time.Time) error {

- o, err := hostclaim.Read(ctx, s, host)

- if err != nil {

- return err

- }

- if o.Absent || o.Claim.Holder == nil {

- return ErrIdentity

- }

- h := o.Claim.Holder

- _, err = setLifecycleRecordMove(ctx, s, o, func(rec *provision.Record) error {

- c, err := hostclaim.Read(ctx, s, host)

- if err != nil {

- return err

- }

- if c.Absent || c.Claim.Holder == nil || c.ETag != o.ETag || rec.Move == nil || rec.Move.Rollback == nil || c.Claim.Holder.Incarnation != h.Incarnation || rec.Move.Incarnation != h.Incarnation || rec.Move.ID != c.Claim.Op.ID || (c.Claim.Op.Kind != hostclaim.KindRestoring && c.Claim.Op.Kind != hostclaim.KindLanding) {

- return ErrIdentity

- }

- target := c.Claim.Op.Intent["target"]

- if target == "" || !slices.Contains(c.Claim.Lost, target) || rec.Move.To == "" {

- return ErrIdentity

- }

- for _, chore := range c.Claim.Obligations {

- if chore.Op == c.Claim.Op.ID && (chore.Box == target || chore.Owner == "box:"+target || chore.Region == rec.Move.To || chore.Owner == "region:"+rec.Move.To) {

- return ErrPending

- }

- return provision.MoveLostTargetParked(provision.MoveGuard{ID: c.Claim.Op.ID, Incarnation: h.Incarnation}, rec.Move.Rollback.ID, now)(rec)

- })

- return err

-}

diff --git a/internal/hostops/move_recovery_bridge_test.go b/internal/hostops/move_recovery_bridge_test.go

index eda45bb3b..79a092493 100644

--- a/internal/hostops/move_recovery_bridge_test.go

+++ b/internal/hostops/move_recovery_bridge_test.go

@@ -457,44 +457,34 @@ func TestC7RecoveryRefusesBeforeAnyWrite(t *testing.T) {

}

func TestC7TargetParkedAtWaitsForRegionAndBoxChores(t *testing.T) {

- ctx := context.Background()

- now := time.Now().UTC()

- for _, kind := range []string{"none", "box", "region"} {

+ for _, kind := range []string{"none", "box", "region", "participant", "executor", "region-mismatch"} {

t.Run(kind, func(t *testing.T) {

- s := release.NewDirStore(t.TempDir())

- host := "shop.example.com"

- inc, err := hostclaim.Incarnation("bab", "jam", host, now)

- if err != nil {

- t.Fatal(err)

- }

- c := hostclaim.Claim{V: 2, Host: host, Gen: 1, Holder: &hostclaim.Holder{Family: "instance", Account: "bab", App: "jam", Key: provision.RecordKey("bab", "jam", host), Incarnation: inc, CreatedAt: now, Location: "instances/" + host},

- Op: hostclaim.Op{Kind: hostclaim.KindRestoring, ID: "move-1", Intent: map[string]string{"target": "target", "app_id": "canonical", "admission_id": "move-1"}}, Lost: []string{"target"}}

- switch kind {

- case "box":

- c.Obligations = []hostclaim.Obligation{{ID: "local", Op: "move-1", Kind: "park-local", Box: "target", Owner: "box:target", Retirement: "r"}}

- case "region":

- c.Obligations = []hostclaim.Obligation{{ID: "region", Op: "move-1", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "r"}}

- }

- if err := c.Validate(); err != nil {

- t.Fatal(err)

- }

- raw, _ := json.Marshal(c)

- key, _ := hostclaim.Key(host)

- if _, err = s.Put(ctx, key, raw, ""); err != nil {

- t.Fatal(err)

- }

- rec := provision.Record{Schema: 1, Host: host, Slug: "jam", Account: "bab", App: "jam", Channel: "edge", Region: "ie", CreatedAt: now, Move: &provision.MoveState{ID: "move-1", Incarnation: inc, From: "ie", To: "uk", State: provision.MoveRollingBack, Rollback: &provision.MoveRollback{ID: "rb-1", TargetPhase: provision.RbTargetReleased}}}

- rb, _ := json.Marshal(rec)

- if _, err = s.Put(ctx, provision.RecordKey("bab", "jam", host), rb, ""); err != nil {

- t.Fatal(err)

- }

- admission := AppAdmission{V: 1, AppID: "canonical", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]Registration{"move-1": {Kind: RegistrationMove, AppID: "canonical", Epoch: 1, Host: host, Incarnation: inc}}, Effects: map[string]Effect{}}

- admissionBody, _ := json.Marshal(admission)

- if _, err = s.Put(ctx, "apps/bab/jam/admission.json", admissionBody, ""); err != nil {

- t.Fatal(err)

+ w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseReleased, "none"

+ c.Op.Participants = []string{"source"}

+ c.Lost = []string{"target"}

+ c.Obligations = nil

+ switch kind {

+ case "box":

+ c.Obligations = []hostclaim.Obligation{{ID: "local", Op: "move", Kind: "park-local", Box: "target", Owner: "box:target", Retirement: "r"}}

+ case "region":

+ c.Obligations = []hostclaim.Obligation{{ID: "region", Op: "move", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "r"}}

+ case "participant":

+ c.Op.Participants = append(c.Op.Participants, "target")

+ case "executor":

+ c.Op.Exec = "target"

+ case "region-mismatch":

+ c.Op.Intent["to"] = "ie"

+ }

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", TargetPhase: provision.RbTargetReleased}

+ })

+ stamp := func() error {

+ return w.control.RunRecovery(t.Context(), func(ctx context.Context) error {

+ return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) })

+ })

}

- wrapped := conditionalRecordStore{raw: s}

- err = StampLostMoveTargetParked(ctx, wrapped, host, now)

+ err := stamp()

if kind != "none" {

if !errors.Is(err, ErrPending) {

t.Fatalf("pending: %v", err)

@@ -502,7 +492,7 @@ func TestC7TargetParkedAtWaitsForRegionAndBoxChores(t *testing.T) {

} else if err != nil {

t.Fatal(err)

}

- after, err := provision.GetRecord(ctx, s, "bab", "jam", host)

+ after, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host)

if err != nil {

t.Fatal(err)

}

@@ -510,11 +500,12 @@ func TestC7TargetParkedAtWaitsForRegionAndBoxChores(t *testing.T) {

t.Fatal("wrong park stamp")

}

if kind == "none" {

- if err := StampLostMoveTargetParked(ctx, wrapped, host, now.Add(time.Hour)); err != nil {

+ tag := w.recordTag(t)

+ w.clock = w.now.Add(time.Hour)

+ if err := stamp(); err != nil {

t.Fatal(err)

}

- after, _ = provision.GetRecord(ctx, s, "bab", "jam", host)

- if !after.Move.Rollback.TargetParkedAt.Equal(now) {

+ if w.recordTag(t) != tag {

t.Fatal("changed original park time")

}

diff --git a/internal/hostops/move_recovery_grant.go b/internal/hostops/move_recovery_grant.go

index 690ed4eb5..47ad94212 100644

--- a/internal/hostops/move_recovery_grant.go

+++ b/internal/hostops/move_recovery_grant.go

@@ -18,9 +18,8 @@ import (

// MoveRecoveryGrant is one callback's authority over an exact move and record.

// It exposes no store or arbitrary transition. Mutations consume the observation;

// another action requires another pass. The grant expires when the hook returns.

-// Plan 5 must align its owners to close the record before releasing the claim

-// before wiring this hook. Existing release-first owners must finish their own

-// closes: a serving claim with a live move record cannot acquire this grant.

+// Record-first owners close before releasing the claim; LastMove repairs a

+// crash between those writes on the next pass.

type MoveRecoveryGrant struct {

state *moveRecoveryGrantState

w *ControlWriter

@@ -145,7 +144,6 @@ const (

RecoveryMoveClose MoveRecoveryRecordAction = "close"

RecoveryMoveRolledBackClose MoveRecoveryRecordAction = "rolled-back-close"

RecoveryMoveRevertClose MoveRecoveryRecordAction = "revert-close"

- RecoveryMoveAbortComplete MoveRecoveryRecordAction = "abort-complete"

RecoveryMoveDeadlineRollback MoveRecoveryRecordAction = "deadline-rollback"

)

@@ -164,7 +162,7 @@ func (g *MoveRecoveryGrant) Record(ctx context.Context, action MoveRecoveryRecor

return ErrIdentity

}

switch action {

- case RecoveryMoveClose, RecoveryMoveRolledBackClose, RecoveryMoveRevertClose, RecoveryMoveAbortComplete:

+ case RecoveryMoveClose, RecoveryMoveRolledBackClose, RecoveryMoveRevertClose:

for _, chore := range o.Claim.Obligations {

if chore.Op == g.o.OpID {

return ErrPending

@@ -189,11 +187,6 @@ func (g *MoveRecoveryGrant) Record(ctx context.Context, action MoveRecoveryRecor

return ErrPending

}

mutate = provision.MoveRevertClose(guard, now)

- case RecoveryMoveAbortComplete:

- if o.Claim.Op.Kind != hostclaim.KindMoving || o.Claim.Op.Phase != hostclaim.PhaseAborting || len(o.Claim.Obligations) != 0 {

- return ErrPending

- }

- mutate = provision.MoveGuardedAbortComplete(guard, now)

case RecoveryMoveDeadlineRollback:

if o.Claim.Op.Kind != hostclaim.KindLanding && !(o.Claim.Op.Kind == hostclaim.KindMoving && o.Claim.Op.Phase == hostclaim.PhaseSealed) {

return ErrPending

diff --git a/internal/hostops/move_recovery_grant_test.go b/internal/hostops/move_recovery_grant_test.go

index 470289fc4..ec0ecf908 100644

--- a/internal/hostops/move_recovery_grant_test.go

+++ b/internal/hostops/move_recovery_grant_test.go

@@ -414,55 +414,6 @@ func TestMoveRecoveryPassUsesTypedHookOnlyInsideRunRecovery(t *testing.T) {

}

-func TestMoveRecoveryAbortCloseFreshGrantThenRelease(t *testing.T) {

- w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {

- c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindMoving, hostclaim.PhaseAborting, "source"

- c.Obligations = nil

- r.Move.State = provision.MoveAborting

- })

- ctx := context.Background()

- run := func(action func(context.Context, *MoveRecoveryGrant) error) error {

- return w.control.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), action) })

- }

- before := *w.claim(t).Claim.Holder

- if err := run(func(ctx context.Context, g *MoveRecoveryGrant) error {

- if err := g.Record(ctx, RecoveryMoveAbortComplete); err != nil {

- return err

- }

- // Closing spends the grant and advances record_seq. A new grant is required.

- if err := g.Claim(ctx, RecoveryMoveRelease); !errors.Is(err, ErrPending) {

- t.Fatalf("spent close grant = %v", err)

- }

- return nil

- }); err != nil {

- t.Fatal(err)

- }

- rec, err := provision.GetRecord(ctx, w.raw, "bab", "app", w.host)

- if err != nil {

- t.Fatal(err)

- }

- if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != "move" || rec.LastMove.Outcome != "aborted" {

- t.Fatalf("abort not closed: %+v", rec)

- }

- if c := w.claim(t).Claim; c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseAborting {

- t.Fatal("close released claim")

- }

- // A later recovery pass mints a new grant over the closed record.

- if err := run(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }); err != nil {

- t.Fatal(err)

- }

- c := w.claim(t).Claim

- if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "source" {

- t.Fatalf("abort not released: %+v", c.Op)

- }

- if c.Holder.Incarnation != before.Incarnation || c.Holder.Stamp != before.Stamp || c.Holder.Session != before.Session || c.Holder.Generation != before.Generation {

- t.Fatal("release changed writer identity")

- }

- if err := hostclaim.RequireReleasedMove("move")(c); err != nil {

- t.Fatal(err)

- }

-}

-

func TestMoveRecoveryReleaseRequiresMatchingOutcomeAndExecutor(t *testing.T) {

for _, kind := range []string{hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindMoving} {

for _, outcome := range []string{"done", "rolled_back", "aborted"} {

diff --git a/internal/hostops/plan12_fence_test.go b/internal/hostops/plan12_fence_test.go

new file mode 100644

index 000000000..60cd19933

--- /dev/null

+++ b/internal/hostops/plan12_fence_test.go

@@ -0,0 +1,262 @@

+package hostops

+

+import (

+ "context"

+ "crypto/ed25519"

+ "encoding/json"

+ "errors"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/termfence"

+ "strings"

+ "testing"

+)

+

+type plan12BumpFence struct {

+ c7ReplayAuthority

+ bump func()

+ reject bool

+}

+

+func (a *plan12BumpFence) ValidateFence(ctx context.Context, f termfence.Record) (hostclaim.FenceBinding, error) {

+ if a.reject {

+ return hostclaim.FenceBinding{}, hostclaim.ErrFence

+ }

+ if a.bump != nil {

+ fn := a.bump

+ a.bump = nil

+ fn()

+ }

+ return a.c7ReplayAuthority.ValidateFence(ctx, f)

+}

+func TestPlan12RecoveryFenceGrant(t *testing.T) {

+ for _, mode := range []string{"ok", "admitted", "installed", "from-landing", "from-restoring", "record-released", "changed-sequence", "dirty", "escaped", "wrong-ref"} {

+ t.Run(mode, func(t *testing.T) {

+ w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseProbing, "source"

+ c.Op.Intent["source"], c.Op.Intent["target"], c.Op.Intent["to"], c.Op.Intent["target_enrollment"] = "target", "source", "ie", strings.Repeat("a", 32)

+ c.Op.Participants = []string{"target", "source"}

+ c.Obligations = nil

+ c.Op.Journal = map[string]string{hostclaim.JournalSourceStamp: "1", hostclaim.JournalSrcGen: "src"}

+ r.Region = "uk"

+ r.Move.From, r.Move.To, r.Move.State = "uk", "ie", provision.MoveRollingBack

+ at := r.CreatedAt

+ r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", Reason: provision.RollbackTargetFenced, Evidence: "control/box-fences/" + strings.Repeat("a", 32) + ".json", RequestedAt: &at, TargetPhase: provision.RbTargetLost, LossRef: "control/box-fences/" + strings.Repeat("a", 32) + ".json", LostAt: &at, SourcePhase: provision.RbSourceWaiting, Dirty: mode == "dirty"}

+ if mode == "admitted" || mode == "installed" {

+ c.Holder.Session = 7

+ c.Holder.Stamp = 7

+ c.Holder.Generation = "target-gen"

+ c.Holder.WriterBox = "source"

+ c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: 7, Box: "source", Generation: "target-gen", Mode: "cold", Region: "ie"}}

+ }

+ if mode == "installed" {

+ c.Inventory = map[string]hostclaim.InventoryEntry{"source": {Layout: "flat", Paths: map[string]uint64{"app.db": 7}}}

+ }

+ if mode == "from-restoring" || mode == "record-released" {

+ c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseReleasing

+ c.Op.Intent["rollback"] = r.Move.Rollback.ID

+ c.Op.Journal[hostclaim.JournalTargetStamp] = "7"

+ r.Move.Rollback.TargetPhase = provision.RbTargetReleasing

+ r.Move.Rollback.LossRef = ""

+ r.Move.Rollback.LostAt = nil

+ r.Move.Rollback.Reason = provision.RollbackLandingDeadline

+ }

+ if mode == "record-released" {

+ r.Move.Rollback.TargetPhase = provision.RbTargetReleased

+ r.Move.Rollback.Evidence = moves.StoppedKey(moves.ReportPrefix("bab", "app", c.Host, c.Op.ID), "target")

+ }

+ if mode == "from-landing" {

+ r.Region = "ie"

+ r.Move.State = provision.MoveLanding

+ r.Move.Rollback = nil

+ }

+ })

+ if mode == "record-released" {

+ c := w.claim(t).Claim

+ report := moves.StoppedReport{V: 1, Role: "target", Box: "source", Host: w.host, OpID: c.Op.ID, Incarnation: c.Holder.Incarnation, CheckedAt: w.now, Steps: map[string]moves.Check{moves.StepTree: {OK: true}, moves.StepStray: {OK: true}, moves.StepFDs: {OK: true}, moves.StepLease: {OK: true}, moves.StepReplica: {OK: true}}}

+ body, _ := json.Marshal(report)

+ if _, err := w.raw.Put(t.Context(), moves.StoppedKey(moves.ReportPrefix("bab", "app", w.host, c.Op.ID), "target"), body, ""); err != nil {

+ t.Fatal(err)

+ }

+ authority := &plan12BumpFence{c7ReplayAuthority: w.authority}

+ if mode == "changed-sequence" {

+ authority.bump = func() {

+ r, e := ReadRecord(t.Context(), w.raw, w.key)

+ if e != nil {

+ t.Fatal(e)

+ }

+ if _, _, e = provision.WriteRecord(provision.WithRecordSequence(t.Context()), w.raw, w.key, r.Body, r.Body, r.ETag); e != nil {

+ t.Fatal(e)

+ }

+ ref := w.ref

+ if mode == "wrong-ref" {

+ ref.SHA256 = strings.Repeat("e", 64)

+ }

+ var escaped *MoveRecoveryGrant

+ run := func() error {

+ return w.control.RunRecovery(t.Context(), func(ctx context.Context) error {

+ return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error {

+ if mode == "escaped" {

+ escaped = g

+ return nil

+ }

+ return g.ReleaseFencedTarget(ctx, ref)

+ }, authority)

+ })

+ }

+ err := run()

+ if mode == "escaped" {

+ err = escaped.ReleaseFencedTarget(t.Context(), ref)

+ }

+ if mode != "ok" && mode != "admitted" && mode != "installed" && mode != "from-landing" && mode != "from-restoring" && mode != "record-released" {

+ if err == nil {

+ t.Fatal("accepted", mode)

+ }

+ if mode == "changed-sequence" && !errors.Is(err, release.ErrCASConflict) {

+ t.Fatal(err)

+ }

+ return

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ if mode == "from-landing" || mode == "from-restoring" {

+ if mode == "from-landing" && w.claim(t).Claim.Op.Kind != hostclaim.KindLanding {

+ t.Fatal("record-first pass changed claim")

+ }

+ if err := run(); err != nil {

+ t.Fatal("claim pass", err)

+ }

+ o := w.claim(t)

+ if o.Claim.Op.Phase != hostclaim.PhaseReleased || len(o.Claim.Op.Participants) != 1 || len(o.Claim.Obligations) != func() int {

+ if mode == "installed" {

+ return 3

+ }

+ return 2

+ }() {

+ t.Fatal(o.Claim)

+ }

+ if len(o.Claim.Lost) != 1 || len(o.Claim.LossRefs) != 1 || len(o.Claim.EvidenceRefs) != 1 || o.Claim.EvidenceRefs[0].Reported {

+ t.Fatal("missing pending automatic loss", o.Claim)

+ }

+ if mode == "admitted" || mode == "installed" {

+ l := o.Claim.EvidenceRefs[0]

+ if l.Session != 7 || l.Generation != "target-gen" || l.LastAdmission == nil || l.LastAdmission.Covered {

+ t.Fatal("lost writer provenance", l)

+ }

+ // A retired target or witness no longer validates against today's roster.

+ authority.reject = true

+ before, _ := json.Marshal(o.Claim.Obligations)

+ if err := run(); err != nil {

+ t.Fatal("replay", err)

+ }

+ after, _ := json.Marshal(w.claim(t).Claim.Obligations)

+ if string(before) != string(after) {

+ t.Fatal("requeued parks")

+ }

+ rec, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host)

+ if err != nil || rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || (mode != "record-released" && rec.Move.Rollback.Evidence != ref.Key) {

+ t.Fatal(rec, err)

+ }

+ if err := run(); err != nil {

+ t.Fatal("completed replay", err)

+ }

+ if mode == "from-restoring" && rec.Move.Rollback.Reason != provision.RollbackLandingDeadline {

+ t.Fatal("rewrote first reason")

+ }

+ act := func(fn func(context.Context, *MoveRecoveryGrant) error) error {

+ return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), fn, authority) })

+ }

+ if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }); err == nil {

+ t.Fatal("stamped with parks owed")

+ }

+ current := w.claim(t)

+ for _, ob := range current.Claim.Obligations {

+ var e error

+ current, e = hostclaim.Update(t.Context(), w.raw, current, hostclaim.CompleteObligation(ob.Op, ob.ID))

+ if e != nil {

+ t.Fatal(e)

+ }

+ if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }); err != nil {

+ t.Fatal("stamp", err)

+ }

+ _, stampTag, err := provision.GetRecordETag(t.Context(), w.raw, "bab", "app", w.host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ claimTag := w.claim(t).ETag

+ if err := run(); err != nil {

+ t.Fatal("release replay after parked stamp", err)

+ }

+ _, replayTag, err := provision.GetRecordETag(t.Context(), w.raw, "bab", "app", w.host)

+ if err != nil || replayTag != stampTag || w.claim(t).ETag != claimTag {

+ t.Fatal("parked replay wrote state", err)

+ }

+ // Source opening is owned by the source; seed its completed evidence.

+ current = w.claim(t)

+ if _, err := hostclaim.Update(t.Context(), w.raw, current, func(c *hostclaim.Claim) error {

+ c.Op.Phase = hostclaim.PhaseOpened

+ c.Op.Exec = c.Op.Intent["source"]

+ return nil

+ }); err != nil {

+ t.Fatal(err)

+ }

+ rr, e := ReadRecord(t.Context(), w.raw, w.key)

+ if e != nil {

+ t.Fatal(e)

+ }

+ decoded, e := provision.DecodeRecord(rr.Body)

+ if e != nil {

+ t.Fatal(e)

+ }

+ rec = &decoded

+ rec.Move.Rollback.OpenedAt, rec.Move.Rollback.RecoveredAt = &w.now, &w.now

+ body, _ := json.Marshal(rec)

+ if _, _, e = provision.WriteRecord(provision.WithRecordSequence(t.Context()), w.raw, w.key, rr.Body, body, rr.ETag); e != nil {

+ t.Fatal(e)

+ }

+ if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error {

+ return g.Record(ctx, RecoveryMoveRolledBackClose)

+ }); err != nil {

+ t.Fatal("close", err)

+ }

+ if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }); err != nil {

+ t.Fatal("release", err)

+ }

+ afterClose := w.claim(t).Claim

+ if afterClose.Op.Kind != hostclaim.KindServing || len(afterClose.EvidenceRefs) != 1 || len(afterClose.LossRefs) != 1 {

+ t.Fatal("loss evidence erased", afterClose)

+ }

+ // Reporting works after operation replacement and needs no live roster.

+ pub, priv, e := ed25519.GenerateKey(nil)

+ if e != nil {

+ t.Fatal(e)

+ }

+ w.control.pub = pub

+ if err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error {

+ return w.control.ReportAutomaticLosses(ctx, release.RootSigner(priv), w.host)

+ }); err != nil {

+ t.Fatal("report", err)

+ }

+ l := w.claim(t).Claim.EvidenceRefs[0]

+ reported, e := hostclaim.ReadLossRecord(t.Context(), w.raw, pub, l.ID)

+ if e != nil || reported.Record.State != "committed" || !reported.Record.AlertsPending || reported.Record.Automatic == nil || reported.Record.Automatic.Fence != ref || !l.Reported {

+ t.Fatal("durable reporting", reported, e)

+ }

+ if err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error {

+ return w.control.ReportAutomaticLosses(ctx, release.RootSigner(priv), w.host)

+ }); err != nil {

+ t.Fatal("report replay", err)

+ }

+

+ })

+ }

+}

diff --git a/internal/hostops/plan12_loss_test.go b/internal/hostops/plan12_loss_test.go

new file mode 100644

index 000000000..a0242c85f

--- /dev/null

+++ b/internal/hostops/plan12_loss_test.go

@@ -0,0 +1,42 @@

+package hostops

+

+import (

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "testing"

+)

+

+func TestPlan12LossOwnerPolicy(t *testing.T) {

+ for _, bad := range []string{"", "stamp", "executor", "rollback", "coverage", "intent"} {

+ in := moveLossIntent{Kind: moveLossIntentKind, Move: "move", Incarnation: "inc", Rollback: "rb_abcdefghijklmnopqrstuvwxyz", Target: "target", Source: "source", TargetRegion: "uk", SourceRegion: "ie", RecordKey: "record", RecordETag: "etag", RecordState: moveLossRecordLanding}

+ raw, _ := in.encode()

+ r := hostclaim.LossRecord{ID: "loss_1", OpID: "move", Box: "target", HolderIncarnation: "inc", ConfirmedConsequences: hostclaim.LossConsequences{RecoveryIntent: raw}}

+ c := hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc", Key: "record", Stamp: 7}, Op: hostclaim.Op{Kind: hostclaim.KindRestoring, ID: "move", Phase: hostclaim.PhaseProbing, Exec: "source", Intent: map[string]string{"source": "source", "target": "target", "to": "uk", "rollback": in.Rollback}, Journal: map[string]string{hostclaim.JournalSourceStamp: "7", "lost_target_released": "loss_1"}}}

+ a := hostclaim.Admission{Box: "source", Region: "ie", Mode: "cold"}

+ switch bad {

+ case "stamp":

+ c.Holder.Stamp++

+ case "executor":

+ c.Op.Exec = "target"

+ case "rollback":

+ c.Op.Intent["rollback"] = "other"

+ case "coverage":

+ a.Coverage = "clean-return"

+ case "intent":

+ r.ConfirmedConsequences.RecoveryIntent = []byte(`{}`)

+ }

+ err := MoveLossRecoveryOwner().ValidateLossRecovery(c, r, a)

+ if (err == nil) != (bad == "") {

+ t.Fatal(bad, err)

+ }

+ for _, s := range []string{"replica:ie/gen", "seal:move"} {

+ if _, _, _, err := ParseMoveLossBaseline(s); err != nil {

+ t.Fatal(s, err)

+ }

+ for _, s := range []string{"replica:ie/", "seal:", "replica:ie/gen/other"} {

+ if _, _, _, err := ParseMoveLossBaseline(s); err == nil {

+ t.Fatal(s)

+ }

+}

diff --git a/internal/hostops/recovery.go b/internal/hostops/recovery.go

index f8e8e0f4e..11a5cc2e1 100644

--- a/internal/hostops/recovery.go

+++ b/internal/hostops/recovery.go

@@ -36,8 +36,9 @@ func ListClaims(ctx context.Context, s release.Store) ([]string, error) {

}

type Recovery struct {

- MoveFenceAuthority hostclaim.FenceAuthority

- Moves func(context.Context, *MoveRecoveryGrant) error

+ ReportAutomaticLosses func(context.Context, string) error

+ MoveFenceAuthority hostclaim.FenceAuthority

+ Moves func(context.Context, *MoveRecoveryGrant) error

// MoveLogs, when set, runs once a pass, after every claim: the console's

// delivery of the move log entries its move pass queued, through the

// pass's MoveLogWriter (move_log_writer.go).

@@ -76,6 +77,17 @@ func (r Recovery) Pass(ctx context.Context) error {

if obs.Absent {

continue

}

+ if r.ReportAutomaticLosses != nil && len(obs.Claim.EvidenceRefs) > 0 {

+ if err := r.ReportAutomaticLosses(ctx, host); err != nil {

+ all = append(all, err)

+ continue

+ }

+ obs, err = hostclaim.Read(ctx, c.Store, host)

+ if err != nil {

+ all = append(all, err)

+ continue

+ }

op := obs.Claim.Op

// Zero time is unknown age, never old enough. Freeing empty parking is safe

// independently of age; this includes Plan 4's holderless sweep retries.

diff --git a/internal/park/remote.go b/internal/park/remote.go

index 549a49c24..4d033959c 100644

--- a/internal/park/remote.go

+++ b/internal/park/remote.go

@@ -385,7 +385,18 @@ func (e Engine) selected(ctx context.Context, w Work, o hostclaim.Obligation, he

// Run performs one resumable event. ErrPending is a scheduler retry, never an

// acknowledgement. A newly rotated event is persisted before returning it.

-func (e Engine) Run(ctx context.Context, w Work) error {

+func (e Engine) Run(ctx context.Context, w Work) error { return e.run(ctx, w, nil) }

+

+// RunHeld borrows an exact location lease. The caller retains its renewal and

+// release duties through publication. Only the configured lease resolver may

+// borrow it; the same authorizer, census and engine effects still run.

+func (e Engine) RunHeld(ctx context.Context, w Work, held HeldLease) error {

+ if held == nil {

+ return ErrUnsafe

+ }

+ return e.run(ctx, w, held)

+}

+func (e Engine) run(ctx context.Context, w Work, own HeldLease) error {

if e.Store == nil || e.Claims == nil || e.Leases == nil || e.Now == nil || e.NewID == nil || e.View == nil || e.Authorize == nil || e.Bucket == "" || e.Region == "" {

return ErrUnsafe

}

@@ -406,10 +417,24 @@ func (e Engine) Run(ctx context.Context, w Work) error {

if o.Region != e.Region {

return ErrUnsafe

}

- held, err := e.Leases.Acquire(ctx, e.Bucket, o.Park.Location)

+ var held HeldLease

+ if own == nil {

+ held, err = e.Leases.Acquire(ctx, e.Bucket, o.Park.Location)

+ } else {

+ resolver, ok := e.Leases.(interface {

+ AcquireHeld(context.Context, string, string, HeldLease) (HeldLease, error)

+ })

+ if !ok {

+ return ErrUnsafe

+ }

+ held, err = resolver.AcquireHeld(ctx, e.Bucket, o.Park.Location, own)

+ }

if err != nil {

return err

}

+ if held == nil {

+ return ErrUnsafe

+ }

defer held.Close()

if bound, ok := held.(interface {

Context(context.Context) context.Context

diff --git a/internal/park/sweep.go b/internal/park/sweep.go

index 92ceb109b..8eabf28df 100644

--- a/internal/park/sweep.go

+++ b/internal/park/sweep.go

@@ -18,6 +18,10 @@ import (

// Pinned includes the exact noncurrent versions needed by the installed root.

// MarkerKnown means its read succeeded; an absent marker may denote @root.

type View struct {

+ // ArchivedIncarnations labels proven retired locations independently of

+ // the moves-sweep proof's incarnation. Missing entries never invent labels.

+ ArchivedIncarnations map[string]string

+ RefusedLocations map[string]error

ClaimETag, Region, AuthoritativeRegion string

MarkerKnown bool

MarkerGeneration string

@@ -156,11 +160,18 @@ func Decide(obs hostclaim.Observed, v View, versions []lease.ObjectVersion, lega

if isLock(c.Host, loc, x.Key) {

continue

}

+ if refusal := v.RefusedLocations[loc]; refusal != nil && !(c.Holder != nil && loc == c.Holder.Location && v.Region == v.AuthoritativeRegion) {

+ d.Alerts = append(d.Alerts, Alert{loc, refusal.Error()})

+ continue

+ }

candidate := Candidate{Location: loc}

if tail := strings.TrimPrefix(loc, "instances/"+c.Host+"/"); tail != loc {

candidate.Incarnation = tail

} else {

candidate.Incarnation = legacyIncarnation

+ if loc == "instances/"+c.Host && v.ArchivedIncarnations[loc] != "" {

+ candidate.Incarnation = v.ArchivedIncarnations[loc]

+ }

}

if c.Holder != nil && loc == c.Holder.Location && v.Region == v.AuthoritativeRegion {

if liveErr != nil {

@@ -417,3 +428,23 @@ func (e Engine) sweep(ctx context.Context, expected hostclaim.Observed, expected

}

return d, nil

}

+

+// ResidueLocations uses the same namespace and lock classifier as Decide.

+func ResidueLocations(host string, versions []lease.ObjectVersion) ([]string, error) {

+ seen := map[string]bool{}

+ for _, v := range versions {

+ loc, err := locationOf(host, v.Key)

+ if err != nil || v.VersionID == "" {

+ return nil, ErrUnsafe

+ }

+ if !isLock(host, loc, v.Key) {

+ seen[loc] = true

+ }

+ out := make([]string, 0, len(seen))

+ for loc := range seen {

+ out = append(out, loc)

+ }

+ sort.Strings(out)

+ return out, nil

+}

diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go

index 505b6d9b4..601b60f26 100644

--- a/internal/provision/plan9_b8_census_test.go

+++ b/internal/provision/plan9_b8_census_test.go

@@ -472,12 +472,21 @@ func Reviewed(ctx context.Context, s Store, k string) error {

// BEGIN REVIEWED INVENTORY

var plan9B8CensusInventory = []censusEntry{

+ {"internal/hostbox/parkleases.go", "ReplicaParkLeaseSet", "key-site", "lit:instances", "0ca8d9b743402fda9d15cc215464423b35e21c1c2b47f91294d76188b7d43fb6"},

+ {"internal/host/claimretirementcensus.go", "ClaimRetirementCensus", "key-site", "lit:instances", "abc0bd8c5f5d328abfac8821f42f8f23954723ed6506747f33b0197824e2611f"},

+ {"internal/host/claimretirementcensus.go", "productionClaimRegional.retirement", "key-site", "lit:instances", "19306389daacc704a94cfb434b7d557ced96abe09958a01be38a0707b3ea0a6f"},

+ {"internal/host/claimproduction.go", "productionClaimRegional.View", "key-site", "lit:instances", "61ff9ff47054b6a96cb24467c7aaf41c112b7c5129987af5ce27e0b7f862348e"},

+ {"internal/hostops/move_loss_admission.go", "moveLossOwner.ValidateLossRecovery", "key-site", "key:RecordKey", "ea2aa35a069ffdc846ecb208c33940245fc865795188818e86ea7a93135dd6a9"},

+ {"internal/host/claimretirementcensus.go", "claimArchivedLocations", "key-site", "lit:instances", "9ba8ea4ecb5ad63621c1322467b9a80f8f5703896ba1fc3f7cb1addd56418950"},

+ {"internal/host/claimretirementcensus.go", "claimLegacyIncarnation", "key-site", "lit:instances", "31a14ea105c67e00a24f06b9500a221ee5bfc47402b9c33cecfde807e51db530"},

+ {"internal/hostops/move_fence_release.go", "MoveRecoveryGrant.ReleaseFencedTarget", "record-writer", "key:WriteRecord,key:writeRecord", "9af3294f6fcd6c00266eca06a5002b5dd904342938aeae7da155d025f23677d7"},

+ {"internal/hostops/move_fence_release.go", "MoveRecoveryGrant.StampLostMoveTargetParked", "record-writer", "key:WriteRecord", "b082db5634a2f4d9a5b45cc4a9cb725964e9bb24b7aeb53a6346167d7737227c"},

{"internal/hostops/move_recovery_grant.go", "MoveRecoveryGrant.Convert", "record-writer", "key:WriteRecord", "794963a24a0a50d980505f83ac912973a10267b21f74bd41c56584ab9b0b25a1"},

// The recovery pass's move log writer: one create-only entry of a move's

// log under its report prefix, for a host whose claim still names the

// app; never a record (Plan 6 Task 3 fix round 3, ruling 1).

{"internal/hostops/move_log_writer.go", "MoveLogWriter.WriteMoveLog", "nonrecord", "sel:Put", "2241e022d7c13c03ad3011ef8175ec5308b9fdfbcd3452589269ee47003a4e7f"},

- {"internal/hostops/move_recovery_grant.go", "MoveRecoveryGrant.Record", "record-writer", "key:WriteRecord", "f7f04a9e2c579eaa7d254ce6310e2726d95855307d61f3f442ee9c1fa2a5d825"},

+ {"internal/hostops/move_recovery_grant.go", "MoveRecoveryGrant.Record", "record-writer", "key:WriteRecord", "5f7e2b19846e5994e54e92592ec2b713c18d8b7c43353ad2f3a0b7c864ab3b38"},

// Plan 10: reviewed typed console move and loss writers.

{"internal/hostops/console_move_grant.go", "ControlWriter.ConsoleMove", "record-writer", "key:WriteRecord", "21e7a6ddc4dbe5b3bf06e65b39a228e2eb588af1edc43beb09d91dd72570dd44"},

{"internal/hostops/console_move_grant.go", "ControlWriter.consoleMoveClaim", "key-site", "key:RecordKey", "265291bd356de061663d89e5ebe33b74ea32c086f8238736d6f971a5afcab722"},

@@ -714,7 +723,7 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/hostclaim/adopt.go", "Adopt", "key-site", "lit:instances", "24c06b99cce6c638acb19826985cfadd3c736fd0ace7289bcf7ac3dfe784f05a"},

{"internal/hostclaim/claim.go", "ParkScope.Validate", "key-site", "lit:instances", "80b798b0cb679d8c533ff562c0c4804c06f53e7dba126250a3c8182f7ff0c08c"},

{"internal/hostclaim/claim.go", "holderKeyMatches", "key-site", "lit:instances,lit:sidecars", "d15a6841ea6f8cbfa9c8407fd3af020ddeacdfa0743b9e17de412f113faf7979"},

- {"internal/hostclaim/claim.go", "Claim.Validate", "key-site", "lit:instances", "ddcde93d00c6fe420c60a30f561928580338617061a0bdd911de2db386e00fa9"},

+ {"internal/hostclaim/claim.go", "Claim.Validate", "key-site", "lit:instances", "3278f2b4b24b6627b9dcd0863c3bef58c6694a65c8a19d05ce280ae34801bdfc"},

{"internal/hostclaim/create.go", "BeginCreate", "key-site", "lit:instances", "45ee1e2b1fc9fa8ffdef4f27cf983dccfa3acf7218dccd06f0ee6d12572f4a44"},

{"internal/hostclaim/lossrecord.go", "LossRecordKey", "key-site", "key:LossRecordKey", "1b109b49a8fcdb09f01dbf908a173394072f01afb0c9e324023fb9b3f71f98b3"},

{"internal/hostclaim/lossrecord.go", "ReadLossRecord", "key-site", "key:LossRecordKey", "1237dd3fa3e79a81e6c35f371bf545c9a04841a6297c078d96852daf25b7b483"},

@@ -839,8 +848,8 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/objstore/sidecar.go", "claimRawPutSidecar", "nonrecord", "sel:Put", "233e3d808dd462ff83aecdac5176b180556b948c552aa26f6e0b778df91a24c7"},

{"internal/ops/litestream.go", "FleetDBsWithSkipped", "key-site", "lit:instances", "f62291df0685db15529c4946dca358a29aba66fc31e81aab299de579e37f60f1"},

{"internal/park/remote.go", "locationOf", "key-site", "lit:instances", "83b3efca31b3faadcd5997dc181d81b7bb8fc866cc4a448683bf8423b1bea866"},

- {"internal/park/remote.go", "Engine.Run", "versioned-transport", "sel:CopyVersion,sel:PutIfAbsent", "dd4b60ef9b3f5e186b9866cee818f2684ed93bd4e1c934cd66d1af9ef376d81e"},

- {"internal/park/sweep.go", "Decide", "key-site", "lit:instances", "c4ec278026ad3b99ed088471bf32a2b2787b19b948587b34f784e97a31a29f1d"},

+ {"internal/park/remote.go", "Engine.run", "versioned-transport", "sel:CopyVersion,sel:PutIfAbsent", "3de5b215e4b7aba67768243891b3f6d8ff8c77a465aefe5168c789cb012b1ea9"},

+ {"internal/park/sweep.go", "Decide", "key-site", "lit:instances", "334144f96a8b19448790578370923c290b1ef50142e8ffc7a7f5933281f7e057"},

{"internal/park/sweep.go", "Engine.Discover", "key-site", "lit:instances", "69b83e6e859c1c9809ba76d46cb0192888df1ecdcd161b4f2b6435733eecdfa3"},

{"internal/provision/claim_app_writes.go", "UpdateRecord", "record-writer", "key:UpdateRecord,key:claimRawUpdateRecord", "caf714cb78e1280d489c7ad5b3cefb1420e812d541108b1fe13d733abed97229"},

{"internal/provision/provision.go", "ManifestKey", "key-site", "lit:instances", "b8ee0e77bdfade19af79a4012c411c1eb0b6ddca16037e5536fa8597f7cde270"},

diff --git a/internal/provision/provision.go b/internal/provision/provision.go

index bd948a634..c40e65e48 100644

--- a/internal/provision/provision.go

+++ b/internal/provision/provision.go

@@ -1879,6 +1879,9 @@ func MoveClose(g MoveGuard, now time.Time) func(*Record) error {

}

+// RollbackTargetFenced is the recorded reason for a verified target fence.

+const RollbackTargetFenced = "target fenced"

+

// RollbackDeclaredLost is the rollback reason the declare-lost door

// records (§7.10): the target box was declared lost before the move

// committed. A recorded sentence, so it names no box; the box is in the

@@ -2013,6 +2016,8 @@ func MoveRollBack(g MoveGuard, rbID, reason string, now time.Time) func(*Record)

rb.Reason = RollbackCommitDeadline

case RollbackProbeUnpriced:

rb.Reason = RollbackProbeUnpriced

+ case RollbackTargetFenced:

+ rb.Reason = RollbackTargetFenced

case RollbackDeclaredLost:

rb.Reason = RollbackDeclaredLost

default:

@@ -2032,7 +2037,7 @@ func MoveRollBack(g MoveGuard, rbID, reason string, now time.Time) func(*Record)

func recordedRollbackReason(r string) bool {

switch r {

case RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch,

- RollbackFilesOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost, RollbackProbeUnpriced:

+ RollbackFilesOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost, RollbackTargetFenced, RollbackProbeUnpriced:

return true

}

return false

diff --git a/internal/registry/plan12_recovery_test.go b/internal/registry/plan12_recovery_test.go

new file mode 100644

index 000000000..d31a4ad29

--- /dev/null

+++ b/internal/registry/plan12_recovery_test.go

@@ -0,0 +1,109 @@

+package registry

+

+import (

+ "errors"

+ "path/filepath"

+ "testing"

+)

+

+func TestPlan12RebindClaimRows(t *testing.T) {

+ for _, mode := range []string{"ok", "changed-route", "changed-owner", "changed-payload", "extra-timer"} {

+ t.Run(mode, func(t *testing.T) {

+ r, e := Open(filepath.Join(t.TempDir(), "registry.db"))

+ if e != nil {

+ t.Fatal(e)

+ }

+ defer r.Close()

+ row := Route{Host: "app.example", Kind: "instance", ClaimOwnership: ClaimOwnership{"inc", 1, "old", "old-row"}}

+ if e = r.Upsert(row); e != nil {

+ t.Fatal(e)

+ }

+ saved, e := r.Get(row.Host)

+ if e != nil {

+ t.Fatal(e)

+ }

+ old := ColdRows{Route: *saved}

+ next := old

+ next.Route.ClaimOwnership = ClaimOwnership{"inc", 2, "new", "new-row"}

+ switch mode {

+ case "changed-route":

+ r.sql.Exec("UPDATE routes SET version='other' WHERE host=?", row.Host)

+ case "changed-owner":

+ r.sql.Exec("UPDATE routes SET claim_owner='other' WHERE host=?", row.Host)

+ case "changed-payload":

+ next.Route.Version = "other"

+ case "extra-timer":

+ if e = r.UpsertTimer(Timer{Host: row.Host, Name: "extra", Kind: "cron", Spec: "* * * * *"}); e != nil {

+ t.Fatal(e)

+ }

+ e = r.RebindClaimRows(t.Context(), old, next)

+ if mode != "ok" {

+ if !errors.Is(e, ErrClaimOwnership) {

+ t.Fatal("accepted changed installation", e)

+ }

+ return

+ }

+ if e != nil {

+ t.Fatal(e)

+ }

+ if e = r.RebindClaimRows(t.Context(), old, next); e != nil {

+ t.Fatal("replay", e)

+ }

+ got, e := r.Get(row.Host)

+ if e != nil || *got != next.Route {

+ t.Fatal(got, e)

+ }

+ })

+ }

+}

+

+func TestPlan12RebindRefusesForeignTimerOwnership(t *testing.T) {

+ for _, mode := range []string{"foreign-owner", "stale-incarnation", "stale-stamp"} {

+ t.Run(mode, func(t *testing.T) {

+ r, err := Open(filepath.Join(t.TempDir(), "registry.db"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ defer r.Close()

+ row := Route{Host: "app.example", Kind: "instance", ClaimOwnership: ClaimOwnership{"inc", 1, "old", "row"}}

+ if err = r.Upsert(row); err != nil {

+ t.Fatal(err)

+ }

+ saved, err := r.Get(row.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ timer := Timer{Host: row.Host, Name: "extra", Kind: "cron", Spec: "* * * * *", ClaimOwnership: ClaimOwnership{"inc", 1, "old", "timer"}}

+ switch mode {

+ case "foreign-owner":

+ timer.ClaimOwner = "foreign"

+ case "stale-incarnation":

+ timer.ClaimIncarnation = "stale"

+ case "stale-stamp":

+ timer.ClaimStamp = 9

+ }

+ if err = r.UpsertTimer(timer); err != nil {

+ t.Fatal(err)

+ }

+ if _, err = r.sql.Exec("UPDATE timers SET claim_incarnation=?,claim_stamp=?,claim_owner=?,claim_row_id=? WHERE host=?", timer.ClaimIncarnation, timer.ClaimStamp, timer.ClaimOwner, timer.ClaimRowID, row.Host); err != nil {

+ t.Fatal(err)

+ }

+ timers, err := r.TimersFor(row.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ old := ColdRows{Route: *saved, Timers: timers}

+ next := ColdRows{Route: *saved, Timers: append([]Timer(nil), timers...)}

+ next.Route.ClaimOwnership = ClaimOwnership{"inc", 2, "new", "new-row"}

+ next.Timers[0].ClaimOwnership = ClaimOwnership{"inc", 2, "new", "new-timer"}

+ if err = r.RebindClaimRows(t.Context(), old, next); !errors.Is(err, ErrClaimOwnership) {

+ t.Fatalf("stole timer: %v", err)

+ }

+ got, err := r.Get(row.Host)

+ if err != nil || *got != *saved {

+ t.Fatal("refusal changed route", err)

+ }

+ })

+ }

+}

diff --git a/internal/registry/recoveryhandover.go b/internal/registry/recoveryhandover.go

new file mode 100644

index 000000000..cacc7cd42

--- /dev/null

+++ b/internal/registry/recoveryhandover.go

@@ -0,0 +1,129 @@

+package registry

+

+import (

+ "context"

+ "database/sql"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "reflect"

+)

+

+// RebindClaimRows transfers an exact stopped installation's registry rows.

+// The caller holds hostguard and proves the source cut and successor admission.

+// A transaction preserves row contents and refuses any concurrent replacement.

+func (r *Registry) RebindClaimRows(ctx context.Context, old, next ColdRows) error {

+ if r.remote != nil {

+ _, err := r.remote.call(ctx, remoteRequest{Method: "RebindClaimRows", Old: old, NextValue: next}, nil)

+ return err

+ }

+ if _, err := r.sql.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS recovery_rows(id TEXT PRIMARY KEY,body BLOB NOT NULL)`); err != nil {

+ return err

+ }

+ tx, err := r.beginClaimTransaction(ctx, true)

+ if err != nil {

+ return err

+ }

+ defer tx.Rollback()

+ id := fmt.Sprintf("%s/%s/%d", next.Route.Host, next.Route.ClaimIncarnation, next.Route.ClaimStamp)

+ var saved []byte

+ err = tx.QueryRowContext(ctx, `SELECT body FROM recovery_rows WHERE id=?`, id).Scan(&saved)

+ recorded := err == nil

+ if recorded {

+ var rows struct{ Old, Next ColdRows }

+ if json.Unmarshal(saved, &rows) != nil || rows.Old.Route != old.Route || !reflect.DeepEqual(rows.Next, next) {

+ return ErrClaimOwnership

+ }

+ old = rows.Old

+ } else if !errors.Is(err, sql.ErrNoRows) {

+ return err

+ }

+ if !old.Route.ClaimOwnership.Valid() || !next.Route.ClaimOwnership.Valid() || old.Route.Host != next.Route.Host || old.Route.ClaimIncarnation != next.Route.ClaimIncarnation || next.Route.ClaimStamp <= old.Route.ClaimStamp || len(old.Timers) != len(next.Timers) {

+ return ErrClaimOwnership

+ }

+ a, b := old.Route, next.Route

+ a.ClaimOwnership, b.ClaimOwnership = ClaimOwnership{}, ClaimOwnership{}

+ if a != b {

+ return ErrClaimOwnership

+ }

+ for i, t := range next.Timers {

+ prior := old.Timers[i]

+ if !prior.ClaimOwnership.Valid() || prior.Host != old.Route.Host || prior.ClaimIncarnation != old.Route.ClaimIncarnation || prior.ClaimStamp != old.Route.ClaimStamp || prior.ClaimOwner != old.Route.ClaimOwner {

+ return ErrClaimOwnership

+ }

+ a, b := prior, t

+ a.ClaimOwnership, b.ClaimOwnership = ClaimOwnership{}, ClaimOwnership{}

+ if a != b || !t.ClaimOwnership.Valid() || t.Host != next.Route.Host || t.ClaimIncarnation != next.Route.ClaimIncarnation || t.ClaimStamp != next.Route.ClaimStamp || t.ClaimOwner != next.Route.ClaimOwner {

+ return ErrClaimOwnership

+ }

+ row, err := scanRoute(tx.QueryRowContext(ctx, `SELECT `+routeCols+` FROM routes WHERE host=?`, old.Route.Host).Scan)

+ if err != nil {

+ return err

+ }

+ replay := row == next.Route

+ if !replay && row != old.Route {

+ return ErrClaimOwnership

+ }

+ rows, err := tx.conn.QueryContext(ctx, `SELECT `+timerCols+` FROM timers WHERE host=? ORDER BY name`, old.Route.Host)

+ if err != nil {

+ return err

+ }

+ var timers []Timer

+ for rows.Next() {

+ t, e := scanTimer(rows)

+ if e != nil {

+ rows.Close()

+ return e

+ }

+ timers = append(timers, t)

+ }

+ err = rows.Err()

+ rows.Close()

+ if err != nil {

+ return err

+ }

+ want := old.Timers

+ if replay {

+ want = next.Timers

+ }

+ if len(timers) != len(want) {

+ return ErrClaimOwnership

+ }

+ for _, t := range timers {

+ found := false

+ for _, w := range want {

+ if reflect.DeepEqual(t, w) {

+ found = true

+ break

+ }

+ if !found {

+ return ErrClaimOwnership

+ }

+ if replay {

+ if !recorded {

+ return ErrClaimOwnership

+ }

+ return nil

+ }

+ o := next.Route.ClaimOwnership

+ if _, err = tx.ExecContext(ctx, `UPDATE routes SET claim_incarnation=?,claim_stamp=?,claim_owner=?,claim_row_id=? WHERE host=?`, o.ClaimIncarnation, o.ClaimStamp, o.ClaimOwner, o.ClaimRowID, next.Route.Host); err != nil {

+ return err

+ }

+ for _, t := range next.Timers {

+ o := t.ClaimOwnership

+ if _, err = tx.ExecContext(ctx, `UPDATE timers SET claim_incarnation=?,claim_stamp=?,claim_owner=?,claim_row_id=? WHERE host=? AND name=?`, o.ClaimIncarnation, o.ClaimStamp, o.ClaimOwner, o.ClaimRowID, t.Host, t.Name); err != nil {

+ return err

+ }

+ body, err := json.Marshal(struct{ Old, Next ColdRows }{old, next})

+ if err != nil {

+ return err

+ }

+ if _, err = tx.ExecContext(ctx, `INSERT INTO recovery_rows(id,body) VALUES(?,?)`, id, body); err != nil {

+ return err

+ }

+ return tx.Commit()

+}

diff --git a/internal/registry/recoveryjournal.go b/internal/registry/recoveryjournal.go

new file mode 100644

index 000000000..70501d3f7

--- /dev/null

+++ b/internal/registry/recoveryjournal.go

@@ -0,0 +1,82 @@

+package registry

+

+import (

+ "context"

+ "database/sql"

+ "errors"

+)

+

+// Recovery handovers remain pending until the successor's prelaunch is durable.

+// The existing isolated registry transport owns these writes, including at root.

+func (r *Registry) BeginRecoveryHandover(ctx context.Context, id, body string) error {

+ if r.remote != nil {

+ _, err := r.remote.call(ctx, remoteRequest{Method: "BeginRecoveryHandover", Id: id, Journal: body}, nil)

+ return err

+ }

+ if id == "" || body == "" {

+ return ErrClaimOwnership

+ }

+ if err := r.recoveryJournal(ctx); err != nil {

+ return err

+ }

+ tx, err := r.beginClaimTransaction(ctx, true)

+ if err != nil {

+ return err

+ }

+ defer tx.Rollback()

+ var saved string

+ err = tx.QueryRowContext(ctx, `SELECT body FROM recovery_handovers WHERE id=?`, id).Scan(&saved)

+ if err == nil {

+ if saved != body {

+ return ErrClaimOwnership

+ }

+ return nil

+ }

+ if !errors.Is(err, sql.ErrNoRows) {

+ return err

+ }

+ if _, err = tx.ExecContext(ctx, `INSERT INTO recovery_handovers(id,body,complete) VALUES(?,?,0)`, id, body); err != nil {

+ return err

+ }

+ return tx.Commit()

+}

+func (r *Registry) PendingRecoveryHandover(ctx context.Context, id string) (string, error) {

+ if r.remote != nil {

+ out, err := r.remote.call(ctx, remoteRequest{Method: "PendingRecoveryHandover", Id: id}, nil)

+ return out.Text, err

+ }

+ if err := r.recoveryJournal(ctx); err != nil {

+ return "", err

+ }

+ var body string

+ err := r.sql.QueryRowContext(ctx, `SELECT body FROM recovery_handovers WHERE id=? AND complete=0`, id).Scan(&body)

+ if errors.Is(err, sql.ErrNoRows) {

+ return "", nil

+ }

+ return body, err

+}

+func (r *Registry) CompleteRecoveryHandover(ctx context.Context, id, body string) error {

+ if r.remote != nil {

+ _, err := r.remote.call(ctx, remoteRequest{Method: "CompleteRecoveryHandover", Id: id, Journal: body}, nil)

+ return err

+ }

+ if err := r.recoveryJournal(ctx); err != nil {

+ return err

+ }

+ result, err := r.sql.ExecContext(ctx, `UPDATE recovery_handovers SET complete=1 WHERE id=? AND body=?`, id, body)

+ if err != nil {

+ return err

+ }

+ n, err := result.RowsAffected()

+ if err != nil {

+ return err

+ }

+ if n != 1 {

+ return ErrClaimOwnership

+ }

+ return nil

+}

+func (r *Registry) recoveryJournal(ctx context.Context) error {

+ _, err := r.sql.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS recovery_handovers(id TEXT PRIMARY KEY,body TEXT NOT NULL,complete INTEGER NOT NULL)`)

+ return err

+}

diff --git a/internal/registry/remote_dispatch.go b/internal/registry/remote_dispatch.go

index 9d84778b3..00b7f40f1 100644

--- a/internal/registry/remote_dispatch.go

+++ b/internal/registry/remote_dispatch.go

@@ -134,6 +134,14 @@ func (r *Registry) dispatchRemote(ctx context.Context, q remoteRequest, callback

err = r.SetTimerStatusRegistration(q.Host, q.Name, q.Registration, q.Status)

case "FireTimerNowRegistration":

err = r.FireTimerNowRegistration(q.Host, q.Name, q.Registration, q.Now)

+ case "BeginRecoveryHandover":

+ err = r.BeginRecoveryHandover(ctx, q.Id, q.Journal)

+ case "PendingRecoveryHandover":

+ out.Text, err = r.PendingRecoveryHandover(ctx, q.Id)

+ case "CompleteRecoveryHandover":

+ err = r.CompleteRecoveryHandover(ctx, q.Id, q.Journal)

+ case "RebindClaimRows":

+ err = r.RebindClaimRows(ctx, q.Old, q.NextValue)

case "RestoreClaimRows":

err = r.RestoreClaimRows(ctx, q.Id, q.Retirement, q.Old, q.NextValue)

case "UpsertFleet":

carlos / platform Public

hostclaim: Plan 12, moves integration completion (inert behind CARLOS_HOST_CLAIM)