carlos / platform Public

Merge origin/main (Plan 5 Task 5 5c/5d, Plan 5 Task 7 record-first closes) into moves-phase1-6

The record-first closes (closeMove, recoverRun.finalize, the revert's
opened, abortSource) keep main's structure; their old logEvent calls go,
since the observed close write logs the transition (Plan 6 Task 3). The
copy gate's lists take both sides; the B8 census takes main's rows with
the four closes' digests recomputed for the log calls; moves/log.go joins
movesScanFiles; two test helpers renamed where main's tests share names.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago b24c3804e2dce4b6bc4a978fadfabd9f11503dfd parent ef267fbparent e79d71b

34 files changed, +3324 −207

approved_moves_copy_test.go +7 −0
cmd/console/main.go +16 −0
cmd/console/movesswitch_test.go +57 −0
docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +15 −0
docs/superpowers/specs/2026-10-02-move-any-app-design.md +51 −17
internal/alerting/alerting.go +32 −0
internal/alerting/alerting_test.go +33 −0
internal/console/claim_authority_census_test.go +3 −0
internal/console/claim_invariant_moves_test.go +19 −0
internal/console/claim_runtime.go +15 −0
internal/console/movesrecovery.go +516 −0
internal/console/movesrecovery_test.go +721 −0
internal/console/server.go +12 −3
internal/console/store.go +31 −0
internal/console/store_gate.go +1 −0
internal/console/testdata/claim-authority-sites.txt +8 −0
internal/host/movesabort.go +43 −25
internal/host/movesclose_order_test.go +135 −0
internal/host/movesevents_test.go +27 −27
internal/host/movesevidence.go +315 −0
internal/host/movesevidence_test.go +897 −0
internal/host/moveslandcommit.go +2 −2
internal/host/movesparks.go +71 −30
internal/host/movesparks_test.go +29 −15
internal/host/movesrecover.go +28 −22
internal/host/movesrecover_test.go +5 −10
internal/host/movesrevert.go +36 −29
internal/provision/plan9_b8_census_test.go +12 −5
internal/sim/hostlife/console_test.go +13 −2
internal/sim/hostlife/moves_test.go +80 −5
internal/sim/hostlife/rules_test.go +1 −0
internal/sim/hostlife/sim_test.go +24 −8
internal/sim/hostlife/witness_test.go +65 −7
internal/sim/hostlife/world_test.go +4 −0

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index 8b8716aa6..cca8c8d2c 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -76,6 +76,7 @@ var movesCopyIDs = []string{

"MV-N313", "MV-N314",

"MV-N315", "MV-N316", "MV-N317", "MV-N318",

"MV-N319",

+ "MV-N325", "MV-N326", "MV-N327", "MV-N328", "MV-N329", "MV-N330", "MV-N331", "MV-N332", "MV-N333", "MV-N334",

"MV-N350", "MV-N351", "MV-N352", "MV-N353", "MV-N354", "MV-N355", "MV-N356",

"MV-N357", "MV-N358", "MV-N359", "MV-N360", "MV-N361", "MV-N362", "MV-N363", "MV-N364",

"MV-N365", "MV-N366", "MV-N367", "MV-N368", "MV-N369", "MV-N370", "MV-N371",

@@ -549,6 +550,9 @@ func TestMovesTextAppliedPlaceholders(t *testing.T) {

// copy" task appends the files it touched; TestMovesScanFilesCoverTheDiff

// fails when the list falls behind.

var movesScanFiles = []string{

+ "internal/console/movesrecovery.go",

+ "internal/console/claim_runtime.go",

+ "internal/alerting/alerting.go",

"internal/hostops/move_recovery_grant.go",

"internal/hostops/move_loss_owner.go",

"internal/host/movesrecoveryprobe.go",

@@ -654,6 +658,7 @@ var movesScanFiles = []string{

"internal/moves/ids.go",

"internal/moves/journal.go",

"internal/moves/lease.go",

+ "internal/moves/log.go",

"internal/moves/ltxbuild.go",

"internal/moves/parkscope.go",

"internal/moves/probe.go",

@@ -766,6 +771,8 @@ var movesReturnedSentences = map[string]map[string]bool{

"internal/host/movesevents.go": {"memberLine": true},

// moveDeleteRefusal is M-C19, which the delete door answers (Task 2b).

"internal/console/apiinstancemove.go": {"moveDeleteRefusal": true},

+ // The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N334).

+ "internal/console/movesrecovery.go": {"moveWaitingPage": true, "movePreparingPageText": true, "moveOrphanPage": true, "moveCloseOverduePage": true, "moveAbortOverduePage": true},

}

// movesLiteralFuncs are functions whose every string literal is a sentence

diff --git a/cmd/console/main.go b/cmd/console/main.go

index ab1ef043d..f4b243819 100644

--- a/cmd/console/main.go

+++ b/cmd/console/main.go

@@ -88,6 +88,7 @@ import (

"unicode/utf8"

"github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/dnsrec"

@@ -419,6 +420,9 @@ func serve() {

if os.Getenv("CARLOS_MAIL_LEGACY_SWEEP") == "1" {

go sweepLegacyMail(cfg.HostClaim, rel, st)

}

+ // The move pass's pages (moves Plan 5 Task 7): the console's alert sinks,

+ // armed before the claim runtime binds the pass, so no pass runs unarmed.

+ armMovesPaging(context.Background(), cfg, os.Getenv)

if err := srv.ConfigureClaimRuntime(context.Background()); err != nil {

log.Fatalf("console: claim recovery: %v", err)

}

@@ -1345,6 +1349,18 @@ func applyMovesSwitch(cfg *console.Config, getenv func(string) string) {

cfg.MovesEnabled = moves.Enabled(getenv)

}

+// armMovesPaging arms alerting's sinks from CARLOS_ALERT_WEBHOOK and

+// CARLOS_ALERT_SNS_TOPIC_ARN, the agent's variables, for the console's move

+// pass (moves Plan 5 Task 7, delta M14). Only a host-claim console with the

+// moves switch on runs that pass, so only it arms; it reports whether it did.

+func armMovesPaging(ctx context.Context, cfg console.Config, getenv func(string) string) bool {

+ if !cfg.HostClaim || !cfg.MovesEnabled {

+ return false

+ }

+ alerting.Arm(alerting.FromEnv(ctx, getenv, nil))

+ return true

func hostClaimEnabled(getenv func(string) string) bool {

return strings.TrimSpace(getenv("CARLOS_HOST_CLAIM")) != ""

}

diff --git a/cmd/console/movesswitch_test.go b/cmd/console/movesswitch_test.go

index 5fc3f903d..6173a520a 100644

--- a/cmd/console/movesswitch_test.go

+++ b/cmd/console/movesswitch_test.go

@@ -1,11 +1,13 @@

package main

import (

+ "context"

"go/ast"

"go/parser"

"go/token"

"testing"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/moves"

)

@@ -98,3 +100,58 @@ func isIdent(e ast.Expr, name string) bool {

id, ok := e.(*ast.Ident)

return ok && id.Name == name

}

+// The console pages from its move pass (moves Plan 5 Task 7, delta M14):

+// armMovesPaging arms alerting's sinks from the agent's variables on a

+// host-claim console with the moves switch on, and nowhere else; serve()

+// calls it with os.Getenv before ConfigureClaimRuntime, so the recovery

+// pass never runs unarmed.

+func TestArmMovesPagingOnlyWithBothSwitches(t *testing.T) {

+ env := func(k string) string {

+ if k == "CARLOS_ALERT_WEBHOOK" {

+ return "http://example.test/hook"

+ }

+ return ""

+ }

+ for _, tc := range []struct{ claim, moves, want bool }{{false, false, false}, {true, false, false}, {false, true, false}, {true, true, true}} {

+ cfg := console.Config{HostClaim: tc.claim, MovesEnabled: tc.moves}

+ if got := armMovesPaging(context.Background(), cfg, env); got != tc.want {

+ t.Errorf("claim %v moves %v: armed %v, want %v", tc.claim, tc.moves, got, tc.want)

+ }

+ t.Cleanup(func() { alerting.Arm(nil) })

+ f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)

+ if err != nil {

+ t.Fatal(err)

+ }

+ var armed, configured token.Pos

+ for _, d := range f.Decls {

+ fn, ok := d.(*ast.FuncDecl)

+ if !ok || fn.Name.Name != "serve" {

+ continue

+ }

+ ast.Inspect(fn.Body, func(n ast.Node) bool {

+ call, ok := n.(*ast.CallExpr)

+ if !ok {

+ return true

+ }

+ switch fun := call.Fun.(type) {

+ case *ast.Ident:

+ if fun.Name == "armMovesPaging" && len(call.Args) == 3 {

+ if sel, ok := call.Args[2].(*ast.SelectorExpr); ok && isIdent(sel.X, "os") && sel.Sel.Name == "Getenv" {

+ armed = call.Pos()

+ }

+ case *ast.SelectorExpr:

+ if fun.Sel.Name == "ConfigureClaimRuntime" {

+ configured = call.Pos()

+ }

+ return true

+ })

+ }

+ if armed == 0 || configured == 0 || armed > configured {

+ t.Fatalf("serve() must call armMovesPaging(ctx, cfg, os.Getenv) before ConfigureClaimRuntime (armed %v, configured %v)", armed, configured)

+ }

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index cd800c89c..efeb5d740 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -271,6 +271,21 @@ Columns: id, file, text, status.

| MV-N314 | internal/host/movesrollback.go | The move <id> of <host> stopped for an operator on <box>, its <role>: <reason>. Run carlos move resolve --host <host> --retry once the cause is fixed. | pending |

+Task 7's pages, the console's move pass (`internal/console/movesrecovery.go`, §13). MV-N319 to MV-N324 are Plan 6 Task 2's.

+|---|---|---|---|

+| MV-N326 | internal/console/movesrecovery.go | The move <id> of <host> is rolling back, and <to> has not released it by its deadline. <host> stays offline until a box in <to> confirms it has stopped, or that box is declared lost. | pending |

+| MV-N330 | internal/console/movesrecovery.go | The host claim of <host> is held by the move <id>, and the instance record carries no move. Nothing releases it on its own. An operator must check the record. | pending |

+| MV-N334 | internal/console/movesrecovery.go | The abort of the move <id> of <host> has nothing left to park in <to>, and <box> has not closed it. <box> closes it once it has removed the move's snapshots from its own bucket. Check <box>. | pending |

## Floor gate fix (#267 review)

(root's floor gate for the system units, `carlos system floor-allows-prev`; its stderr reaches the unit's journal)

diff --git a/docs/superpowers/specs/2026-10-02-move-any-app-design.md b/docs/superpowers/specs/2026-10-02-move-any-app-design.md

index c8a59c34d..fc6d8c40e 100644

--- a/docs/superpowers/specs/2026-10-02-move-any-app-design.md

+++ b/docs/superpowers/specs/2026-10-02-move-any-app-design.md

@@ -1,8 +1,32 @@

# Move any app or instance to another region: Phase 1, a safe single-instance move

**Date:** 2026-10-06

-**Revision:** 7.22.

+**Revision:** 7.23.

+- **Rev 7.23** records Paul's ruling M4 (2026-10-10, moves Plan 5 delta

+ pre-flight after claim Plan 10): **every close is record first, then

+ the claim's release.** Rev 7.22 released the claim to `serving` first

+ and cleared `move` second (§7.5, §19 K8). Claim Plan 10's move-recovery

+ grant closes the record first, and a serving claim with a live move

+ record can never acquire that grant, so the two orders could not both

+ stand. Each close (the commit's, a rollback's, a revert's, an abort's)

+ now checks that no obligation of the op remains, CAS-clears `move` into

+ `last_move`, and then releases. A crash between the halves leaves the

+ record at rest and the claim still under the op with nothing owed; the

+ box passes are driven by records with a live move and do not come

+ back, so the console's recovery pass releases from `last_move`.

+ Amended: §7.5, §7.8 finalization, §7.9 step 5, §7.11, the `reverting`

+ table's `opened` row, §14.2.2's model row and §19 K8. **The model**

+ (`internal/sim/hostlife`, moves Plan 5 Task 7 fix round 1) closes the

+ same way: a box's close CASes the record (`last_move`), and on every

+ third step stops before the release; the recovery pass releases from

+ `last_move` (`fixRecord`). A new invariant, K8, fails any move's release

+ whose record still carries the move, and a new rule,

+ `record-first-close`, whose mutation is rev 7.22's claim-first close, is

+ caught by it (1,060 / 3,000 seeds, first seed 1; no witness). The

+ 3,000-seed sweep of every rule otherwise matches its expected outcomes. The `claim-cas-opid` witness is

+ restaged: its stalled goroutine is now the source between the record's

+ close and its release (§14.2.5).

- **Rev 7.22** answers **joint round 19** (`CODEX-FINDINGS-JOINT-R19.md`,

"R19-1", the round's one finding, P2) and fixes the install

manifest's keys. **R19-1:** rev 7.21 bound the prepared files manifest

@@ -2607,7 +2631,7 @@ itself as execution holder, holds the row as `restoring` with

| `verify` | entered from `stopped`, `staged` or `archived`: check files and timers against the digests recorded at `stopped` | replayable **until `probing` is journaled** |

| `probing` | render held config and materialize the release (§7.7), then probe | restarts only the probe. **Never re-runs `verify`**: the probe's process may have written, and those writes are now the authoritative state (INV-2) |

| `probed` | conditional open on `hold_op = move id` | replayable |

-| `opened` | the claim waits for the staging obligation (§7.11), then releases to `serving` | |

+| `opened` | the claim waits for the staging obligation (§7.11); then the record clears (`last_move` = `rolled_back`) and the claim releases to `serving` (record first, §7.5) | |

Rev 3 compared digests again on re-entry, so a crash after the recovery

probe had written left the move stuck on a "mismatch" that was really the

@@ -2657,11 +2681,19 @@ with crashes, console loss and the recovery pass between them (§14.2.2).

The source row stays held (`moving`); reconcile keeps it because the claim

names this box as a participant (§9.6).

-**The close is the other way round.** Releasing to `serving` is a claim

-CAS first (§19 K8), then the record CAS that clears `move` and names the

-executor's region; the recovery pass finishes the second half after a

-crash. Until it lands, the record still carries a finished `move`, which

-admits nothing.

+**The close is record first too (rev 7.23, Paul's ruling M4).** The

+actor that closes first checks that no obligation this op created

+remains, then CASes the record: `move` is cleared and `last_move` names

+the move and its outcome (`done`, `rolled_back` or `aborted`). The claim's

+release to `serving` (§19 K8) is the second CAS, determined by

+`last_move`: the op id, its kind and phase, and the executor the outcome

+names. Until it lands, the claim still holds the op, so nothing else

+starts on the host. After a crash between the two, no box revisits the

+host (the box passes are driven by records with a live move), and the

+console's recovery pass writes the release from `last_move`. Rev 7.22

+had this the other way round, claim first; claim Plan 10's recovery

+grant can act only while the claim holds the move op, so a claim

+released over a live `move` could never be finished by it.

**Late record writes.** Every move write to the record is a conditional

CAS. One that completes after its client gave up either still matches,

@@ -2873,9 +2905,9 @@ install supersedes. These are the claim spec's durable chores (§2 there);

**Finalization:** when `recovered_at`, `opened_at` and `target_parked_at`

are all stamped, and no obligation this op created remains, whichever

actor stamps last writes the terminal log event (`rolled_back`, with every

-timing, digest and reason), releases the claim to `serving` (§19 K8), and

-CAS-clears `move`. Until then `move` and the claim stay, so nothing else

-can start on the host (codex 6). The source serves from `opened` on; the

+timing, digest and reason), CAS-clears `move`, and then releases the

+claim to `serving` (§19 K8; record first, §7.5). Until then `move` and

+the claim stay, so nothing else can start on the host (codex 6). The source serves from `opened` on; the

held claim blocks only other operations.

Nine failed recovery probe attempts (three times the landing's budget of

@@ -2917,8 +2949,9 @@ runs anywhere. It converges when the target returns (INV-3).

`moves/<host>/`): `moves/<host>/<move id>/` into

`superseded/<move id>/<host>/`; remove its obligation.

5. **Close** (when both obligations are gone): terminal log event `done`;

- release the claim to `serving` with the target as execution holder

- (§19 K8); CAS-clear `move`.

+ CAS-clear `move` (`last_move` = `done`); then release the claim to

+ `serving` with the target as execution holder (§19 K8; record first,

+ §7.5).

### 7.10 Stuck and declare-lost (M16)

@@ -3068,7 +3101,8 @@ source retires them from its own bucket, every version of

`move-snapshots/<host>/<op>/` through `DeleteVersion`; that prefix is outside

the park scope grammar, and F3's retention of snapshots covers completed

moves only.) The claim stays `moving` until every obligation

-has acknowledged, then releases to `serving`. The model gives the source

+has acknowledged; then the record clears (`last_move` = `aborted`) and

+the claim releases to `serving` (record first, §7.5). The model gives the source

the target's park and records an IAM violation on every abort.

### 7.12 Deletes and parks under the claim

@@ -3824,7 +3858,7 @@ the text. What changed, and what did not:

|---|---|

| restore stronger than the spec (an exact base pointer, historical lineage selection) | restore reads `holder.location` and `holder.generation` and replays every object of that generation (§5.1). The mutation restores the newest object under the location |

| the start check stronger than the spec (compared with the committed lineage) | the start check compares the box's own three markers with each other and, as a separate rule, with the claim's generation (§5.1); rev 5's check is the mutation |

-| claim transitions followed by an atomic record write | record-first flip, commit and rollback, then the claim half as a separate step, written by the actor, the other participant or the recovery pass; the close the other way round (§7.5) |

+| claim transitions followed by an atomic record write | record-first flip, commit and rollback, then the claim half as a separate step, written by the actor, the other participant or the recovery pass; the close record first too, then the claim's release, which the recovery pass finishes from `last_move` after a crash (§7.5, rev 7.23) |

| every request completes by `issue + H` | one long-tail request in 20 completes up to 11 `H` late, after publication and after retirement; the restore contract keeps them out of data, the sweep retires them (§5.1, §6.4) |

| check and sign, check and local write, atomic | staging writers sign at the check and may then pause, a frozen box without bound (§7.3); host-guard writers are independent processes (`carlos-config-sync`, `carlos-adopt`) that survive an agent crash, pause before the guard and re-validate under it (claim §6.4); door listings read O1, pause, select, read O2 (claim §6.5) |

| atomic `localPark` | one artifact per step, the retired marker first |

@@ -5914,7 +5948,7 @@ witness to hold there; each witness violates with the failure it names.

| `dirty-return` (INV-2) | the target writes during its probe and replicates; the rollback is dirty and the return lane carries the write | **violated**: treated as clean with the source's replica covering the stop, the return is covered `clean-return`, nothing is declared and the serving database lacks the target's write (INV-2 local at the final recovery; rev 7.11 saw INV-15 at the declaration) | | 1 / 3,000 |

+| `claim-cas-opid` (C2) | the source closes a rollback's record and its goroutine stalls before the release, holding the rollback's last read (rev 7.23: before it, a goroutine stalled before a claim-first release); the recovery pass releases from `last_move`, a purge claims the host, and the goroutine resumes into its release | held | with `revalidate-before-effect`: **violated**, the store is torn down under a serving host | 0 / 3,000 |

| `exec-admission`, `hold-op-open`, `start-check-claim-gen`, `exec-writer` (J2-R2-8; claim rev 6.8.1) | a paused open from an earlier operation lands after the commit while the source's row is still held; a request wakes the source | held, each, and under every proper subset | all four: **violated**, INV-1 (the non-exec source is admitted cold into a fresh generation restored from its own region's old files and serves beside the target) | 0 / 3,000 |

| `start-check` (INV-4) | the same, dropping the whole start check | retired in rev 7.10: with every restore from the box's own region, the stale source finds nothing of the target's generation to restore and cannot start; subsumed (`expected-outcomes.txt`) | | 0 |

@@ -6350,7 +6384,7 @@ The db door refuses on every move kind (claim spec §8).

| **K5 Revalidation** | Before each destructive step the actor re-reads the claim and requires op id, kind, phase and `exec` to be what the step expects (claim spec C3); it holds the lease of every location it writes, acting before `SafeUntil`, with the exception in §19.3 D4. A staging writer checks `SafeUntil` immediately before it signs each request, after any pause (§7.3, model 14). |

| **K6 Wake admission** | `doWake` reads the claim fresh (claim spec §8) and applies K1. A box that reads a claim whose `exec` does not name it kills that host's process tree (§5.3 step 1) before anything else. |

| **K7 Other operations** | Delete, purge, park and push start only from `serving` (claim spec §4); while a move op holds the claim they are refused. Create requires free or absent. |

-| **K8 Release** | Only the move releases to `serving`, naming `exec` (target after `done`; source after `rolled_back` or `aborted`), and only when every obligation this op created is gone (§7.8, §7.9). |

+| **K8 Release** | Only the move releases to `serving`, naming `exec` (target after `done`; source after `rolled_back` or `aborted`), and only when every obligation this op created is gone (§7.8, §7.9). The release follows the record's close: the closing actor checks that nothing is owed under the op, CAS-clears `move` into `last_move`, then releases; the console's recovery pass finishes the release from `last_move` after a crash (§7.5; rev 7.23, Paul's ruling M4). |

| **K9 Other passes while a move op holds the claim** | Reconcile mints participant rows held and never `ready`, and never removes or orphan-retires a participant row; the schedule converger neither upserts nor prunes the host's timers; `SyncExec` and release adoption skip the held rows (they already skip non-`ready`). |

| **K10 Version floor** | The claim spec's floor (§9 there) and `moves_protocol` (§11) are one object and one check. |

| **K11 Declare-lost** | §7.10; §19.3 D6. |

diff --git a/internal/alerting/alerting.go b/internal/alerting/alerting.go

index 8956e2fec..22ceb2df0 100644

--- a/internal/alerting/alerting.go

+++ b/internal/alerting/alerting.go

@@ -187,6 +187,38 @@ func Page(ctx context.Context, level, subject, body string) error {

return s.Page(ctx, level, subject, body)

}

+// FromEnv builds a process's sinks from the agent's two variables,

+// CARLOS_ALERT_WEBHOOK (posted through PostJSON) and

+// CARLOS_ALERT_SNS_TOPIC_ARN (published from the topic's own region through

+// newClient, NewSNSClient when nil). The console arms these for its move

+// pass (moves Plan 5 Task 7). A topic that does not parse, or whose client

+// cannot be built, leaves SNS off and says so; the webhook is unaffected.

+// It never returns nil: with neither set, Page logs that no sink is

+// configured.

+func FromEnv(ctx context.Context, getenv func(string) string, newClient func(context.Context, string) (Publisher, error)) *Sinks {

+ if newClient == nil {

+ newClient = NewSNSClient

+ }

+ s := &Sinks{}

+ if url := strings.TrimSpace(getenv("CARLOS_ALERT_WEBHOOK")); url != "" {

+ s.WebhookURL, s.Post = url, PostJSON

+ }

+ if topic := strings.TrimSpace(getenv("CARLOS_ALERT_SNS_TOPIC_ARN")); topic != "" {

+ region, err := ParseTopicARN(topic)

+ if err != nil {

+ log.Printf("alerting: CARLOS_ALERT_SNS_TOPIC_ARN is not usable, SNS sink off: %v", err)

+ return s

+ }

+ pub, err := newClient(ctx, region)

+ if err != nil {

+ log.Printf("alerting: build the SNS client, SNS sink off: %v", err)

+ return s

+ }

+ s.TopicARN, s.Pub = topic, pub

+ }

+ return s

// PostJSON is the production webhook post: a 5 second POST with a JSON

// content type. Any non-2xx status is an error.

func PostJSON(url string, body []byte) error {

diff --git a/internal/alerting/alerting_test.go b/internal/alerting/alerting_test.go

index 90df95149..c84a93f87 100644

--- a/internal/alerting/alerting_test.go

+++ b/internal/alerting/alerting_test.go

@@ -206,3 +206,36 @@ func TestParseTopicARN(t *testing.T) {

}

+// FromEnv is the console's sinks (moves Plan 5 Task 7, delta M14): the

+// agent's two variables, the webhook through PostJSON and the topic's own

+// region for SNS. A topic that does not parse, or whose client cannot be

+// built, leaves SNS off and the webhook on; nothing set is no sink.

+func TestFromEnvBuildsTheConsoleSinks(t *testing.T) {

+ env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } }

+ var asked []string

+ client := func(_ context.Context, region string) (Publisher, error) {

+ asked = append(asked, region)

+ return &fakeSNS{}, nil

+ }

+ s := FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_WEBHOOK": " http://example.test/hook ", "CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), client)

+ if !s.HasWebhook() || s.WebhookURL != "http://example.test/hook" || !s.HasSNS() || s.TopicARN != testTopicARN {

+ t.Fatalf("both sinks: %+v", s)

+ }

+ if len(asked) != 1 || asked[0] != "eu-west-1" {

+ t.Fatalf("the SNS client was built for %v, want the topic's region", asked)

+ }

+ buf := captureLog(t)

+ s = FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_WEBHOOK": "http://example.test/hook", "CARLOS_ALERT_SNS_TOPIC_ARN": "arn:aws:sns:eu-west-1:1:x:sub"}), client)

+ if !s.HasWebhook() || s.HasSNS() || !strings.Contains(buf.String(), "SNS sink off") {

+ t.Fatalf("a bad topic: %+v (log %q)", s, buf.String())

+ }

+ s = FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), func(context.Context, string) (Publisher, error) { return nil, errors.New("no credentials") })

+ if s.HasWebhook() || s.HasSNS() {

+ t.Fatalf("a client that cannot be built: %+v", s)

+ }

+ s = FromEnv(context.Background(), env(nil), client)

+ if s == nil || s.HasWebhook() || s.HasSNS() {

+ t.Fatalf("nothing set: %+v", s)

+ }

diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go

index 1a702ccfc..455e34e1f 100644

--- a/internal/console/claim_authority_census_test.go

+++ b/internal/console/claim_authority_census_test.go

@@ -26,6 +26,9 @@ func claimAuthoritySites(t *testing.T) []string {

"ResumeExclusive": true, "RegisterStart": true, "FinishWakeRetryReset": true, "ReservePurge": true,

"ConsoleMove": true, "InspectMoveLoss": true, "DeclareMoveLoss": true,

"RunRecovery": true, "runMoveRecovery": true, "captureMoveStartBase": true, "resolveUncertain": true, "ConvertMovingToRelocating": true,

+ // The console's move pass (moves Plan 5 Task 7): every write of the

+ // move-recovery grant, and the lost target's stamp.

+ "ResolveStart": true, "Convert": true, "Claim": true, "Record": true, "StampLostMoveTargetParked": true, "stampLostMoveTarget": true,

"BeginPurge": true, "SavePurgeJournal": true, "RemovePurgeJournal": true,

"AcquireMailGate": true, "ReleaseMailGate": true, "EnrollNewApp": true,

"EnrollBackfillApp": true, "OpenGated": true, "WithAppMutation": true,

diff --git a/internal/console/claim_invariant_moves_test.go b/internal/console/claim_invariant_moves_test.go

index 51df1b2a3..94f14449f 100644

--- a/internal/console/claim_invariant_moves_test.go

+++ b/internal/console/claim_invariant_moves_test.go

@@ -132,3 +132,22 @@ func TestClaimPublishMovesConsoleIsInertWithTheSwitchOff(t *testing.T) {

t.Fatalf("the worker wrote with the moves switch off: %v", events)

}

+// TestClaimMovesConsoleBindsTheMovePass (moves Plan 5 Task 7): a moves-on

+// claim console binds its move pass as claim Plan 10's Recovery.Moves hook,

+// with the lost target's stamp; it adds no route and no worker (the pass runs

+// inside StartClaimRecoveryPass). A moves-off console binds neither, so the

+// recovery pass skips move claims as main does.

+func TestClaimMovesConsoleBindsTheMovePass(t *testing.T) {

+ off := newClaimCase(t).srv

+ if off.cfg.ClaimMoveRecovery != nil || off.cfg.HostClaimRecovery == nil || off.cfg.HostClaimRecovery.Moves != nil || off.stampLostMoveTarget != nil {

+ t.Fatal("a moves-off console bound the move pass")

+ }

+ on := newClaimCaseWith(t, claimMovesOn).srv

+ if on.cfg.ClaimMoveRecovery == nil || on.cfg.HostClaimRecovery == nil || on.cfg.HostClaimRecovery.Moves == nil || on.stampLostMoveTarget == nil {

+ t.Fatal("a moves-on console did not bind the move pass")

+ }

+ if on.cfg.HostClaimRecovery.MoveFenceAuthority == nil {

+ t.Fatal("the move pass has no fence authority for its conversions")

+ }

diff --git a/internal/console/claim_runtime.go b/internal/console/claim_runtime.go

index 6a41d1e10..c1bc23eb5 100644

--- a/internal/console/claim_runtime.go

+++ b/internal/console/claim_runtime.go

@@ -109,6 +109,21 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {

s.cfg.ClaimPlacement = placement

s.claimOwner = owner

s.claimRuntimeConfigured = true

+ if s.cfg.MovesEnabled && s.cfg.ClaimMoveRecovery == nil {

+ // The console's move pass (moves Plan 5 Task 7): the moves body of

+ // claim Plan 10's Recovery.Moves hook, bound only with the moves

+ // switch on, and only now that every box close is record first

+ // (ruling M4, P3-6; TestClaimMoveRecoveryWiringWaitsForRecordFirstOwners).

+ s.cfg.ClaimMoveRecovery = s.movesRecovery

+ // The lost target's target_parked_at (p10 Task G) is written

+ // outside the grant by the lifecycle record writer, which the

+ // conditional record store carries and the data boundary (an app

+ // permit's) does not. The pass gets this one transition, never

+ // the store.

+ s.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

+ return hostops.StampLostMoveTargetParked(ctx, records, host, now)

+ }

s.cfg.HostClaimRecovery = &hostops.Recovery{

Moves: s.cfg.ClaimMoveRecovery,

MoveFenceAuthority: hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: c.Now},

diff --git a/internal/console/movesrecovery.go b/internal/console/movesrecovery.go

new file mode 100644

index 000000000..db883ea40

--- /dev/null

+++ b/internal/console/movesrecovery.go

@@ -0,0 +1,516 @@

+package console

+import (

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "errors"

+ "fmt"

+ "log"

+ "os"

+ "slices"

+ "time"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostfence"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+// The console's move pass (moves Plan 5 Task 7, spec §7.5, §7.10, §7.11,

+// §13; Paul's ruling M4). It is the moves body of claim Plan 10's

+// Recovery.Moves hook: the recovery pass (ControlWriter.RecoveryPass) calls

+// movesRecovery once per claim of a move kind, with a MoveRecoveryGrant bound

+// to that claim, its move registration and the exact record it observed.

+// ConfigureClaimRuntime binds it only with the moves switch on.

+//

+// Each call reads the claim and the record again through the console's store,

+// gives up if either moved on since the grant's observation (the next pass

+// observes afresh), decides at most one action (decideMoveRecovery) and takes

+// it. The grant is spent by its first write, so one observation is never two

+// writes. In the order the decision tries them:

+//

+// 1. An orphan: a claim of a move kind whose record carries neither the move

+// nor its outcome. A moving claim before the seal (the request door's

+// record write was lost) is left for ten minutes, then its start effect

+// is settled without the request's handle (ResolveStart, K3), and on a

+// later pass, with the start settled not applied, released

+// (release-not-applied). Any other orphan is paged, once, and left: an

+// operation past the seal with no record is an incident.

+// 2. A source terminated before the flip (K13): Convert with the source's

+// termination fence, which writes the record's relocation decision on

+// one pass and converts the claim to relocating on the next.

+// 3. The six-hour abort (K18): a move still preparing moves.AbortAfter

+// after its request is aborted through ConsoleMove(ConsoleMoveAbort);

+// the source's own abort (movesabort.go) takes it from there.

+// 4. The claim half the record decided (§7.5, Review Focus 5): flip,

+// commit or rollback. Anything else is left alone.

+// 5. The deadline: a landing past the earlier of its cut and landing

+// deadlines is rolled back by the record (J11, J9's recorded reason).

+// 6. A lost target released with nothing of its box or region owed:

+// target_parked_at, through hostops.StampLostMoveTargetParked over the

+// conditional record store (stampLostMoveTarget), outside the grant

+// (p10 Task G).

+// 7. A close whose stamps are in and whose op owes nothing: the record's

+// close (close, rolled-back-close or revert-close). The boxes close on

+// their own in the pass the close becomes due; the console writes it

+// only when it has been due for moveCloseGrace, and pages that it did.

+// An abort is never closed here (fix round 1, ruling 1): only the

+// source closes it, after it retired its own snapshots, receipts,

+// ledger and journal (movesabort.go), and nothing on the record shows

+// that it did. An abort due past the lane's horizon (the claim's

+// quiet_after for the target region) plus moveCloseGrace is paged

+// once, and left to the source.

+// 8. The release: the record closed (LastMove names the op) and the claim

+// not yet released. This is the second half of every record-first close

+// (ruling M4); after a box crashed between the halves, only this pass

+// writes it, because no box visits a record without a move.

+//

+// Pages (§13) go through alerting.Page on the sinks cmd/console arms, best

+// effort: a rollback waiting past its release deadline (an incident, again

+// only a full hour after the last attempt), a move preparing for over an

+// hour, an orphan past the seal (an incident), an overdue close the console

+// wrote, and an overdue abort the console leaves to its source (each once).

+// The rate limit is durable: the last attempt's time is kept per finding in

+// the console's store (move_pages, fix round 1 ruling 3), written whether or

+// not delivery worked, so a restarted console keeps the spacing.

+//

+// A relocating claim is region HA's, and the pass does nothing for it.

+const (

+ // moveOrphanAfter is how old, by Op.StartedAt, a moving claim with no

+ // record move must be before the pass settles its start.

+ moveOrphanAfter = 10 * time.Minute

+ // movePreparingPage is how long a move may prepare before it pages

+ // (§13: 60 minutes with no promotable round).

+ movePreparingPage = time.Hour

+ // moveCloseGrace is how long a close must have been due before the

+ // console writes it instead of the boxes.

+ moveCloseGrace = 10 * time.Minute

+type moveRecoveryAction string

+const (

+ moveActNone moveRecoveryAction = ""

+ moveActResolveStart moveRecoveryAction = "resolve-start"

+ moveActConvert moveRecoveryAction = "convert"

+ moveActAbort moveRecoveryAction = "abort"

+ moveActClaim moveRecoveryAction = "claim"

+ moveActRecord moveRecoveryAction = "record"

+ moveActStampLost moveRecoveryAction = "stamp-lost"

+ moveActClose moveRecoveryAction = "close"

+ // moveActAbortOverdue writes nothing: an abort due to close, which only

+ // its source may close, is checked for being overdue and paged.

+ moveActAbortOverdue moveRecoveryAction = "abort-overdue"

+type movePageKind string

+const (

+ movePageReleaseDeadline movePageKind = "release-deadline"

+ movePagePreparing movePageKind = "preparing"

+ movePageOrphan movePageKind = "orphan"

+ movePageCloseOverdue movePageKind = "close-overdue"

+ movePageAbortOverdue movePageKind = "abort-overdue"

+// moveRecoveryInput is what one observation saw. Start is the op's move-start

+// effect, read only for a moving claim with no record move; Fence is the

+// source's termination fence, read only for a moving claim.

+type moveRecoveryInput struct {

+ Now time.Time

+ Claim *hostclaim.Claim

+ Rec provision.Record

+ Start *hostops.Effect

+ Fence *hostclaim.FenceRef

+// moveRecoveryDecision is at most one action, with the grant action it

+// names, and the pages the observation calls for.

+type moveRecoveryDecision struct {

+ Action moveRecoveryAction

+ Claim hostops.MoveRecoveryClaimAction

+ Record hostops.MoveRecoveryRecordAction

+ Pages []movePageKind

+// decideMoveRecovery is the pass's whole policy, on values (see the file

+// comment for the order).

+func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision {

+ var d moveRecoveryDecision

+ c := in.Claim

+ if c == nil || c.Holder == nil || c.Op.ID == "" || c.Op.Kind == hostclaim.KindRelocating {

+ return d

+ }

+ op, rec, m := c.Op, in.Rec, in.Rec.Move

+ sealed := op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseSealed

+ if m == nil || m.Legacy() || m.ID != op.ID {

+ if m != nil {

+ return d // another move's record: not this op's to settle

+ }

+ if rec.LastMove != nil && rec.LastMove.ID == op.ID {

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRelease

+ return d

+ }

+ if op.Kind != hostclaim.KindMoving || op.Phase != hostclaim.PhaseMoving {

+ d.Pages = append(d.Pages, movePageOrphan)

+ return d

+ }

+ if op.StartedAt.IsZero() || in.Now.Before(op.StartedAt.Add(moveOrphanAfter)) || in.Start == nil {

+ return d

+ }

+ switch {

+ case in.Start.State == "uncertain":

+ d.Action = moveActResolveStart

+ case in.Start.State == "complete" && in.Start.ResultReference == "not-applied":

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveReleaseNotApplied

+ }

+ return d

+ }

+ if m.Incarnation != c.Holder.Incarnation {

+ return d

+ }

+ rb := m.Rollback

+ if m.State == provision.MoveRollingBack && rb != nil && rb.ReleaseDeadline != nil && in.Now.After(*rb.ReleaseDeadline) && rb.TargetPhase == provision.RbTargetReleasing {

+ d.Pages = append(d.Pages, movePageReleaseDeadline)

+ }

+ if m.State == provision.MovePreparing && !m.RequestedAt.IsZero() && !in.Now.Before(preparingSince(m).Add(movePreparingPage)) {

+ d.Pages = append(d.Pages, movePagePreparing)

+ }

+ owed := owedUnderOp(c, op.ID, nil)

+ switch {

+ case op.Kind == hostclaim.KindMoving && in.Fence != nil && (m.State == provision.MovePreparing || m.State == provision.MoveReady || m.State == provision.MoveCutting || m.State == provision.MoveRelocating):

+ d.Action = moveActConvert

+ case op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseMoving && moves.AbortDue(rec, in.Now):

+ d.Action = moveActAbort

+ case m.State == provision.MoveLanding && m.FlippedAt != nil && rec.Region == m.To && sealed:

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveFlip

+ case m.State == provision.MoveCommitted && m.CommittedAt != nil && m.FlippedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseLandingProbed:

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveCommit

+ case m.State == provision.MoveRollingBack && rb != nil && rec.Region == m.From && (op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted || sealed):

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRollback

+ case m.State == provision.MoveLanding && op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted && moveDeadlinePassed(m, in.Now):

+ d.Action, d.Record = moveActRecord, hostops.RecoveryMoveDeadlineRollback

+ case m.State == provision.MoveRollingBack && rb != nil && rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty && rb.TargetParkedAt == nil &&

+ (op.Kind == hostclaim.KindRestoring || op.Kind == hostclaim.KindLanding) && m.To != "" && op.Intent["target"] != "" && slices.Contains(c.Lost, op.Intent["target"]) &&

+ owedUnderOp(c, op.ID, func(o hostclaim.Obligation) bool {

+ t := op.Intent["target"]

+ return o.Box == t || o.Owner == "box:"+t || o.Region == m.To || o.Owner == "region:"+m.To

+ }) == 0:

+ d.Action = moveActStampLost

+ case owed > 0:

+ case m.State == provision.MoveCommitted && m.OpenedAt != nil && m.SourceParkedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseCommitted:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveClose

+ case m.State == provision.MoveRollingBack && rb != nil && rb.RecoveredAt != nil && rb.OpenedAt != nil && rb.TargetParkedAt != nil && rec.Region == m.From && op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseOpened:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveRolledBackClose

+ case m.State == provision.MoveReverting && m.ReopenedAt != nil && rec.Region == m.From && op.Kind == hostclaim.KindReverting && op.Phase == hostclaim.PhaseOpened:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveRevertClose

+ case m.State == provision.MoveAborting && op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseAborting:

+ d.Action = moveActAbortOverdue

+ }

+ return d

+// preparingSince is where a preparing move's clocks run from: the later of

+// its request and its readiness (moves.AbortDue's base, K18).

+func preparingSince(m *provision.MoveState) time.Time {

+ from := m.RequestedAt

+ if m.ReadyAt != nil && m.ReadyAt.After(from) {

+ from = *m.ReadyAt

+ }

+ return from

+// moveDeadlinePassed is the grant's deadline rule: the earlier of the cut and

+// landing deadlines (J11) is behind now.

+func moveDeadlinePassed(m *provision.MoveState, now time.Time) bool {

+ deadline := m.CutDeadline

+ if l := m.LandingDeadline; l != nil && (deadline == nil || l.Before(*deadline)) {

+ deadline = l

+ }

+ return deadline != nil && now.After(*deadline)

+// owedUnderOp counts the claim's obligations under op id, those match

+// accepts when it is set.

+func owedUnderOp(c *hostclaim.Claim, id string, match func(hostclaim.Obligation) bool) int {

+ n := 0

+ for _, o := range c.Obligations {

+ if o.Op == id && (match == nil || match(o)) {

+ n++

+ }

+ return n

+func (s *Server) movesNow() time.Time {

+ if s.movesClock != nil {

+ return s.movesClock().UTC()

+ }

+ return time.Now().UTC()

+// movesRecovery is the Recovery.Moves hook (see the file comment).

+func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant) error {

+ a := s.cfg.ClaimAuthority

+ store := s.cfg.Releases

+ if a == nil || a.Control == nil || store == nil || g == nil {

+ return hostops.ErrContract

+ }

+ o := g.Observation()

+ obs, err := hostclaim.Read(ctx, store, o.Host)

+ if err != nil {

+ return err

+ }

+ if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.ETag != o.ClaimETag || obs.Claim.Op.ID != o.OpID {

+ return nil // moved on since the observation: the next pass observes again

+ }

+ c := obs.Claim

+ r, err := hostops.ReadRecord(ctx, store, c.Holder.Key)

+ if err != nil {

+ return err

+ }

+ if r.ETag != o.RecordETag {

+ return nil

+ }

+ rec, err := provision.DecodeRecord(r.Body)

+ if err != nil {

+ return err

+ }

+ now := s.movesNow()

+ in := moveRecoveryInput{Now: now, Claim: c, Rec: rec}

+ ours := rec.Move != nil && rec.Move.ID == o.OpID

+ if c.Op.Kind == hostclaim.KindMoving && c.Op.Phase == hostclaim.PhaseMoving && !ours {

+ adm, err := hostops.ReadAdmission(ctx, store, o.Account, o.App, o.AppID)

+ if err != nil {

+ return err

+ }

+ if e, ok := adm.Value.Effects[o.OpID]; ok {

+ in.Start = &e

+ }

+ if c.Op.Kind == hostclaim.KindMoving && ours {

+ if in.Fence, err = s.sourceFence(ctx, c); err != nil {

+ return err

+ }

+ d := decideMoveRecovery(in)

+ for _, kind := range d.Pages {

+ s.pageMove(ctx, kind, o, c, rec, now)

+ }

+ switch d.Action {

+ case moveActResolveStart:

+ out, err := g.ResolveStart(ctx)

+ if errors.Is(err, hostops.ErrMoveStartFenceUnproven) {

+ log.Printf("console: moves: %s: the start of move %s is uncertain and its record has not advanced past it; left", o.Host, o.OpID)

+ return nil

+ }

+ if err == nil {

+ log.Printf("console: moves: %s: the start of move %s settled %s", o.Host, o.OpID, out)

+ }

+ return err

+ case moveActConvert:

+ return g.Convert(ctx, *in.Fence)

+ case moveActAbort:

+ co, err := hostops.ObserveConsoleMove(ctx, store, o.Account, o.App, o.Host)

+ if err != nil {

+ return err

+ }

+ if co != o {

+ return nil

+ }

+ if _, err := a.Control.ConsoleMove(ctx, co, hostops.ConsoleMoveAbort); err != nil {

+ return err

+ }

+ log.Printf("console: moves: %s: move %s is still preparing after %s; aborted", o.Host, o.OpID, moves.AbortAfter)

+ return nil

+ case moveActClaim:

+ return g.Claim(ctx, d.Claim)

+ case moveActRecord:

+ return g.Record(ctx, d.Record)

+ case moveActStampLost:

+ if s.stampLostMoveTarget == nil {

+ return hostops.ErrContract

+ }

+ return s.stampLostMoveTarget(ctx, o.Host, now)

+ case moveActClose:

+ if !s.moveCloseOverdue(o, d.Record, now) {

+ return nil

+ }

+ if err := g.Record(ctx, d.Record); err != nil {

+ return err

+ }

+ log.Printf("console: moves: %s: move %s closed by the console (%s); the release follows", o.Host, o.OpID, d.Record)

+ s.pageMove(ctx, movePageCloseOverdue, o, c, rec, now)

+ return nil

+ case moveActAbortOverdue:

+ if s.moveAbortOverdue(o, c.QuietAfter[rec.Move.To], now) {

+ s.pageMove(ctx, movePageAbortOverdue, o, c, rec, now)

+ }

+ return nil

+ }

+ return nil

+// sourceFence is the moving source's termination fence, when one exists:

+// the reference to control/box-fences/<source enrollment>.json and its exact

+// bytes. Convert verifies it; nothing here trusts it.

+func (s *Server) sourceFence(ctx context.Context, c *hostclaim.Claim) (*hostclaim.FenceRef, error) {

+ enrollment := c.Op.Intent["source_enrollment"]

+ if enrollment == "" {

+ return nil, nil

+ }

+ key := hostfence.FenceKey(enrollment)

+ body, _, err := s.cfg.Releases.Get(ctx, key)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, nil

+ }

+ if err != nil {

+ return nil, err

+ }

+ sum := sha256.Sum256(body)

+ return &hostclaim.FenceRef{Enrollment: enrollment, Key: key, SHA256: hex.EncodeToString(sum[:])}, nil

+// moveDueSince is when the pass first saw this close due, remembering now

+// if it had not. The memory is the process's: a restarted console waits the

+// grace again, which only delays the console's close or page. Entries a day

+// old are dropped.

+func (s *Server) moveDueSince(key string, now time.Time) time.Time {

+ s.moveCloseMu.Lock()

+ defer s.moveCloseMu.Unlock()

+ if s.moveCloseSeen == nil {

+ s.moveCloseSeen = map[string]time.Time{}

+ }

+ for k, at := range s.moveCloseSeen {

+ if now.Sub(at) > 24*time.Hour {

+ delete(s.moveCloseSeen, k)

+ }

+ first, ok := s.moveCloseSeen[key]

+ if !ok {

+ s.moveCloseSeen[key] = now

+ return now

+ }

+ return first

+// moveCloseOverdue reports whether this close has been due for

+// moveCloseGrace; when it has, the memory is cleared for the close.

+func (s *Server) moveCloseOverdue(o hostops.ConsoleMoveObservation, action hostops.MoveRecoveryRecordAction, now time.Time) bool {

+ key := o.Host + "|" + o.OpID + "|" + string(action)

+ if now.Sub(s.moveDueSince(key, now)) < moveCloseGrace {

+ return false

+ }

+ s.moveCloseMu.Lock()

+ delete(s.moveCloseSeen, key)

+ s.moveCloseMu.Unlock()

+ return true

+// moveAbortOverdue reports whether an abort with nothing owed has stayed

+// unclosed past its lane's horizon (quiet, the claim's quiet_after for the

+// target region, when the claim has one) plus moveCloseGrace, and has been

+// seen due for moveCloseGrace itself. The page it leads to is once per move

+// (pageMove), so the memory is kept.

+func (s *Server) moveAbortOverdue(o hostops.ConsoleMoveObservation, quiet, now time.Time) bool {

+ since := s.moveDueSince(o.Host+"|"+o.OpID+"|abort", now)

+ if now.Sub(since) < moveCloseGrace {

+ return false

+ }

+ return quiet.IsZero() || !now.Before(quiet.Add(moveCloseGrace))

+// pageMove sends one page for a finding (its kind, host and move) unless

+// the finding was paged before: ever, for a once-per-move page, or within

+// the last full hour, for the release deadline. The attempt's time is

+// recorded whether or not delivery worked (pages are best effort).

+func (s *Server) pageMove(ctx context.Context, kind movePageKind, o hostops.ConsoleMoveObservation, c *hostclaim.Claim, rec provision.Record, now time.Time) {

+ rule := string(kind) + " " + o.Host + " " + o.OpID

+ level := alerting.LevelWarning

+ every := time.Duration(0) // once per move

+ var subject, body string

+ switch kind {

+ case movePageReleaseDeadline:

+ level, every = alerting.LevelIncident, time.Hour

+ subject, body = moveWaitingPage(o.Host, o.OpID, rec.Move.To)

+ case movePagePreparing:

+ subject, body = movePreparingPageText(o.Host, o.OpID, rec.Move.To)

+ case movePageOrphan:

+ level = alerting.LevelIncident

+ subject, body = moveOrphanPage(o.Host, o.OpID)

+ case movePageCloseOverdue:

+ subject, body = moveCloseOverduePage(o.Host, o.OpID)

+ case movePageAbortOverdue:

+ subject, body = moveAbortOverduePage(o.Host, o.OpID, rec.Move.To, c.Op.Intent["source"])

+ default:

+ return

+ }

+ if s.store == nil {

+ return

+ }

+ last, paged, err := s.store.MovePageAttempt(rule)

+ if err != nil {

+ log.Printf("console: moves: %s: the page record for %s: %v", o.Host, rule, err)

+ return

+ }

+ if paged && (every == 0 || now.Sub(last) < every) {

+ return

+ }

+ if err := s.store.SetMovePageAttempt(rule, now); err != nil {

+ log.Printf("console: moves: %s: record the page %s: %v", o.Host, rule, err)

+ return

+ }

+ if err := alerting.Page(ctx, level, subject, body); err != nil {

+ log.Printf("console: moves: %s: page %s: %v", o.Host, kind, err)

+ }

+// moveWaitingPage is a rollback waiting past its release deadline (§7.8

+// waiting): the subject (MV-N325) and the body (MV-N326).

+func moveWaitingPage(host, id, to string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move waiting for %s to release: %s", to, host)

+ body = fmt.Sprintf("The move %s of %s is rolling back, and %s has not released it by its deadline. %s stays offline until a box in %s confirms it has stopped, or that box is declared lost.",

+ id, host, to, host, to)

+ return subject, body

+// movePreparingPageText is a move preparing for over an hour (§13): the

+// subject (MV-N327) and the body (MV-N328).

+func movePreparingPageText(host, id, to string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move preparing for over an hour: %s", host)

+ body = fmt.Sprintf("The move %s of %s to %s has been preparing for over an hour. It aborts itself six hours after the request if %s is still not ready.",

+ id, host, to, to)

+ return subject, body

+// moveOrphanPage is a claim held by a move the record does not carry (K3):

+// the subject (MV-N329) and the body (MV-N330).

+func moveOrphanPage(host, id string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move without a record: %s", host)

+ body = fmt.Sprintf("The host claim of %s is held by the move %s, and the instance record carries no move. Nothing releases it on its own. An operator must check the record.",

+ host, id)

+ return subject, body

+// moveCloseOverduePage is a close the console wrote because no box did: the

+// subject (MV-N331) and the body (MV-N332).

+func moveCloseOverduePage(host, id string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move close overdue: %s", host)

+ body = fmt.Sprintf("The move %s of %s was ready to close for more than 10 minutes and no box closed it. The console has closed it.", id, host)

+ return subject, body

+// moveAbortOverduePage is an abort that owes nothing and that its source has

+// not closed (fix round 1, ruling 1): the subject (MV-N333) and the body

+// (MV-N334).

+func moveAbortOverduePage(host, id, to, box string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move abort not closed: %s", host)

+ body = fmt.Sprintf("The abort of the move %s of %s has nothing left to park in %s, and %s has not closed it. %s closes it once it has removed the move's snapshots from its own bucket. Check %s.",

+ id, host, to, box, box, box)

+ return subject, body

diff --git a/internal/console/movesrecovery_test.go b/internal/console/movesrecovery_test.go

new file mode 100644

index 000000000..0e43f0ce7

--- /dev/null

+++ b/internal/console/movesrecovery_test.go

@@ -0,0 +1,721 @@

+package console

+import (

+ "context"

+ "crypto/rand"

+ "encoding/json"

+ "strings"

+ "sync"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+// The console's move pass (moves Plan 5 Task 7; claim Plan 10's

+// Recovery.Moves hook under the move-recovery grant). The decision table is

+// tested on values; every action is then driven end to end through the real

+// recovery pass (ControlWriter.RecoveryPass over hostops.Recovery.Pass) on a

+// host-claim console whose move the door started.

+// ---- the decision, on values ----

+var mvT0 = time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)

+func mvClaim(kind, phase string, mut ...func(*hostclaim.Claim)) *hostclaim.Claim {

+ c := &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"},

+ Op: hostclaim.Op{Kind: kind, Phase: phase, ID: "mv", Exec: "ie-box", Participants: []string{"ie-box", "uk-box"}, StartedAt: mvT0,

+ Intent: map[string]string{"source": "ie-box", "target": "uk-box"}}}

+ for _, m := range mut {

+ m(c)

+ }

+ return c

+func mvRecord(state string, mut ...func(*provision.Record)) provision.Record {

+ at := mvT0

+ r := provision.Record{Region: "ie", Move: &provision.MoveState{ID: "mv", Incarnation: "inc", From: "ie", To: "uk", State: state, RequestedAt: mvT0}}

+ switch state {

+ case provision.MoveLanding, provision.MoveCommitted:

+ r.Region = "uk"

+ r.Move.FlippedAt = &at

+ }

+ if state == provision.MoveCommitted {

+ r.Move.CommittedAt = &at

+ }

+ if state == provision.MoveRollingBack {

+ r.Move.FlippedAt = &at

+ r.Move.Rollback = &provision.MoveRollback{ID: "rb", TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceWaiting}

+ }

+ for _, m := range mut {

+ m(&r)

+ }

+ return r

+func mvAt(d time.Duration) *time.Time { t := mvT0.Add(d); return &t }

+func TestRecoveryPassWritesOnlyDecidedHalves(t *testing.T) {

+ now := mvT0.Add(time.Minute)

+ for _, c := range []struct {

+ name string

+ claim *hostclaim.Claim

+ rec provision.Record

+ action moveRecoveryAction

+ half hostops.MoveRecoveryClaimAction

+ }{

+ {"record landing, claim moving/sealed: the flip half", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding), moveActClaim, hostops.RecoveryMoveFlip},

+ {"record committed, claim landing/probed: the commit half", mvClaim(hostclaim.KindLanding, hostclaim.PhaseLandingProbed), mvRecord(provision.MoveCommitted), moveActClaim, hostops.RecoveryMoveCommit},

+ {"record rolling_back, claim landing: the rollback half", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack), moveActClaim, hostops.RecoveryMoveRollback},

+ {"record rolling_back, claim moving/sealed: RollBackFlipped (K22)", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveRollingBack), moveActClaim, hostops.RecoveryMoveRollback},

+ {"record landing, claim landing: in step", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveLanding), moveActNone, ""},

+ {"record cutting, claim moving/sealed: no decision", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveCutting), moveActNone, ""},

+ {"record committed, claim landing/probing: the claim is behind the probe", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveCommitted), moveActNone, ""},

+ {"record committed, claim landing/committed, stamps missing", mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), mvRecord(provision.MoveCommitted), moveActNone, ""},

+ {"record landing without its flip stamp", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.FlippedAt = nil }), moveActNone, ""},

+ {"record rolling_back in the target's region", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack, func(r *provision.Record) { r.Region = "uk" }), moveActNone, ""},

+ {"record preparing, claim moving: nothing yet", mvClaim(hostclaim.KindMoving, hostclaim.PhaseMoving), mvRecord(provision.MovePreparing), moveActNone, ""},

+ {"another move's record", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.ID = "other" }), moveActNone, ""},

+ {"another incarnation's record", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.Incarnation = "old" }), moveActNone, ""},

+ {"a relocating claim is region HA's", mvClaim(hostclaim.KindRelocating, "pinning"), mvRecord(provision.MoveRelocating), moveActNone, ""},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ got := decideMoveRecovery(moveRecoveryInput{Now: now, Claim: c.claim, Rec: c.rec})

+ if got.Action != c.action || got.Claim != c.half {

+ t.Fatalf("decided %+v, want %s %s", got, c.action, c.half)

+ }

+ })

+ }

+// One eligible action per observation, in the brief's order: a pending

+// claim half comes before the deadline, the deadline before a close, the

+// lost target's stamp before the close it enables.

+func TestRecoveryPassOneActionPerObservation(t *testing.T) {

+ past := mvT0.Add(-time.Hour)

+ // Flip half pending and the landing deadline passed: the half first.

+ d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })})

+ if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveFlip {

+ t.Fatalf("flip and deadline: %+v", d)

+ }

+ // With the claim in step, the deadline.

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })})

+ if d.Action != moveActRecord || d.Record != hostops.RecoveryMoveDeadlineRollback {

+ t.Fatalf("the deadline: %+v", d)

+ }

+ // The earlier of the cut and landing deadlines (J11).

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.CutDeadline = &past; r.Move.LandingDeadline = mvAt(time.Hour) })})

+ if d.Action != moveActRecord {

+ t.Fatalf("the cut deadline: %+v", d)

+ }

+ // A lost, released target: the stamp, never the close it enables.

+ lost := func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"} }

+ rb := func(r *provision.Record) {

+ r.Move.Rollback.TargetPhase, r.Move.Rollback.SourcePhase = provision.RbTargetReleased, provision.RbSourceOpened

+ r.Move.Rollback.RecoveredAt, r.Move.Rollback.OpenedAt = mvAt(0), mvAt(0)

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened, lost), Rec: mvRecord(provision.MoveRollingBack, rb)})

+ if d.Action != moveActStampLost {

+ t.Fatalf("the lost target: %+v", d)

+ }

+ // A chore of the target's region still owed: neither.

+ owed := func(c *hostclaim.Claim) {

+ c.Obligations = []hostclaim.Obligation{{ID: "o", Op: "mv", Kind: "park-replica", Region: "uk", Owner: "region:uk"}}

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened, lost, owed), Rec: mvRecord(provision.MoveRollingBack, rb)})

+ if d.Action != moveActNone {

+ t.Fatalf("the lost target with its region's chore owed: %+v", d)

+ }

+ // Every close waits for the op's chores (the grant's rule, M4).

+ stamped := func(r *provision.Record) { r.Move.OpenedAt, r.Move.SourceParkedAt = mvAt(0), mvAt(0) }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted, owed), Rec: mvRecord(provision.MoveCommitted, stamped)})

+ if d.Action != moveActNone {

+ t.Fatalf("a close with a chore owed: %+v", d)

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), Rec: mvRecord(provision.MoveCommitted, stamped)})

+ if d.Action != moveActClose || d.Record != hostops.RecoveryMoveClose {

+ t.Fatalf("the committed close: %+v", d)

+ }

+ for _, c := range []struct {

+ claim *hostclaim.Claim

+ rec provision.Record

+ want hostops.MoveRecoveryRecordAction

+ }{

+ {mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened), mvRecord(provision.MoveRollingBack, rb, func(r *provision.Record) { r.Move.Rollback.TargetParkedAt = mvAt(0) }), hostops.RecoveryMoveRolledBackClose},

+ {mvClaim(hostclaim.KindReverting, hostclaim.PhaseOpened), mvRecord(provision.MoveReverting, func(r *provision.Record) { r.Move.ReopenedAt = mvAt(0) }), hostops.RecoveryMoveRevertClose},

+ } {

+ if d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: c.claim, Rec: c.rec}); d.Action != moveActClose || d.Record != c.want {

+ t.Fatalf("%s/%s: %+v, want close %s", c.claim.Op.Kind, c.claim.Op.Phase, d, c.want)

+ }

+ // An abort the console never closes (fix round 1, ruling 1): only the

+ // source closes it, after it retired its own state. Due, it is at most

+ // a page.

+ if d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindMoving, hostclaim.PhaseAborting), Rec: mvRecord(provision.MoveAborting)}); d.Action != moveActAbortOverdue || d.Record != "" {

+ t.Fatalf("a due abort: %+v, want only the overdue check", d)

+ }

+ // The record closed, the claim not released: the release.

+ closed := func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: "mv", Outcome: "done", At: mvT0}

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), Rec: mvRecord(provision.MoveCommitted, closed)})

+ if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveRelease {

+ t.Fatalf("the release from LastMove: %+v", d)

+ }

+// ---- the pass, end to end ----

+// movePassWorld is a host-claim console whose move the request door

+// started (j17StartedMove), with the console's own move pass bound as the

+// Recovery.Moves hook, pages captured from the armed alert sinks, and the

+// pass's clock under the test's control.

+type movePassWorld struct {

+ fx *consoleFixture

+ id string

+ auth hostclaim.FenceAuthority

+ clock time.Time

+ mu sync.Mutex

+ pages []pagePayload

+type pagePayload struct{ Kind, Level, Subject, Body string }

+func newMovePassWorld(t *testing.T, fx *consoleFixture, id string) *movePassWorld {

+ t.Helper()

+ w := &movePassWorld{fx: fx, id: id, clock: time.Now().UTC()}

+ fx.srv.movesClock = func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.clock }

+ // As ConfigureClaimRuntime binds it: the lost target's stamp over the

+ // conditional record store under the fixture's data boundary.

+ records, err := hostops.NewConditionalRecordStore(fx.rel, fx.pub)

+ if err != nil {

+ t.Fatal(err)

+ }

+ fx.srv.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

+ return hostops.StampLostMoveTargetParked(ctx, records, host, now)

+ }

+ alerting.Arm(&alerting.Sinks{WebhookURL: "http://pages.test/hook", Post: func(_ string, body []byte) error {

+ var p pagePayload

+ if err := json.Unmarshal(body, &p); err != nil {

+ return err

+ }

+ w.mu.Lock()

+ w.pages = append(w.pages, p)

+ w.mu.Unlock()

+ return nil

+ }})

+ t.Cleanup(func() { alerting.Arm(nil) })

+ return w

+func (w *movePassWorld) advance(d time.Duration) {

+ w.mu.Lock()

+ w.clock = w.clock.Add(d)

+ w.mu.Unlock()

+// pass runs one console recovery pass with only the moves hook live.

+func (w *movePassWorld) pass(t *testing.T) {

+ t.Helper()

+ a := w.fx.srv.cfg.ClaimAuthority

+ nop := func(context.Context) error { return nil }

+ nopObs := func(context.Context, hostclaim.Observed) error { return nil }

+ r := hostops.Recovery{Moves: w.fx.srv.movesRecovery, MoveFenceAuthority: w.auth,

+ Controller: hostops.Controller{Store: w.fx.srv.cfg.Releases, Control: a.Control, Authority: a, Now: time.Now, NewID: claimOperationID},

+ Purge: nopObs, Placement: nopObs, SweepRecords: nop, SweepPurges: nop, SweepAdmissions: nop}

+ if err := a.Control.RecoveryPass(context.Background(), r); err != nil {

+ t.Fatalf("the recovery pass: %v", err)

+ }

+func (w *movePassWorld) claim(t *testing.T) *hostclaim.Claim {

+ t.Helper()

+ o, err := hostclaim.Read(context.Background(), w.fx.rel, "acme.go.tito.io")

+ if err != nil || o.Absent {

+ t.Fatalf("read the claim: %v", err)

+ }

+ return o.Claim

+func (w *movePassWorld) record(t *testing.T) provision.Record {

+ t.Helper()

+ rec, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "titogo", "acme.go.tito.io")

+ if err != nil || rec == nil {

+ t.Fatalf("read the record: %v", err)

+ }

+ return *rec

+// setClaim rewrites the claim under the raw bucket, as a box would.

+func (w *movePassWorld) setClaim(t *testing.T, mut func(*hostclaim.Claim)) {

+ t.Helper()

+ ctx := context.Background()

+ obs, err := hostclaim.Read(ctx, w.fx.rel, "acme.go.tito.io")

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(ctx, w.fx.rel, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil {

+ t.Fatalf("rewrite the claim: %v", err)

+ }

+func (w *movePassWorld) startEffect(t *testing.T) hostops.Effect {

+ t.Helper()

+ a, err := hostops.ReadAdmission(context.Background(), w.fx.rel, "bab", "titogo", claimMoveAppID)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return a.Value.Effects[w.id]

+// setStartEffect rewrites the move-start effect of the admission object.

+func (w *movePassWorld) setStartEffect(t *testing.T, mut func(*hostops.Effect)) {

+ t.Helper()

+ ctx := context.Background()

+ body, etag, err := w.fx.rel.Get(ctx, "apps/bab/titogo/admission.json")

+ if err != nil {

+ t.Fatal(err)

+ }

+ var a hostops.AppAdmission

+ if err := json.Unmarshal(body, &a); err != nil {

+ t.Fatal(err)

+ }

+ e := a.Effects[w.id]

+ mut(&e)

+ a.Effects[w.id] = e

+ if body, err = json.Marshal(a); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := w.fx.rel.Put(ctx, "apps/bab/titogo/admission.json", body, etag); err != nil {

+ t.Fatal(err)

+ }

+func (w *movePassWorld) paged(t *testing.T) []pagePayload {

+ t.Helper()

+ w.mu.Lock()

+ defer w.mu.Unlock()

+ return append([]pagePayload(nil), w.pages...)

+func startedMovePass(t *testing.T, code string) *movePassWorld {

+ t.Helper()

+ fx, _, _, id := j17StartedMove(t, code)

+ return newMovePassWorld(t, fx, id)

+func landingMovePass(t *testing.T, code string) *movePassWorld {

+ t.Helper()

+ fx, _, _ := j4LandingMove(t, code)

+ return newMovePassWorld(t, fx, j17MoveState(t, fx).ID)

+func newRollbackID(t *testing.T) string {

+ t.Helper()

+ id, err := moves.NewRollbackID(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return id

+// TestRecoveryPassRollsBackAtTheLandingDeadline: a landing past the earlier

+// of its deadlines is rolled back by the record (one pass), and the claim

+// follows on the next (the rollback half). Each pass writes one object.

+func TestRecoveryPassRollsBackAtTheLandingDeadline(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-deadline")

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveLanding {

+ t.Fatalf("rolled back before the deadline: %+v", m)

+ }

+ past := time.Now().UTC().Add(-time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move.LandingDeadline = &past })

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move.State != provision.MoveRollingBack || rec.Move.Rollback == nil || rec.Move.Rollback.Reason != provision.RollbackLandingDeadline || rec.Region != "ie" {

+ t.Fatalf("the deadline's rollback: region %s %+v", rec.Region, rec.Move)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the pass that wrote the record wrote the claim too")

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindRestoring || c.Op.ID != w.id {

+ t.Fatalf("the claim did not follow the rollback: %+v", c.Op)

+ }

+ if after := w.record(t); after.Move.Rollback.ID != rec.Move.Rollback.ID || after.RecordSeq != rec.RecordSeq {

+ t.Fatalf("the claim's pass wrote the record: %+v", after.Move)

+ }

+// TestRecoveryPassSettlesAnOrphanStartThenReleases (K3): the request door's

+// record write was lost, so the claim is moving/"" and the record carries no

+// move, with the start effect uncertain. Under ten minutes the pass leaves

+// it. After, it settles the start not-applied from record_seq (no handle),

+// and on the next pass releases the claim to serving under the source. A

+// landing orphan is paged once and never released.

+func TestRecoveryPassSettlesAnOrphanStartThenReleases(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-orphan")

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move = nil })

+ w.setStartEffect(t, func(e *hostops.Effect) { e.State, e.ResultReference = "uncertain", "" })

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving || w.startEffect(t).State != "uncertain" {

+ t.Fatalf("acted inside ten minutes: claim %+v effect %+v", c.Op, w.startEffect(t))

+ }

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.StartedAt = time.Now().UTC().Add(-11 * time.Minute) })

+ w.pass(t)

+ if e := w.startEffect(t); e.State != "complete" || e.ResultReference != "not-applied" {

+ t.Fatalf("the start was not settled: %+v", e)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving {

+ t.Fatalf("the settling pass released too: %+v", c.Op)

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(c) != nil {

+ t.Fatalf("the orphan was not released: %+v", c.Op)

+ }

+ if p := w.paged(t); len(p) != 0 {

+ t.Fatalf("a moving orphan paged: %+v", p)

+ }

+ // A landing orphan pages once, and is left.

+ l := landingMovePass(t, "code-mvpass-orphan-landing")

+ j17SetRecord(t, l.fx, func(r *provision.Record) { r.Move = nil })

+ before := claimBytes(t, l.fx, "acme.go.tito.io")

+ l.pass(t)

+ if claimBytes(t, l.fx, "acme.go.tito.io") != before {

+ t.Fatal("a landing orphan's claim was written")

+ }

+ p := l.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelIncident || !strings.Contains(p[0].Subject, "acme.go.tito.io") || !strings.Contains(p[0].Body, l.id) {

+ t.Fatalf("the landing orphan's pages: %+v", p)

+ }

+// oneBoxFenceAuthority is c7MoveAuthority with the one-box-region check the

+// grant's Convert needs (the console's hostfence.ClaimFenceAuthority has

+// both).

+type oneBoxFenceAuthority struct{ c7MoveAuthority }

+func (oneBoxFenceAuthority) CheckOneBoxRegion(context.Context, string, string) error { return nil }

+// TestRecoveryPassConvertsAFencedSourceInTwoPasses (K13): a source

+// terminated before the flip. The first pass writes the record's relocation

+// decision, the second the claim's conversion to relocating.

+func TestRecoveryPassConvertsAFencedSourceInTwoPasses(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-convert")

+ w.auth = oneBoxFenceAuthority{c7MoveAuthority{region: "ie"}}

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MovePreparing {

+ t.Fatalf("converted with no fence: %+v", m)

+ }

+ terminatedSourceFence(t, w.fx, "ie-box", "uk-box")

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveRelocating {

+ t.Fatalf("the decision pass: %+v", m)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving {

+ t.Fatalf("the decision pass converted the claim: %+v", c.Op)

+ }

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != "pinning" || c.Op.ID != w.id {

+ t.Fatalf("the conversion pass: %+v", c.Op)

+ }

+ w.pass(t) // a relocating claim is region HA's: nothing

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRelocating {

+ t.Fatalf("after the conversion: %+v", c.Op)

+ }

+// lostTargetRollback puts the landing world into a rollback whose target

+// box was lost and released: the source recovered and opened, the claim

+// restoring/opened under the source.

+func lostTargetRollback(t *testing.T, w *movePassWorld) {

+ t.Helper()

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box"

+ c.Lost = []string{"uk-box"}

+ c.Obligations = nil

+ })

+ rb, at := newRollbackID(t), time.Now().UTC().Add(-time.Hour)

+ deadline := at.Add(15 * time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Region = "ie"

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: rb, RequestedAt: &at, Reason: provision.RollbackDeclaredLost, TargetPhase: provision.RbTargetReleased,

+ SourcePhase: provision.RbSourceOpened, RecoveredAt: &at, OpenedAt: &at, ReleaseDeadline: &deadline}

+ })

+// TestRecoveryPassStampsALostTargetThenCloses (p10 Task G): with the target

+// lost and released and nothing of its box or region owed, the pass stamps

+// target_parked_at outside the grant; the close follows once overdue, then

+// the release.

+func TestRecoveryPassStampsALostTargetThenCloses(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-lost")

+ lostTargetRollback(t, w)

+ w.pass(t)

+ rb := w.record(t).Move.Rollback

+ if rb.TargetParkedAt == nil || rb.TargetPhase != provision.RbTargetCleaned {

+ t.Fatalf("the lost target was not stamped: %+v", rb)

+ }

+ w.pass(t)

+ if w.record(t).Move == nil {

+ t.Fatal("closed inside the boxes' grace")

+ }

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "rolled_back" {

+ t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRestoring {

+ t.Fatalf("the close's pass released too: %+v", c.Op)

+ }

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

+ t.Fatalf("not released: %+v", c.Op)

+ }

+// committedAndOpened puts the landing world at a committed, opened and

+// source-parked move with nothing owed: the close is due.

+func committedAndOpened(t *testing.T, w *movePassWorld) {

+ t.Helper()

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseCommitted, nil })

+ at := time.Now().UTC().Add(-time.Hour)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move.State, r.Move.LandingPhase = provision.MoveCommitted, provision.LandProbed

+ r.Move.CommittedAt, r.Move.OpenedAt, r.Move.SourceParkedAt = &at, &at, &at

+ })

+// TestRecoveryPassFinishesTheClose: a close the boxes did not write within

+// moveCloseGrace is overdue. The pass pages it once, closes the record

+// (LastMove done) and, on the next pass, releases under the target.

+func TestRecoveryPassFinishesTheClose(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-close")

+ committedAndOpened(t, w)

+ w.pass(t)

+ if w.record(t).Move == nil || len(w.paged(t)) != 0 {

+ t.Fatalf("closed or paged inside the grace: %+v", w.paged(t))

+ }

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != w.id || rec.LastMove.Outcome != "done" {

+ t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted {

+ t.Fatalf("released in the close's pass: %+v", c.Op)

+ }

+ if p := w.paged(t); len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Body, w.id) {

+ t.Fatalf("the overdue close's page: %+v", p)

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "uk-box" || hostclaim.RequireReleasedMove(w.id)(c) != nil {

+ t.Fatalf("not released under the target: %+v", c.Op)

+ }

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again: %+v", w.paged(t))

+ }

+// TestRecordFirstCloseCrashIsFinishedByThePass (ruling M4): a box closed the

+// record and crashed before the claim's release. The record carries

+// LastMove and no move; the claim is still in the move's closing op. The

+// console's pass releases from LastMove, under the executor the outcome

+// names, with no grace (the release is the record's decision).

+func TestRecordFirstCloseCrashIsFinishedByThePass(t *testing.T) {

+ for _, c := range []struct {

+ name, outcome, exec string

+ setup func(*testing.T, *movePassWorld)

+ }{

+ {"committed", "done", "uk-box", func(t *testing.T, w *movePassWorld) {

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseCommitted, nil })

+ }},

+ {"rolled back", "rolled_back", "ie-box", func(t *testing.T, w *movePassWorld) {

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec, c.Obligations = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box", nil

+ })

+ }},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-crash-"+strings.ReplaceAll(c.name, " ", "-"))

+ c.setup(t, w)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ if c.outcome == "rolled_back" {

+ r.Region = "ie"

+ }

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: c.outcome, At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != c.exec || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+ }

+ t.Run("reverted", func(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-crash-reverted")

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Obligations = hostclaim.KindReverting, hostclaim.PhaseOpened, nil

+ if c.Op.Journal == nil {

+ c.Op.Journal = map[string]string{}

+ }

+ c.Op.Journal[hostclaim.JournalRevPhase] = hostclaim.PhaseOpened

+ })

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: "rolled_back", At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+ t.Run("aborted", func(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-crash-aborted")

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseAborting, nil })

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: "aborted", At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+// TestRecoveryPassAbortsAfterSixHours (K18): a move still preparing

+// moves.AbortAfter after its request is aborted through ConsoleMove; before

+// that the pass pages once at an hour and writes nothing.

+func TestRecoveryPassAbortsAfterSixHours(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-abort")

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ requested := w.record(t).Move.RequestedAt

+ w.advance(30 * time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 0 {

+ t.Fatalf("paged at 30 minutes: %+v", w.paged(t))

+ }

+ w.advance(time.Hour)

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MovePreparing {

+ t.Fatalf("aborted early: %+v", m)

+ }

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Subject, "acme.go.tito.io") {

+ t.Fatalf("the hour's page: %+v", p)

+ }

+ w.mu.Lock()

+ w.clock = requested.Add(moves.AbortAfter + time.Minute)

+ w.mu.Unlock()

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveAborting {

+ t.Fatalf("not aborted at six hours: %+v", m)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the abort wrote the claim (the source's abort takes it to aborting)")

+ }

+// TestRecoveryPassNeverClosesAnAbort (fix round 1, ruling 1): the source's

+// retirement of its own state is blocked, so it never closes the aborting

+// record, though nothing is owed under the op. The console does not close

+// it either: once the abort has been due past the lane's horizon (the

+// claim's quiet_after for the target region) plus moveCloseGrace, it pages

+// once, and the record stays aborting.

+func TestRecoveryPassNeverClosesAnAbort(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-abort-blocked")

+ horizon := w.clock.Add(20 * time.Minute)

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Phase, c.Obligations = hostclaim.PhaseAborting, nil

+ if c.QuietAfter == nil {

+ c.QuietAfter = map[string]time.Time{}

+ }

+ c.QuietAfter["uk"] = horizon

+ })

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move.State = provision.MoveAborting })

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ w.pass(t)

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 0 {

+ t.Fatalf("paged before the lane's horizon plus the grace: %+v", w.paged(t))

+ }

+ w.advance(20 * time.Minute) // past quiet_after + the grace

+ w.pass(t)

+ w.advance(2 * time.Hour)

+ w.pass(t)

+ if m := w.record(t).Move; m == nil || m.State != provision.MoveAborting {

+ t.Fatalf("the console closed the abort: %+v %+v", m, w.record(t).LastMove)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the console wrote the aborting claim")

+ }

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Body, w.id) || !strings.Contains(p[0].Body, "ie-box") {

+ t.Fatalf("the overdue abort's pages: %+v", p)

+ }

+// TestConsolePagesThroughAlerting (§13, M14; fix round 1, ruling 3): the

+// pass pages through alerting.Page on the armed sinks. A rollback waiting

+// past its release deadline is an incident, paged again only a full hour

+// after the last attempt: two passes either side of a whole hour since the

+// deadline do not both page. The last attempt's time is durable, in the

+// console's store, so a restarted console keeps the spacing.

+func TestConsolePagesThroughAlerting(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-pages")

+ rb, at := newRollbackID(t), w.clock.Add(-74*time.Minute)

+ deadline := at.Add(15 * time.Minute) // 59 minutes ago

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Region = "ie"

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: rb, RequestedAt: &at, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetReleasing,

+ SourcePhase: provision.RbSourceWaiting, ReleaseDeadline: &deadline}

+ })

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseClaimed })

+ first := w.clock

+ w.pass(t)

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Kind != "page" || p[0].Level != alerting.LevelIncident || !strings.Contains(p[0].Subject, "uk") || !strings.Contains(p[0].Body, w.id) {

+ t.Fatalf("the release deadline's page: %+v", p)

+ }

+ // Across the whole hour since the deadline (59 to 61 minutes): no page.

+ w.advance(2 * time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again two minutes later, across the hour since the deadline: %+v", w.paged(t))

+ }

+ // A restarted console: the in-memory state is gone, the store is not.

+ w.fx.srv.moveCloseSeen = nil

+ if last, ok, err := w.fx.st.MovePageAttempt("release-deadline acme.go.tito.io " + w.id); err != nil || !ok || !last.Equal(first.Truncate(time.Second)) {

+ t.Fatalf("the last attempt's durable time: %v %v %v, want %v", last, ok, err, first)

+ }

+ w.advance(56 * time.Minute) // 58 minutes after the first page

+ w.pass(t)

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again inside the hour: %+v", w.paged(t))

+ }

+ w.advance(3 * time.Minute) // 61 minutes after the first page

+ w.pass(t)

+ if len(w.paged(t)) != 2 {

+ t.Fatalf("not paged again a full hour after the last attempt: %+v", w.paged(t))

+ }

diff --git a/internal/console/server.go b/internal/console/server.go

index 89eb5c44e..1e5c08d1d 100644

--- a/internal/console/server.go

+++ b/internal/console/server.go

@@ -828,9 +828,18 @@ type Server struct {

claimBackendObserver func(backendEvent) // fixture recorder only

claimEntries claimInventory

claimWorkers map[string]func(context.Context)

- dbHandleKey []byte

- cfg Config

- store *Store

+ // The console's move pass (movesrecovery.go): its clock (nil is

+ // time.Now; a TEST SEAM) and when each close was first seen due.

+ movesClock func() time.Time

+ moveCloseMu sync.Mutex

+ moveCloseSeen map[string]time.Time

+ // stampLostMoveTarget is hostops.StampLostMoveTargetParked over the

+ // conditional record store, bound by ConfigureClaimRuntime with the

+ // moves switch on; nil otherwise.

+ stampLostMoveTarget func(ctx context.Context, host string, now time.Time) error

+ dbHandleKey []byte

+ cfg Config

+ store *Store

// appRegionOf and beforeClaimInsert are TEST SEAMS (nil in

// production): a store failure on the app-region read (C20), and a

// competing insert landing after the region was resolved.

diff --git a/internal/console/store.go b/internal/console/store.go

index f005f1ddc..51d0619d6 100644

--- a/internal/console/store.go

+++ b/internal/console/store.go

@@ -350,6 +350,14 @@ func (s *Store) migrate() error {

`CREATE TABLE IF NOT EXISTS edge_proof_seen (

id INTEGER PRIMARY KEY CHECK (id = 1),

seen_at INTEGER NOT NULL

+);`,

+ // The console's move pass pages (movesrecovery.go): the last

+ // attempted page per finding, so a repeating page keeps a full

+ // hour between attempts across restarts, and a once-only page is

+ // never repeated.

+ `CREATE TABLE IF NOT EXISTS move_pages (

+ rule TEXT PRIMARY KEY,

+ attempted_at INTEGER NOT NULL

);`,

} {

if _, err := s.db.Exec(stmt); err != nil {

@@ -2182,6 +2190,29 @@ ON CONFLICT(box, rule, address) DO UPDATE SET firing = excluded.firing`, box, ru

return err

}

+// ---- move pass pages (movesrecovery.go) -------------------------------------

+// MovePageAttempt is when the move pass last attempted the page for rule,

+// to the second, and whether it ever did.

+func (s *Store) MovePageAttempt(rule string) (time.Time, bool, error) {

+ var at int64

+ err := s.db.QueryRow(`SELECT attempted_at FROM move_pages WHERE rule = ?`, rule).Scan(&at)

+ if err == sql.ErrNoRows {

+ return time.Time{}, false, nil

+ }

+ if err != nil {

+ return time.Time{}, false, err

+ }

+ return time.Unix(at, 0).UTC(), true, nil

+// SetMovePageAttempt records an attempt at the page for rule at.

+func (s *Store) SetMovePageAttempt(rule string, at time.Time) error {

+ _, err := s.db.Exec(`INSERT INTO move_pages (rule, attempted_at) VALUES (?, ?)

+ON CONFLICT(rule) DO UPDATE SET attempted_at = excluded.attempted_at`, rule, at.Unix())

+ return err

// ---- grants (the console's Keymail access tokens) ---------------------------

// SetGrantToken stores the Keymail access token the user's OAuth approval

diff --git a/internal/console/store_gate.go b/internal/console/store_gate.go

index aac845c62..c2d72d223 100644

--- a/internal/console/store_gate.go

+++ b/internal/console/store_gate.go

@@ -209,6 +209,7 @@ var claimSQLTables = map[string]claimSQLTable{

"memberships": {NonApp: "account membership, keyed by account and address; names no app"},

"invites": {NonApp: "account invites, keyed by account and address; names no app"},

"alert_seen": {NonApp: "operator alert memory, keyed by box, rule and address"},

+ "move_pages": {NonApp: "the move pass's last page attempt per finding (rule names the host and the move); no app column"},

"edge_proof_seen": {NonApp: "single-row edge proof marker"},

"cli_auth_requests": {NonApp: "device-code logins in flight, keyed by code hash"},

"step_ups": {NonApp: "passkey step-up marks, keyed by session hash"},

diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt

index 96aaeeb23..8e7d07046 100644

--- a/internal/console/testdata/claim-authority-sites.txt

+++ b/internal/console/testdata/claim-authority-sites.txt

@@ -92,6 +92,7 @@ claim_runtime.go|ConfigureClaimRuntime|hostops.NewControlWriter(raw, s.cfg.Relea

claim_runtime.go|ConfigureClaimRuntime|hostops.NewObservedMutationBoundary(true, records, check, s.claimMutationObserver)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewProductionMutationCheck(raw, control)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewVersionedBoundary(true, s.cfg.DBVersions, raw, control, s.cfg.Bucket, dataBuckets, s.claimMutationObserver)

+claim_runtime.go|ConfigureClaimRuntime|hostops.StampLostMoveTargetParked(ctx, records, host, now)

claim_runtime.go|sweepOneClaimAdmission|c.Control.RecoverSQLTransaction(ctx, app.Account, app.Name, appID, id, s.store)

claim_sql_auto.go|withAutoAppSQL|s.WithAppMutation(pctx, []sqlGrant{permitGrant(pctx, s.claimBucket, acct.Sqid, scope)}, call)

claim_sql_grants.go|WithAppMutation|p.BindTransaction(database)

@@ -105,6 +106,13 @@ geoippublish.go|Commit|os.Open(st.path)

hostclaimbackfill.go|RunClaimRecoveryPass|a.Control.RunRecovery(ctx, s.runClaimRecoveryPassBody)

hostwakereset.go|registerWakeRetryReset|a.Control.RegisterStart(ctx, h.Account, h.App, reg.AppID, id, reg, registeredAt)

hostwakereset.go|registerWakeRetryReset|start.FinishWakeRetryReset(cleanup)

+movesrecovery.go|movesRecovery|a.Control.ConsoleMove(ctx, co, hostops.ConsoleMoveAbort)

+movesrecovery.go|movesRecovery|g.Claim(ctx, d.Claim)

+movesrecovery.go|movesRecovery|g.Convert(ctx, *in.Fence)

+movesrecovery.go|movesRecovery|g.Record(ctx, d.Record)

+movesrecovery.go|movesRecovery|g.ResolveStart(ctx)

+movesrecovery.go|movesRecovery|s.stampLostMoveTarget(ctx, o.Host, now)

scanpass.go|storeLister|objstore.S3ClientFor(ctx, p.Region, p.Endpoint)

store.go|Open|sql.Open("sqlite", path)

store_gate.go|Connect|k.g.d.Open(k.g.dsn)

diff --git a/internal/host/movesabort.go b/internal/host/movesabort.go

index 31fed78b2..5453eb862 100644

--- a/internal/host/movesabort.go

+++ b/internal/host/movesabort.go

@@ -49,13 +49,16 @@ import (

// release, and is a no-op once everything is gone: nothing infers from

// a missing journal that it already ran, and any error (a listing, a

// delete, the journal directory) ends the pass unreleased;

-// 5. it waits until no obligation under the op remains, then releases with

-// hostclaim.ReleaseAbort (K7: from moving/aborting only, so never before

-// the lane park was queued), and last clears the record with

-// provision.MoveGuardedAbortComplete (§7.5: the claim releases first,

-// the record clears second). A round that finds the claim already

-// serving and recording this move's release completes the record; any

-// other serving claim is left alone.

+// 5. it waits until no obligation under the op remains, then clears the

+// record with provision.MoveGuardedAbortComplete, and last releases

+// with hostclaim.ReleaseAbort (K7: from moving/aborting only, so never

+// before the lane park was queued). Record first, then the claim's

+// release (§7.5 and §19 K8 as Paul's ruling M4 amends them): a crash

+// between leaves the record at rest and the claim moving/aborting with

+// nothing owed, and the console's recovery pass releases from LastMove

+// (Plan 5 Task 7). A round that finds the claim already serving and

+// recording this move's release (a binary that closed claim first)

+// completes the record; any other serving claim is left alone.

// The target region's box, for a record aborting whose To is its region:

// movesParticipantRecord answers it as the target on the state alone (the

@@ -105,6 +108,7 @@ func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec prov

}

op := obs.Claim.Op

mine := op.Kind == hostclaim.KindMoving && op.ID == id

+ releaseAfter := false

switch {

case mine && op.Exec != cfg.Box:

w.notice(rec.Host+"#abort", "moves: %s: aborting move %s: the claim's executor is %q, not this box; left for the console pass", rec.Host, id, op.Exec)

@@ -122,19 +126,23 @@ func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec prov

}

fallthrough

case mine && op.Phase == hostclaim.PhaseAborting:

- // Every pass, before the release: the retirement is idempotent, and

- // a release never follows from inferring that it already ran (fix

+ // Every pass, before the close: the retirement is idempotent, and

+ // a close never follows from inferring that it already ran (fix

// round 1). Any error, the journal directory's included, ends the

- // pass with the claim still aborting.

+ // pass with the record and the claim still aborting.

if err := w.retireOwnMoveState(ctx, rec); err != nil {

return err

}

- if released, err := w.releaseAbort(ctx, s, rec, obs); err != nil || !released {

- return err

+ // Nothing owed under the op, checked before the record's close

+ // (ruling M4), as the console's grant checks it.

+ if w.abortOwes(rec, obs) {

+ return nil

}

+ releaseAfter = true

case hostclaim.RequireReleasedMove(id)(obs.Claim) == nil:

- // This abort released the claim before a crash: finish. The own

- // state went before the release; retiring it again is a no-op.

+ // A binary that closed claim first released before a crash:

+ // finish the record. The own state went before the release;

+ // retiring it again is a no-op.

if err := w.retireOwnMoveState(ctx, rec); err != nil {

return err

}

@@ -157,7 +165,13 @@ func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec prov

}

nudgeSerial(ctx, s)

log.Printf("moves: %s: move %s aborted; the instance stays in %q", rec.Host, id, rec.Region)

- return nil

+ if !releaseAfter {

+ return nil

+ }

+ if err := w.abFault("after-abort-record"); err != nil {

+ return err

+ }

+ return w.releaseAbort(ctx, s, rec)

}

// abortClaim is steps 1 to 3: the staging lease, the journalled bound and

@@ -230,23 +244,27 @@ func (w *movesWorker) abortClaim(ctx context.Context, s release.Store, rec provi

return true, nil

}

-// releaseAbort is step 5: the release once no obligation of the op remains.

-// released is false while the lane park is still owed.

-func (w *movesWorker) releaseAbort(ctx context.Context, s release.Store, rec provision.Record, obs hostclaim.Observed) (bool, error) {

+// abortOwes is step 5's wait: true, with a notice, while an obligation of

+// the op remains (the lane park the target region has not acknowledged).

+func (w *movesWorker) abortOwes(rec provision.Record, obs hostclaim.Observed) bool {

id := rec.Move.ID

for _, o := range obs.Claim.Obligations {

if o.Op == id {

w.notice(rec.Host+"#abort-wait", "moves: %s: aborting move %s waits for %s to park the staging lane (%s %s)", rec.Host, id, rec.Move.To, o.Kind, o.Region+o.Box)

- return false, nil

+ return true

}

- if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseAbort(id, w.cfg.Box)); err != nil {

- return false, fmt.Errorf("release the host claim: %w", err)

- }

- if err := w.abFault("after-release-abort"); err != nil {

- return false, err

+ return false

+// releaseAbort is step 5's second half: the record already says aborted, so

+// hostclaim.ReleaseAbort from a fresh read. A failure is left to the

+// console's recovery pass, which releases from LastMove.

+func (w *movesWorker) releaseAbort(ctx context.Context, s release.Store, rec provision.Record) error {

+ if err := releaseAfterClose(ctx, s, rec.Host, rec.Move.ID, hostclaim.ReleaseAbort(rec.Move.ID, w.cfg.Box)); err != nil {

+ return err

}

- return true, nil

+ return w.abFault("after-release-abort")

}

// retireOwnMoveState is step 4: the move's snapshots in this box's own

diff --git a/internal/host/movesclose_order_test.go b/internal/host/movesclose_order_test.go

new file mode 100644

index 000000000..5182afef2

--- /dev/null

+++ b/internal/host/movesclose_order_test.go

@@ -0,0 +1,135 @@

+package host

+import (

+ "context"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/provision"

+// Paul's ruling M4 (2026-10-10; spec §7.5, §19 K8): every move close is

+// record first, then the claim's release. Each box close checks that nothing

+// is owed under the op, clears the record's move into LastMove, and only then

+// releases the claim. A crash between the two halves leaves the record closed

+// and the claim still under the move; the box pass is driven by records with

+// a live move, so it never comes back, and the console's recovery pass

+// releases from LastMove (internal/console, TestRecordFirstCloseCrashIsFinishedByThePass).

+//

+// Each case crashes at "after-close-record", the seam between the halves,

+// and asserts the crash state: the record at rest with the outcome, the

+// claim still in the move's closing op with nothing owed, and a further

+// pass of every box writes nothing.

+// closedButHeld asserts the state between the halves of a record-first

+// close: the record closed with outcome, the claim still at kind/phase under

+// the move with nothing owed under it.

+func closedButHeld(t *testing.T, rec provision.Record, c *hostclaim.Claim, outcome, kind, phase string) {

+ t.Helper()

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != p1ID || rec.LastMove.Outcome != outcome {

+ t.Fatalf("the record between the halves: move %+v last %+v, want closed %s", rec.Move, rec.LastMove, outcome)

+ }

+ if c.Op.ID != p1ID || c.Op.Kind != kind || c.Op.Phase != phase {

+ t.Fatalf("the claim between the halves: %+v, want %s/%q under %s (the release is second)", c.Op, kind, phase, p1ID)

+ }

+ for _, o := range c.Obligations {

+ if o.Op == p1ID {

+ t.Fatalf("the record closed while %s %s%s was still owed under the op", o.Kind, o.Region, o.Box)

+ }

+func TestBoxClosesAreRecordFirst(t *testing.T) {

+ t.Run("committed close (movesparks.go)", func(t *testing.T) {

+ h := newCloseHarness(t)

+ h.land(t)

+ h.sourcePass(t)

+ h.clock.Advance(24 * time.Hour)

+ if err := h.regionalAck(t, h.chore(t, "ie", false)); err != nil {

+ t.Fatal(err)

+ }

+ h.landAt = "after-close-record"

+ h.crashLand(t)

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "done", hostclaim.KindLanding, hostclaim.PhaseCommitted)

+ gen := h.claim(t).Gen

+ h.freshTarget()

+ h.land(t)

+ h.sourcePass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

+ t.Run("abort (movesabort.go)", func(t *testing.T) {

+ h := newAbortHarness(t)

+ h.abort(t)

+ h.abAt = "after-abort-record"

+ for i := 0; i < 6 && !h.abDone; i++ {

+ movesPass(context.Background(), h.s, h.w())

+ h.tPass(t)

+ if o := h.laneChore(t); o != nil {

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ }

+ if !h.abDone {

+ t.Fatalf("never reached after-abort-record (%v)", h.abHits)

+ }

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "aborted", hostclaim.KindMoving, hostclaim.PhaseAborting)

+ h.ownStateGone(t)

+ gen := h.claim(t).Gen

+ h.worker1 = nil

+ h.pass(t)

+ h.tPass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

+ t.Run("recovery close (movesrecover.go)", func(t *testing.T) {

+ h := newRecoverHarness(t, false)

+ h.released(t)

+ h.srcPass(t)

+ h.opened(t)

+ h.rcAt = "after-close-record"

+ h.closeOut(t)

+ if !h.rcDone {

+ t.Fatalf("never reached after-close-record (%v)", h.rcHits)

+ }

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "rolled_back", hostclaim.KindRestoring, hostclaim.PhaseOpened)

+ if n := strings.Count(strings.Join(h.sEvents, " "), "rolled_back"); n != 1 {

+ t.Fatalf("the terminal event %d times: %v", n, h.sEvents)

+ }

+ gen := h.claim(t).Gen

+ h.sw = nil

+ h.srcPass(t)

+ h.rollbackPass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

+ t.Run("revert close (movesrevert.go)", func(t *testing.T) {

+ h := newRevertHarness(t)

+ h.fromStopped(t)

+ h.arm(t)

+ h.pass(t)

+ h.reopened(t)

+ h.rvAt = "after-close-record"

+ o := h.laneChore(t)

+ h.tPass(t)

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ h.pass(t)

+ if !h.rvDone {

+ t.Fatalf("never reached after-close-record (%v)", h.rvHits)

+ }

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "rolled_back", hostclaim.KindReverting, hostclaim.PhaseOpened)

+ gen := h.claim(t).Gen

+ h.freshWorker()

+ h.pass(t)

+ h.tPass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

diff --git a/internal/host/movesevents_test.go b/internal/host/movesevents_test.go

index 72540ec9f..44bda05aa 100644

--- a/internal/host/movesevents_test.go

+++ b/internal/host/movesevents_test.go

@@ -12,18 +12,18 @@ import (

"github.com/carlosframework/platform/internal/release"

)

-const evMoveID = "mv_abcdefghijklmnopqrstuvwxyz"

+const mvEvMoveID = "mv_abcdefghijklmnopqrstuvwxyz"

// evRec is a record whose move is in the given state, edited by mut.

-func evRec(state string, mut func(*provision.MoveState)) provision.Record {

- m := &provision.MoveState{ID: evMoveID, Incarnation: strings.Repeat("ab", 32), From: "ie", To: "uk", Cut: "auto", State: state, RequestedAt: time.Unix(1_800_000_000, 0).UTC()}

+func mvEvRec(state string, mut func(*provision.MoveState)) provision.Record {

+ m := &provision.MoveState{ID: mvEvMoveID, Incarnation: strings.Repeat("ab", 32), From: "ie", To: "uk", Cut: "auto", State: state, RequestedAt: time.Unix(1_800_000_000, 0).UTC()}

if mut != nil {

mut(m)

}

return provision.Record{Account: "bab", App: "titogo", Host: "acme.go.tito.io", Region: "ie", Move: m}

}

-func evAt(sec int64) *time.Time {

+func mvEvAt(sec int64) *time.Time {

t := time.Unix(1_800_000_000+sec, 0).UTC()

return &t

}

@@ -33,13 +33,13 @@ func evAt(sec int64) *time.Time {

// pass (requested); every later record is a box writer's.

func doneFlow() []provision.Record {

cut := func(phase string) provision.Record {

- return evRec(provision.MoveCutting, func(m *provision.MoveState) {

- m.CutPhase, m.CutAt, m.CutBudgetS = phase, evAt(10), 45

+ return mvEvRec(provision.MoveCutting, func(m *provision.MoveState) {

+ m.CutPhase, m.CutAt, m.CutBudgetS = phase, mvEvAt(10), 45

})

}

land := func(phase string, attempts int) provision.Record {

- r := evRec(provision.MoveLanding, func(m *provision.MoveState) {

- m.CutPhase, m.CutAt, m.CutBudgetS, m.FlippedAt, m.LandingPhase = provision.CutFlipped, evAt(10), 45, evAt(20), phase

+ r := mvEvRec(provision.MoveLanding, func(m *provision.MoveState) {

+ m.CutPhase, m.CutAt, m.CutBudgetS, m.FlippedAt, m.LandingPhase = provision.CutFlipped, mvEvAt(10), 45, mvEvAt(20), phase

if attempts > 0 {

m.Landing = &provision.MoveLandingJournal{Attempts: attempts}

}

@@ -48,8 +48,8 @@ func doneFlow() []provision.Record {

return r

}

committed := func(mut func(*provision.MoveState)) provision.Record {

- r := evRec(provision.MoveCommitted, func(m *provision.MoveState) {

- m.CutPhase, m.CutAt, m.FlippedAt, m.CommittedAt = provision.CutFlipped, evAt(10), evAt(20), evAt(40)

+ r := mvEvRec(provision.MoveCommitted, func(m *provision.MoveState) {

+ m.CutPhase, m.CutAt, m.FlippedAt, m.CommittedAt = provision.CutFlipped, mvEvAt(10), mvEvAt(20), mvEvAt(40)

m.Landing = &provision.MoveLandingJournal{Attempts: 1}

if mut != nil {

mut(m)

@@ -59,10 +59,10 @@ func doneFlow() []provision.Record {

return r

}

closed := provision.Record{Account: "bab", App: "titogo", Host: "acme.go.tito.io", Region: "uk",

- LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "done", At: *evAt(60), OfflineS: 41.2}}

+ LastMove: &provision.MoveOutcome{ID: mvEvMoveID, Outcome: "done", At: *mvEvAt(60), OfflineS: 41.2}}

return []provision.Record{

- evRec(provision.MovePreparing, nil),

- evRec(provision.MoveReady, nil),

+ mvEvRec(provision.MovePreparing, nil),

+ mvEvRec(provision.MoveReady, nil),

cut(provision.CutRequested),

cut(provision.CutAdmitted), cut(provision.CutStopping), cut(provision.CutStopped), cut(provision.CutStaged), cut(provision.CutArchived),

land(provision.LandChecking, 0),

@@ -70,8 +70,8 @@ func doneFlow() []provision.Record {

land(provision.LandConfigured, 0), land(provision.LandRestored, 0), land(provision.LandProbing, 0), land(provision.LandProbing, 1),

land(provision.LandProbed, 1),

committed(nil),

- committed(func(m *provision.MoveState) { m.OpenedAt = evAt(51) }),

- committed(func(m *provision.MoveState) { m.OpenedAt, m.SourceParkedAt = evAt(51), evAt(55) }),

+ committed(func(m *provision.MoveState) { m.OpenedAt = mvEvAt(51) }),

+ committed(func(m *provision.MoveState) { m.OpenedAt, m.SourceParkedAt = mvEvAt(51), mvEvAt(55) }),

closed,

}

@@ -113,7 +113,7 @@ func TestEveryTransitionEmitsOneEvent(t *testing.T) {

var events []string

for _, ev := range got {

events = append(events, ev.Event)

- if ev.Op != evMoveID || ev.Host != "acme.go.tito.io" || ev.Actor != "box-test" || ev.V != 1 || ev.At.IsZero() {

+ if ev.Op != mvEvMoveID || ev.Host != "acme.go.tito.io" || ev.Actor != "box-test" || ev.V != 1 || ev.At.IsZero() {

t.Errorf("event %+v", ev)

}

@@ -148,8 +148,8 @@ func TestRollbackStuckAndRetryEvents(t *testing.T) {

var got []moves.LogEvent

w := eventWorker(&got)

rb := func(target, source string, mut func(*provision.MoveState)) provision.Record {

- return evRec(provision.MoveRollingBack, func(m *provision.MoveState) {

- m.CutAt = evAt(10)

+ return mvEvRec(provision.MoveRollingBack, func(m *provision.MoveState) {

+ m.CutAt = mvEvAt(10)

m.Rollback = &provision.MoveRollback{ID: "rb_x", Reason: provision.RollbackLandingDeadline, TargetPhase: target, SourcePhase: source}

if mut != nil {

mut(m)

@@ -157,13 +157,13 @@ func TestRollbackStuckAndRetryEvents(t *testing.T) {

})

}

stuck := func(epoch uint64) provision.Record {

- return evRec(provision.MoveStuck, func(m *provision.MoveState) {

- m.CutAt = evAt(10)

+ return mvEvRec(provision.MoveStuck, func(m *provision.MoveState) {

+ m.CutAt = mvEvAt(10)

m.Rollback = &provision.MoveRollback{ID: "rb_x", TargetPhase: provision.RbTargetCleaned, SourcePhase: provision.RbSourceVerified, StuckAt: provision.RbSourceVerified}

m.StuckReason, m.RecoveryRetryEpoch = provision.StuckBaselineMismatch, epoch

})

}

- landing := evRec(provision.MoveLanding, func(m *provision.MoveState) { m.LandingPhase = provision.LandProbing })

+ landing := mvEvRec(provision.MoveLanding, func(m *provision.MoveState) { m.LandingPhase = provision.LandProbing })

seq := []provision.Record{

landing,

rb(provision.RbTargetReleasing, provision.RbSourceWaiting, nil),

@@ -182,7 +182,7 @@ func TestRollbackStuckAndRetryEvents(t *testing.T) {

w.observeMove(ctx, nil, retried)

w.logTransition(ctx, nil, retried, stuck(1))

w.logTransition(ctx, nil, stuck(1), provision.Record{Account: "bab", App: "titogo", Host: "acme.go.tito.io", Region: "ie",

- LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "rolled_back", At: *evAt(90), OfflineS: 80}})

+ LastMove: &provision.MoveOutcome{ID: mvEvMoveID, Outcome: "rolled_back", At: *mvEvAt(90), OfflineS: 80}})

var events, ids []string

for _, ev := range got {

events = append(events, ev.Event)

@@ -216,7 +216,7 @@ func TestMoveLogWriterTellsMembersOnce(t *testing.T) {

write := moveLogWriter(sink)

flow := doneFlow()

ev := func(event, identity string, fields map[string]string) moves.LogEvent {

- return moves.LogEvent{V: 1, Op: evMoveID, Host: "acme.go.tito.io", Event: event, Identity: identity, Actor: "box-uk", At: time.Now().UTC(), Fields: fields}

+ return moves.LogEvent{V: 1, Op: mvEvMoveID, Host: "acme.go.tito.io", Event: event, Identity: identity, Actor: "box-uk", At: time.Now().UTC(), Fields: fields}

}

done := flow[len(flow)-1]

write(ctx, s, done, ev("done", "done", map[string]string{"offline_s": "41.2", "from": "ie", "to": "uk"}))

@@ -225,7 +225,7 @@ func TestMoveLogWriterTellsMembersOnce(t *testing.T) {

if len(sink.events) != 1 || sink.events[0] != "bab/titogo/acme.go.tito.io info Moved. acme.go.tito.io is served from uk. It was offline for 41 seconds." {

t.Fatalf("member events %q", sink.events)

}

- prefix := moves.ReportPrefix("bab", "titogo", "acme.go.tito.io", evMoveID)

+ prefix := moves.ReportPrefix("bab", "titogo", "acme.go.tito.io", mvEvMoveID)

logged, err := moves.ReadLog(ctx, s, prefix)

if err != nil || len(logged) != 2 || logged[0].Actor != "box-uk" {

t.Fatalf("log %+v %v", logged, err)

@@ -236,12 +236,12 @@ func TestMoveLogWriterTellsMembersOnce(t *testing.T) {

// M-C8, M-C9, and for stuck a line with no operator command in it.

func TestMemberLines(t *testing.T) {

flow := doneFlow()

- rolling := evRec(provision.MoveRollingBack, func(m *provision.MoveState) {

+ rolling := mvEvRec(provision.MoveRollingBack, func(m *provision.MoveState) {

m.Rollback = &provision.MoveRollback{ID: "rb_x", Reason: provision.RollbackLandingDeadline}

})

- stuck := evRec(provision.MoveStuck, nil)

+ stuck := mvEvRec(provision.MoveStuck, nil)

rolledBack := provision.Record{Account: "bab", App: "titogo", Host: "acme.go.tito.io", Region: "ie",

- LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "rolled_back"}}

+ LastMove: &provision.MoveOutcome{ID: mvEvMoveID, Outcome: "rolled_back"}}

for _, c := range []struct {

event string

rec provision.Record

diff --git a/internal/host/movesevidence.go b/internal/host/movesevidence.go

new file mode 100644

index 000000000..20d7ee717

--- /dev/null

+++ b/internal/host/movesevidence.go

@@ -0,0 +1,315 @@

+package host

+// The moves retirement evidence provider (moves Plan 5 Task 5; rulings K10,

+// M7 and M8 of the Plan 5 ledger). The claim's regional worker

+// (hostbox.RegionalWorker) sweeps instances/<host>/ and moves/<host>/ at rest.

+// Before it sweeps it asks this provider twice:

+//

+// - VerifyRetirement, once per sweep, for the incarnation label of every

+// candidate outside an incarnation directory and for the evidence time of

+// every instances/<host> candidate. Decide labels the moves/<host>

+// candidate with the same incarnation (park/sweep.go), and the engine's

+// proof Check requires the proof to name it, so both answers come from one

+// proof.

+// - VerifyMovesSweep, per moves/<host> candidate (park/move_sweep.go). The

+// engine then stamps the listing's end as that chore's evidence, which is

+// valid only with the precondition proof (K10): the claim shows no move

+// op, and the newest retired move's close is earlier than the listing.

+//

+// The proof comes from the record's LastMove (M8, fix round 1 ruling 2): the

+// durable move id, outcome and close time, bound to the holder's

+// incarnation. It holds only when:

+//

+// - the claim is serving, so no move op is live on the host;

+// - the record is the holder's own and carries no live move;

+// - LastMove names a move closed at a real time before this clock;

+// - every moves/<host>/<op>/ object is that move's own, or was written

+// before its close.

+//

+// The claim's released_move journal is not required: any later serving op

+// replaces it (CompleteMetadataPlacement, for one). With M4's record-first

+// close, LastMove is written before the claim's release, so it is the

+// serving claim, with no move op, that proves the release finished.

+// LastMove.At follows every park of the move's lanes, and so every staging

+// horizon. A zero time is never evidence.

+//

+// VerifyRetirement answers for less than the proof covers (M7; fix round 1

+// ruling 1). The worker binds its one time to every instances/<host>

+// candidate, so it refuses unless the close is genuine evidence for all of

+// them:

+//

+// - instances/<host> residue outside the holder's live location has no

+// moves history (the archived-incarnation census is claim Plan 12's);

+// - at the live location the close covers only generations the move

+// retired: those superseded at or before the move's own admission. A

+// generation admitted from that admission on was written past the

+// close, and ordinary retirement owns it.

+//

+// Nothing registers this provider yet. Task 9a passes it to

+// RegisterProductionClaimMoveParks, which reaches VerifyMovesSweep by type

+// assertion (claimassembly.go).

+import (

+ "context"

+ "encoding/hex"

+ "errors"

+ "fmt"

+ "strings"

+ "time"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostwrite"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/park"

+ "github.com/carlosframework/platform/internal/provision"

+// ErrMovesSweepRefused is what every MovesSweepRefusal unwraps to.

+var ErrMovesSweepRefused = errors.New("host: the moves sweep is refused")

+// MovesSweepRefusal is the provider's refusal of one observation. Each one

+// skips that pass's sweep, and the next pass asks again. Anomaly marks a

+// refusal that no ordinary move produces: the history contradicts itself, or

+// something wrote where no known move could. M8 asks for one page per

+// anomaly; the provider sends none itself, and its wiring (Task 9a) pages.

+// Reason is for the agent's log, never shown to a user.

+type MovesSweepRefusal struct {

+ Host, Reason string

+ Anomaly bool

+func (e *MovesSweepRefusal) Error() string {

+ return "host: the moves sweep of " + e.Host + " is refused: " + e.Reason

+func (e *MovesSweepRefusal) Unwrap() error { return ErrMovesSweepRefused }

+// ErrNotMovesEvidence is what every NotMovesEvidence unwraps to.

+var ErrNotMovesEvidence = errors.New("host: the move's close is not evidence for this residue")

+// NotMovesEvidence is VerifyRetirement's refusal of residue at the live

+// location that the move did not retire: a generation written past the

+// move's close, or an object no generation owns. The close is not its

+// evidence, and ordinary retirement owns it. The worker binds one evidence

+// time to every instances/<host> candidate, so the provider cannot abstain

+// for one generation; this refusal skips the pass's sweep while such

+// residue stands. Reason is for the agent's log, never shown to a user.

+type NotMovesEvidence struct {

+ Host, Generation, Reason string

+func (e *NotMovesEvidence) Error() string {

+ return "host: the move's close is not evidence for " + e.Host + " generation " + e.Generation + ": " + e.Reason

+func (e *NotMovesEvidence) Unwrap() error { return ErrNotMovesEvidence }

+// movesLegacyCensus is the contract VerifyRetirement waits on for residue it

+// has no history for (M7: the legacy census is claim Plan 12's).

+const movesLegacyCensus = "claim Plan 12 archived-incarnation census for instances/<host> residue outside the live location"

+// movesRetirementEvidence implements hostbox.RegionalEvidence and

+// park.MoveSweepVerifier over the host's provisioning record.

+type movesRetirementEvidence struct {

+ Records MovesRecordReader

+ Now func() time.Time

+// movesRetired is the proof both methods answer from.

+type movesRetired struct {

+ host, incarnation, location, moveID string

+ at time.Time

+ holder hostclaim.Holder

+func (e movesRetirementEvidence) VerifyMovesSweep(ctx context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (park.MoveSweepProof, error) {

+ r, err := e.retired(ctx, o, versions)

+ if err != nil {

+ return park.MoveSweepProof{}, err

+ }

+ return park.MoveSweepProof{Host: r.host, ClaimETag: o.ETag, MoveID: r.moveID, Incarnation: r.incarnation, RetiredAt: r.at}, nil

+func (e movesRetirementEvidence) VerifyRetirement(ctx context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (string, time.Time, error) {

+ r, err := e.retired(ctx, o, versions)

+ if err != nil {

+ return "", time.Time{}, err

+ }

+ for _, v := range versions {

+ loc, ok := movesInstancesLocation(r.host, v.Key)

+ if ok && loc != r.location {

+ return "", time.Time{}, &ClaimDependencyError{Contracts: []string{movesLegacyCensus}}

+ }

+ if err := r.retiredGenerations(versions); err != nil {

+ return "", time.Time{}, err

+ }

+ return r.incarnation, r.at, nil

+// retiredGenerations refuses unless the close is evidence for every object

+// at the live location that the engine could make a candidate (fix round 1,

+// ruling 1). The marker key and the holder's current generation are the

+// live installation's. Any other generation must be one the move retired:

+// the claim must still hold the move's own admission (a prepared landing

+// admits the generation named after the move, movesprobeowners.go), the

+// generation must not be admitted at or after it (admissions are in claim

+// order and carry no time), and every version of it must predate the close.

+// A rolled-back or aborted move, or a landing that admitted a fresh name,

+// leaves no such admission, so no generation is vouched for. Objects are

+// classified as the landing's view classifies them (landingGenerationOf),

+// after the lock objects the engine itself skips (movesParkLock, fix round

+// 2): the writer's lease stands at every live location.

+func (r movesRetired) retiredGenerations(versions []lease.ObjectVersion) error {

+ h := r.holder

+ refuse := func(gen, format string, args ...any) error {

+ return &NotMovesEvidence{Host: r.host, Generation: gen, Reason: fmt.Sprintf(format, args...)}

+ }

+ moveAdmission := -1

+ for i := len(h.Admissions) - 1; i >= 0; i-- {

+ if h.Admissions[i].Generation == r.moveID {

+ moveAdmission = i

+ break

+ }

+ afterTheMove := map[string]bool{}

+ if moveAdmission >= 0 {

+ for _, a := range h.Admissions[moveAdmission:] {

+ afterTheMove[a.Generation] = true

+ }

+ for _, v := range versions {

+ if loc, ok := movesInstancesLocation(r.host, v.Key); !ok || loc != r.location || movesParkLock(r.host, v.Key) {

+ continue

+ }

+ gen, ok := landingGenerationOf(r.location, v.Key)

+ switch {

+ case !ok:

+ return refuse("", "%s belongs to no generation", v.Key)

+ case gen == landingMarkerGeneration || gen == h.Generation:

+ continue

+ case moveAdmission < 0:

+ return refuse(gen, "the claim holds no admission of %s to date it by", r.moveID)

+ case afterTheMove[gen]:

+ return refuse(gen, "it was admitted after %s's admission, so written past the close", r.moveID)

+ case v.LastModified.IsZero() || !v.LastModified.Before(r.at):

+ return refuse(gen, "%s was written after %s closed", v.Key, r.moveID)

+ }

+ return nil

+// movesInstancesLocation is the location park.Decide files an

+// instances/<host>/ key under (park's locationOf): a 64-hex first segment

+// with a child is its own incarnation location, and everything else is the

+// bare instances/<host>. ok is false outside instances/<host>/.

+func movesInstancesLocation(host, key string) (string, bool) {

+ base := "instances/" + host

+ tail, ok := strings.CutPrefix(key, base+"/")

+ if !ok {

+ return "", false

+ }

+ first, _, child := strings.Cut(tail, "/")

+ if _, err := hex.DecodeString(first); child && err == nil && len(first) == 64 {

+ return base + "/" + first, true

+ }

+ return base, true

+// retired proves the newest retired move from the claim observation, the

+// holder's record and the complete census, or refuses.

+func (e movesRetirementEvidence) retired(ctx context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (movesRetired, error) {

+ if ctx == nil || e.Records == nil || e.Now == nil {

+ return movesRetired{}, hostwrite.ErrUnavailable

+ }

+ if o.Absent || o.Claim == nil || o.ETag == "" || o.Host == "" || o.Claim.Host != o.Host {

+ return movesRetired{}, hostwrite.ErrUnavailable

+ }

+ c, host := o.Claim, o.Host

+ refuse := func(anomaly bool, format string, args ...any) (movesRetired, error) {

+ return movesRetired{}, &MovesSweepRefusal{Host: host, Reason: fmt.Sprintf(format, args...), Anomaly: anomaly}

+ }

+ if c.Holder == nil {

+ // A free or joining claim names no record, so no move history. The

+ // free-host lane proof is claim Plan 12's (fix round 1, ruling 3).

+ return movesRetired{}, &ClaimDependencyError{Contracts: []string{movesLegacyCensus}}

+ }

+ if c.Op.Kind != hostclaim.KindServing {

+ return refuse(false, "the claim is under a %s op", c.Op.Kind)

+ }

+ h := c.Holder

+ if h.Family != "instance" || h.Key != provision.RecordKey(h.Account, h.App, host) {

+ return refuse(false, "the holder is not an instance record")

+ }

+ // A moved holder is at the bare location, the activator's own lease

+ // (ruling T5-lease, movescut.go writerLease); a scoped holder is at its

+ // incarnation's. Anything else is a location the census cannot place.

+ location := strings.TrimSuffix(h.Location, "/")

+ if location != "instances/"+host && location != "instances/"+host+"/"+h.Incarnation {

+ return refuse(true, "the holder is at %s, neither the bare nor the incarnation location", h.Location)

+ }

+ rec, err := e.Records(ctx, h.Account, h.App, host)

+ if err != nil {

+ return movesRetired{}, err

+ }

+ if rec == nil {

+ return refuse(true, "the holder's record is missing")

+ }

+ inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt)

+ if err != nil || rec.Account != h.Account || rec.App != h.App || rec.Host != host || inc != h.Incarnation {

+ return refuse(true, "the record is not the holder's incarnation")

+ }

+ if rec.Move != nil {

+ return refuse(false, "the record carries the live move %s", rec.Move.ID)

+ }

+ last := rec.LastMove

+ if last == nil || last.ID == "" {

+ return refuse(false, "the record has closed no move")

+ }

+ if last.At.IsZero() || !last.At.Before(e.Now()) {

+ return refuse(true, "the close of %s has no time before now", last.ID)

+ }

+ movesRoot := "moves/" + host + "/"

+ for _, v := range versions {

+ if strings.HasPrefix(v.Key, "instances/"+host+"/") {

+ continue

+ }

+ tail, ok := strings.CutPrefix(v.Key, movesRoot)

+ if !ok {

+ return refuse(true, "the census holds %s, outside this host", v.Key)

+ }

+ if movesParkLock(host, v.Key) {

+ continue // the moves/<host> park's own lease

+ }

+ op, _, inLane := strings.Cut(tail, "/")

+ if !inLane {

+ return refuse(true, "%s is outside every lane", v.Key)

+ }

+ if op == last.ID {

+ continue

+ }

+ if v.LastModified.IsZero() || !v.LastModified.Before(last.At) {

+ return refuse(true, "lane %s was written after %s closed", op, last.ID)

+ }

+ return movesRetired{host: host, incarnation: h.Incarnation, location: location, moveID: last.ID, at: last.At.UTC(), holder: *h}, nil

+// movesParkLock mirrors the park engine's lock predicate (park's isLock over

+// locationOf, both unexported): a key is a lock exactly when it is the

+// lock.json at the root of its own park location, instances/<host>,

+// instances/<host>/<incarnation> or moves/<host>. That is the writer's

+// lease at a holder location (lease.Acquire) and a park's own lease.

+// Decide never makes it a candidate, so the close need not vouch for it.

+// TestMovesParkLockMatchesTheEngine keeps the two in step through

+// park.Decide.

+func movesParkLock(host, key string) bool {

+ loc, ok := movesInstancesLocation(host, key)

+ if !ok {

+ if !strings.HasPrefix(key, "moves/"+host+"/") {

+ return false

+ }

+ loc = "moves/" + host

+ }

+ return key == loc+"/lock.json"

diff --git a/internal/host/movesevidence_test.go b/internal/host/movesevidence_test.go

new file mode 100644

index 000000000..b6c3153b0

--- /dev/null

+++ b/internal/host/movesevidence_test.go

@@ -0,0 +1,897 @@

+package host

+import (

+ "context"

+ "errors"

+ "fmt"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/hostbox"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/lease/leasetest"

+ "github.com/carlosframework/platform/internal/park"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+// The moves retirement evidence provider (Plan 5 Task 5, rulings K10, M7,

+// M8). The fake bucket's clock starts at 2026-10-05T00:00:00Z and ticks a

+// second per write, so every object a fixture writes is older than evAt.

+var (

+ evCreated = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)

+ evAt = time.Date(2026, 10, 5, 1, 0, 0, 0, time.UTC) // LastMove.At, the close

+ evNow = time.Date(2026, 10, 5, 2, 0, 0, 0, time.UTC) // the sweep's clock

+const (

+ evHost = "ev.example"

+ evAcct = "acct"

+ evApp = "app"

+ evMoveID = "mv_retired"

+type evFixture struct {

+ inc, location string

+ claims release.Store

+ bucket *leasetest.VersionedBucket

+ rec *provision.Record

+ reads int

+// newEvFixture is a serving claim released by evMoveID, whose record closed

+// that move at evAt and carries no live move.

+func newEvFixture(t *testing.T) *evFixture {

+ t.Helper()

+ inc, err := hostclaim.Incarnation(evAcct, evApp, evHost, evCreated)

+ if err != nil {

+ t.Fatal(err)

+ }

+ f := &evFixture{inc: inc, location: "instances/" + evHost + "/" + inc, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket()}

+ f.bucket.AddBucket("data")

+ f.rec = &provision.Record{Schema: 1, Host: evHost, App: evApp, Account: evAcct, CreatedAt: evCreated,

+ LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "done", At: evAt}}

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey(evAcct, evApp, evHost), Account: evAcct, App: evApp, Incarnation: inc, CreatedAt: evCreated, Location: f.location, Stamp: 2, Session: 2, WriterBox: "box", Generation: evMoveID,

+ Admissions: evAdmissions(evMoveID)}

+ c.Op = servingReleasedBy(evMoveID)

+ c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}}

+ })

+ return f

+// evAdmissions is the holder's admission history: an origin generation

+// "old", then the move's prepared landing, which admits the generation

+// named after the move (movesprobeowners.go), then any later generations,

+// each the next one's predecessor. The last is the holder's current.

+func evAdmissions(gens ...string) []hostclaim.AdmissionRecord {

+ out := []hostclaim.AdmissionRecord{{Session: 1, Box: "box", Generation: "old", Mode: "cold", Region: "ie"}}

+ for i, g := range gens {

+ out = append(out, hostclaim.AdmissionRecord{Session: uint64(i + 2), Box: "box", Generation: g, Predecessor: out[i].Generation, Mode: "cold", Region: "ie", PredecessorRegion: "ie"})

+ }

+ return out

+// genKey is an object of generation gen at the fixture's live location.

+func (f *evFixture) genKey(gen string) string { return f.location + "/g-" + gen + "/0000/db" }

+// servingReleasedBy is the serving op hostclaim.ReleaseMove leaves.

+func servingReleasedBy(id string) hostclaim.Op {

+ return hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box", Journal: map[string]string{"released_move": id}}

+func (f *evFixture) setClaim(t *testing.T, mut func(*hostclaim.Claim)) {

+ t.Helper()

+ obs, err := hostclaim.Read(t.Context(), f.claims, evHost)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(t.Context(), f.claims, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil {

+ t.Fatal(err)

+ }

+func (f *evFixture) observe(t *testing.T) hostclaim.Observed {

+ t.Helper()

+ obs, err := hostclaim.Read(t.Context(), f.claims, evHost)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return obs

+func (f *evFixture) census(t *testing.T) []lease.ObjectVersion {

+ t.Helper()

+ var out []lease.ObjectVersion

+ for _, p := range []string{"instances/" + evHost + "/", "moves/" + evHost + "/"} {

+ vs, err := f.bucket.ListVersions(t.Context(), "data", p)

+ if err != nil {

+ t.Fatal(err)

+ }

+ out = append(out, vs...)

+ }

+ return out

+func (f *evFixture) provider() movesRetirementEvidence {

+ return movesRetirementEvidence{

+ Records: func(_ context.Context, acct, app, host string) (*provision.Record, error) {

+ f.reads++

+ if acct != evAcct || app != evApp || host != evHost {

+ return nil, fmt.Errorf("read %s/%s/%s", acct, app, host)

+ }

+ if f.rec == nil {

+ return nil, errors.New("no record")

+ }

+ cp := *f.rec

+ return &cp, nil

+ },

+ Now: func() time.Time { return evNow },

+ }

+// lane writes one object of a move's staging lane.

+func (f *evFixture) lane(op, name string) {

+ f.bucket.Put("data", "moves/"+evHost+"/"+op+"/"+name, []byte(op))

+func wantRefusal(t *testing.T, err error, anomaly bool) {

+ t.Helper()

+ var r *MovesSweepRefusal

+ if !errors.As(err, &r) {

+ t.Fatalf("err = %v, want a *MovesSweepRefusal", err)

+ }

+ if r.Host != evHost || r.Reason == "" || r.Anomaly != anomaly {

+ t.Fatalf("refusal = %+v, want host %s, a reason and anomaly %v", r, evHost, anomaly)

+ }

+// bothRefuse asserts the two methods refuse alike: VerifyRetirement's

+// label is the proof's, so neither may answer where the other refuses.

+func bothRefuse(t *testing.T, f *evFixture, anomaly bool) {

+ t.Helper()

+ obs, census := f.observe(t), f.census(t)

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil {

+ t.Fatal("VerifyMovesSweep answered")

+ } else {

+ wantRefusal(t, err, anomaly)

+ }

+ inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)

+ if err == nil || inc != "" || !at.IsZero() {

+ t.Fatalf("VerifyRetirement = %q, %v, %v; want a refusal and no evidence", inc, at, err)

+ }

+ wantRefusal(t, err, anomaly)

+// TestMovesSweepProofRetiredAtIsTheClose (M8): the proof binds the exact

+// claim observation, names LastMove's id, the holder's incarnation and the

+// close's time, and the engine's Check accepts it only for a listing that

+// started after that close.

+func TestMovesSweepProofRetiredAtIsTheClose(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.lane(evMoveID, "replica/0000/db")

+ f.bucket.Put("data", "moves/"+evHost+"/lock.json", []byte("lease"))

+ obs := f.observe(t)

+ proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t))

+ if err != nil {

+ t.Fatal(err)

+ }

+ want := park.MoveSweepProof{Host: evHost, ClaimETag: obs.ETag, MoveID: evMoveID, Incarnation: f.inc, RetiredAt: evAt}

+ if proof != want {

+ t.Fatalf("proof = %+v\nwant %+v", proof, want)

+ }

+ if err := proof.Check(obs, f.inc, evNow); err != nil {

+ t.Fatalf("a listing after the close: %v", err)

+ }

+ for _, listing := range []time.Time{evAt, evAt.Add(-time.Second)} {

+ if err := proof.Check(obs, f.inc, listing); !errors.Is(err, park.ErrUnsafe) {

+ t.Fatalf("a listing at %v, not after the close %v: %v", listing, evAt, err)

+ }

+// TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation (M7):

+// Decide labels every moves/<host> candidate with VerifyRetirement's

+// answer, and Check requires the proof's incarnation to equal it, so the

+// two answers come from one source. The evidence time is the close's, never

+// zero and never the clock's.

+func TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ obs, census := f.observe(t), f.census(t)

+ inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if inc != f.inc || !at.Equal(evAt) {

+ t.Fatalf("VerifyRetirement = %q, %v; want %q, %v", inc, at, f.inc, evAt)

+ }

+ proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if proof.Incarnation != inc {

+ t.Fatalf("label %q, proof %q", inc, proof.Incarnation)

+ }

+ if err := proof.Check(obs, inc, evNow); err != nil {

+ t.Fatal(err)

+ }

+// TestVerifyRetirementRefusesLegacyResidue (M7): residue under

+// instances/<host> outside the holder's live location has no moves-owned

+// archived-incarnation history. VerifyRetirement refuses it with a typed

+// dependency error (the legacy census is claim Plan 12's), which skips the

+// whole at-rest sweep. The moves proof itself is unaffected.

+func TestVerifyRetirementRefusesLegacyResidue(t *testing.T) {

+ for name, key := range map[string]string{

+ "a legacy level object": "instances/" + evHost + "/0000/db",

+ "a legacy generation object": "instances/" + evHost + "/g-old/0000/db",

+ "an older incarnation": "instances/" + evHost + "/" + strings.Repeat("ab", 32) + "/g-old/0000/db",

+ } {

+ t.Run(name, func(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ f.bucket.Put("data", key, []byte("legacy"))

+ obs, census := f.observe(t), f.census(t)

+ inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)

+ var dep *ClaimDependencyError

+ if !errors.As(err, &dep) || inc != "" || !at.IsZero() {

+ t.Fatalf("VerifyRetirement = %q, %v, %v; want a *ClaimDependencyError and no evidence", inc, at, err)

+ }

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err != nil {

+ t.Fatalf("the moves proof does not depend on instances residue: %v", err)

+ }

+ })

+ }

+ t.Run("an incarnation directory beside a bare holder", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.bare(t)

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ f.bucket.Put("data", "instances/"+evHost+"/"+f.inc+"/g-old/0000/db", []byte("scoped"))

+ _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t))

+ var dep *ClaimDependencyError

+ if !errors.As(err, &dep) {

+ t.Fatalf("err = %v, want a *ClaimDependencyError", err)

+ }

+ })

+// bare puts the holder at the activator's own lease location,

+// instances/<host>: the only holder a move cuts (ruling T5-lease,

+// movescut.go writerLease), so every moved host's holder.

+func (f *evFixture) bare(t *testing.T) {

+ t.Helper()

+ f.location = "instances/" + evHost

+ f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Location = f.location })

+// TestMovedHostAtABareLocationIsSwept (controller finding, Plan 12 review):

+// a moved host's holder is at the bare location, and its objects there are

+// its live location's, not residue outside it. The provider answers, and

+// the worker queues the moves/<host> lane chore.

+func TestMovedHostAtABareLocationIsSwept(t *testing.T) {

+ f := newEvFixture(t)

+ f.bare(t)

+ f.lane(evMoveID, "move.json")

+ f.lane(evMoveID, "replica/0000/db")

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ obs, census := f.observe(t), f.census(t)

+ inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)

+ if err != nil || inc != f.inc || !at.Equal(evAt) {

+ t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt)

+ }

+ proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)

+ if err != nil || proof.Incarnation != f.inc || !proof.RetiredAt.Equal(evAt) {

+ t.Fatalf("proof = %+v, %v", proof, err)

+ }

+ if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) {

+ t.Fatalf("the lane chore waits for its horizon: %v", err)

+ }

+ moves, ok := f.chores(t)["moves/"+evHost]

+ if !ok || moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) {

+ t.Fatalf("chores = %+v; want the moves/<host> chore at the listing's end", f.chores(t))

+ }

+// TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired (fix round 1,

+// ruling 1): the close is evidence only for what the move retired. At the

+// live location that is a generation superseded at or before the move's

+// own admission, every version of it older than the close. A generation

+// admitted from the move's admission on was written past the close, so the

+// provider has no evidence for it and says so with a typed error; the

+// moves lane proof is unaffected.

+func TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired(t *testing.T) {

+ answers := func(t *testing.T, f *evFixture) {

+ t.Helper()

+ inc, at, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t))

+ if err != nil || inc != f.inc || !at.Equal(evAt) {

+ t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt)

+ }

+ notEvidence := func(t *testing.T, f *evFixture, gen string) {

+ t.Helper()

+ obs, census := f.observe(t), f.census(t)

+ inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census)

+ var ne *NotMovesEvidence

+ if !errors.As(err, &ne) || inc != "" || !at.IsZero() {

+ t.Fatalf("VerifyRetirement = %q, %v, %v; want a *NotMovesEvidence and no evidence", inc, at, err)

+ }

+ if ne.Host != evHost || ne.Generation != gen || ne.Reason == "" {

+ t.Fatalf("refusal = %+v, want generation %q", ne, gen)

+ }

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err != nil {

+ t.Fatalf("the lane proof does not depend on the generations: %v", err)

+ }

+ t.Run("the current generation and the marker", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))

+ answers(t, f)

+ })

+ t.Run("a generation the move's admission superseded", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.bucket.Put("data", f.genKey("old"), []byte("old"))

+ f.bucket.Put("data", f.location+"/files/g-old/a.txt", []byte("old"))

+ answers(t, f)

+ })

+ t.Run("the move's generation superseded after the close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3

+ })

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("superseded after the close"))

+ f.bucket.Put("data", f.genKey("later"), []byte("live"))

+ notEvidence(t, f, evMoveID)

+ })

+ t.Run("a later generation superseded after the close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later", "latest"), "latest", 4, 4

+ })

+ f.bucket.Put("data", f.genKey("later"), []byte("superseded after the close"))

+ notEvidence(t, f, "later")

+ })

+ t.Run("no admission of the move in the claim", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ c.Holder.Admissions, c.Holder.Generation = evAdmissions("fresh"), "fresh"

+ })

+ f.bucket.Put("data", f.genKey("old"), []byte("old"))

+ notEvidence(t, f, "old")

+ })

+ t.Run("a retired generation written after the close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)

+ f.bucket.Put("data", f.genKey("old"), []byte("late"))

+ notEvidence(t, f, "old")

+ })

+ t.Run("an object no generation owns", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.bucket.Put("data", f.location+"/stray.bin", []byte("?"))

+ notEvidence(t, f, "")

+ })

+// TestMovesSweepProofRefusesALiveMove (K10, M8): no proof while a move op

+// is on the claim, while the record carries a live move, or while the

+// claim was released by a move the record has not closed.

+func TestMovesSweepProofRefusesALiveMove(t *testing.T) {

+ t.Run("the record carries a live move", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing}

+ f.lane("mv_next", "move.json")

+ bothRefuse(t, f, false)

+ })

+ for _, op := range []hostclaim.Op{

+ {Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"},

+ {Kind: hostclaim.KindLanding, ID: evMoveID, Phase: hostclaim.PhaseCommitted, Exec: "box"},

+ {Kind: hostclaim.KindRestoring, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"},

+ {Kind: hostclaim.KindReverting, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"},

+ } {

+ // The last three are the record-first close's window (M4): the

+ // record closed the move, the claim has not released it yet.

+ t.Run("the claim holds "+op.Kind, func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) { c.Op = op })

+ bothRefuse(t, f, false)

+ })

+ }

+ t.Run("the record names no closed move", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.LastMove = nil

+ bothRefuse(t, f, false)

+ })

+// TestMovesProofSurvivesAMetadataPlacement (fix round 1, ruling 2): the

+// durable proof is the record's LastMove, bound to the holder's

+// incarnation, with no move op on the claim. A later operation that

+// replaces the serving op (here CompleteMetadataPlacement, which drops the

+// released_move journal) leaves cleanup running, and so does a serving op

+// released by a move whose record never carried it.

+func TestMovesProofSurvivesAMetadataPlacement(t *testing.T) {

+ t.Run("a metadata placement", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ if err := hostops.BeginPlacement("pl_1", "op@example", "meta", evNow, []string{"ie"}, []string{"box"})(c); err != nil {

+ t.Fatal(err)

+ }

+ c.Op.Intent["app_id"] = "app_1"

+ if err := hostops.CompleteMetadataPlacement("pl_1", "app_1")(c); err != nil {

+ t.Fatal(err)

+ }

+ })

+ obs := f.observe(t)

+ if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Journal["released_move"] != "" {

+ t.Fatalf("the placement left op %+v", obs.Claim.Op)

+ }

+ census := f.census(t)

+ proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)

+ if err != nil || proof.MoveID != evMoveID || !proof.RetiredAt.Equal(evAt) {

+ t.Fatalf("proof = %+v, %v", proof, err)

+ }

+ if inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census); err != nil || inc != f.inc || !at.Equal(evAt) {

+ t.Fatalf("VerifyRetirement = %q, %v, %v", inc, at, err)

+ }

+ if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) {

+ t.Fatalf("the lane chore waits for its horizon: %v", err)

+ }

+ if _, ok := f.chores(t)["moves/"+evHost]; !ok {

+ t.Fatalf("chores = %+v; want the moves/<host> chore", f.chores(t))

+ }

+ })

+ t.Run("a release by a move the record never carried", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) { c.Op = servingReleasedBy("mv_orphan") })

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {

+ t.Fatalf("no lane of the orphan, so nothing unproven: %v", err)

+ }

+ f.bucket.Put("data", "moves/"+evHost+"/mv_orphan/move.json", []byte("orphan"))

+ f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)

+ bothRefuse(t, f, true)

+ })

+// TestMovesEvidenceRefusesAZeroTime (M7, M8): a zero time is never

+// evidence: not the close's, and not a lane object's.

+func TestMovesEvidenceRefusesAZeroTime(t *testing.T) {

+ t.Run("a zero close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.LastMove.At = time.Time{}

+ bothRefuse(t, f, true)

+ })

+ t.Run("a close not before the provider's clock", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.LastMove.At = evNow

+ bothRefuse(t, f, true)

+ })

+ t.Run("an older lane object with no time", func(t *testing.T) {

+ f := newEvFixture(t)

+ obs := f.observe(t)

+ census := []lease.ObjectVersion{{Key: "moves/" + evHost + "/mv_older/move.json", VersionID: "v1", IsLatest: true}}

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil {

+ t.Fatal("a zero LastModified counted as before the close")

+ } else {

+ wantRefusal(t, err, true)

+ }

+ if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); err == nil {

+ t.Fatal("VerifyRetirement answered over a lane with no time")

+ }

+ })

+// TestMovesSweepProofRefusesAnUnknownLane (M8): every moves/<host>/<op>/

+// object is the closed move's own, or older than its close. An object of

+// another op written after the close has an unknown writer.

+func TestMovesSweepProofRefusesAnUnknownLane(t *testing.T) {

+ t.Run("an older lane is covered by the close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane("mv_older", "move.json")

+ f.lane("rb_return", "replica/0000/db")

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {

+ t.Fatal(err)

+ }

+ })

+ t.Run("a lane written after the close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) // before every write

+ f.lane(evMoveID, "move.json")

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil {

+ t.Fatalf("the closed move's own lane is its own: %v", err)

+ }

+ f.lane("mv_unknown", "move.json")

+ bothRefuse(t, f, true)

+ })

+ t.Run("an object outside any lane", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.bucket.Put("data", "moves/"+evHost+"/stray.json", []byte("?"))

+ bothRefuse(t, f, true)

+ })

+ t.Run("an object outside the host's namespaces", func(t *testing.T) {

+ f := newEvFixture(t)

+ census := []lease.ObjectVersion{{Key: "moves/other.example/" + evMoveID + "/move.json", VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), census); err == nil {

+ t.Fatal("another host's object counted")

+ }

+ })

+// TestMovesEvidenceRefusesAnotherIdentity: the record must be the holder's

+// exact record, and a holderless claim has no record to prove from.

+func TestMovesEvidenceRefusesAnotherIdentity(t *testing.T) {

+ t.Run("a record of another incarnation", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.CreatedAt = evCreated.Add(time.Hour)

+ bothRefuse(t, f, true)

+ })

+ t.Run("a stale observation", func(t *testing.T) {

+ f := newEvFixture(t)

+ obs := f.observe(t)

+ obs.ETag = ""

+ if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t)); err == nil {

+ t.Fatal("answered without an etag")

+ }

+ })

+ t.Run("a free claim", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op = nil, hostclaim.Op{} })

+ obs, census := f.observe(t), f.census(t)

+ _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census)

+ var dep *ClaimDependencyError

+ if !errors.As(err, &dep) {

+ t.Fatalf("err = %v, want a *ClaimDependencyError", err)

+ }

+ if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); !errors.As(err, &dep) {

+ t.Fatalf("err = %v, want a *ClaimDependencyError", err)

+ }

+ if f.reads != 0 {

+ t.Fatalf("read %d records with no holder to name one", f.reads)

+ }

+ })

+ t.Run("no providers", func(t *testing.T) {

+ f := newEvFixture(t)

+ if _, err := (movesRetirementEvidence{}).VerifyMovesSweep(t.Context(), f.observe(t), nil); err == nil {

+ t.Fatal("answered with no record reader")

+ }

+ if _, _, err := (movesRetirementEvidence{Records: f.provider().Records}).VerifyRetirement(t.Context(), f.observe(t), nil); err == nil {

+ t.Fatal("answered with no clock")

+ }

+ })

+// The provider satisfies both slots RegisterProductionClaimMoveParks reads:

+// the regional evidence and, by type assertion, the sweep verifier.

+var (

+ _ hostbox.RegionalEvidence = movesRetirementEvidence{}

+ _ park.MoveSweepVerifier = movesRetirementEvidence{}

+// --- against the claim's regional worker, with a fake store ---

+type evLease struct{}

+func (evLease) Check(context.Context) error { return nil }

+func (evLease) Release(context.Context) error { return nil }

+func (evLease) Close() {}

+type evLeases struct{}

+func (evLeases) Acquire(context.Context, string, string) (park.HeldLease, error) {

+ return evLease{}, nil

+type evEnrollment struct{}

+func (evEnrollment) VerifyRegional(context.Context, string, string, time.Duration) error { return nil }

+// worker is hostbox.RegionalWorker over park.Engine, wired as the claim's

+// assembly wires them (host/claimassembly.go): the provider is the worker's

+// Evidence and the engine's VerifyMovesSweep. This region (ie) is not the

+// authoritative one, so the holder's location here is residue too.

+func (f *evFixture) worker(t *testing.T, listing time.Time) hostbox.RegionalWorker {

+ t.Helper()

+ seq := 0

+ p := f.provider()

+ location := f.location

+ engine := park.Engine{

+ Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{},

+ Now: func() time.Time { return listing },

+ NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) },

+ View: func(_ context.Context, o hostclaim.Observed) (park.View, error) {

+ return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk",

+ GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil

+ },

+ Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil },

+ VerifyMovesSweep: p.VerifyMovesSweep,

+ }

+ return hostbox.RegionalWorker{

+ Regional: hostbox.Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }},

+ Enrollment: evEnrollment{},

+ Evidence: p,

+ }

+func (f *evFixture) chores(t *testing.T) map[string]hostclaim.ParkScope {

+ t.Helper()

+ out := map[string]hostclaim.ParkScope{}

+ for _, o := range f.observe(t).Claim.Obligations {

+ if o.Park == nil {

+ t.Fatalf("an unscoped chore: %+v", o)

+ }

+ out[o.Park.Location] = *o.Park

+ }

+ return out

+// TestAllowedSweepQueuesTheMovesChore: with the proof, the worker queues the

+// moves/<host> chore under the serving session, labelled with the holder's

+// incarnation, its evidence the listing's end, so it waits for its horizon.

+// The holder location's residue in this non-authoritative region takes the

+// close as its evidence: that horizon has passed, so the same pass parks it.

+func TestAllowedSweepQueuesTheMovesChore(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.lane("mv_older", "replica/0000/db")

+ left := f.genKey("old") // superseded by the move's own admission

+ f.bucket.Put("data", left, []byte("left behind"))

+ err := f.worker(t, evNow).Run(t.Context())

+ if !errors.Is(err, park.ErrPending) {

+ t.Fatalf("a new sweep waits for its horizon: %v", err)

+ }

+ chores := f.chores(t)

+ moves, ok := chores["moves/"+evHost]

+ if !ok || len(chores) != 1 {

+ t.Fatalf("chores = %+v; want the moves chore alone", chores)

+ }

+ if moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) || moves.RequestLimit != hostbox.RequestLimit {

+ t.Fatalf("moves chore = %+v; want incarnation %s, evidence %v", moves, f.inc, evNow)

+ }

+ if vs, err := f.bucket.ListVersions(t.Context(), "data", left); err != nil || len(vs) != 0 {

+ t.Fatalf("the location's residue was not parked: %v, %v", vs, err)

+ }

+ if vs, err := f.bucket.ListVersions(t.Context(), "data", "parked/"+f.inc+"/"+evHost+"/"); err != nil || len(vs) == 0 {

+ t.Fatalf("no archive under the holder's incarnation: %v, %v", vs, err)

+ }

+ for _, o := range f.observe(t).Claim.Obligations {

+ if o.Op != "session:2" || o.Owner != "region:ie" {

+ t.Fatalf("chore %+v is not the serving session's regional chore", o)

+ }

+ if c := f.observe(t).Claim; c.Op.Kind != hostclaim.KindServing || c.Op.Journal["released_move"] != evMoveID {

+ t.Fatalf("the sweep changed the serving op: %+v", c.Op)

+ }

+// TestSweepSkipsWhileAMoveStages (Review Focus 3, K10): while a move op is

+// on the claim the worker never asks for evidence; while the record carries

+// a live move under a serving claim the provider refuses and nothing is

+// queued.

+func TestSweepSkipsWhileAMoveStages(t *testing.T) {

+ t.Run("a moving claim", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"}

+ })

+ f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing}

+ f.lane("mv_next", "replica/0000/db")

+ if err := f.worker(t, evNow).Run(t.Context()); err != nil {

+ t.Fatal(err)

+ }

+ if len(f.chores(t)) != 0 || f.reads != 0 {

+ t.Fatalf("chores %+v, record reads %d", f.chores(t), f.reads)

+ }

+ })

+ t.Run("a live move on the record", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing}

+ f.lane("mv_next", "replica/0000/db")

+ err := f.worker(t, evNow).Run(t.Context())

+ var r *MovesSweepRefusal

+ if !errors.As(err, &r) {

+ t.Fatalf("err = %v, want the provider's refusal", err)

+ }

+ if len(f.chores(t)) != 0 {

+ t.Fatalf("chores %+v", f.chores(t))

+ }

+ })

+// TestSweepRefusedUnlessTheCloseIsBeforeTheListing (K10): the worker's own

+// Check drops the moves/<host> candidate when the newest retired move's

+// close is not earlier than the listing; the rest of the decision stands.

+func TestSweepRefusedUnlessTheCloseIsBeforeTheListing(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey("old"), []byte("left behind"))

+ err := f.worker(t, evAt).Run(t.Context())

+ if !errors.Is(err, park.ErrPending) {

+ t.Fatalf("the location chore still queues: %v", err)

+ }

+ chores := f.chores(t)

+ if _, ok := chores["moves/"+evHost]; ok || len(chores) != 1 {

+ t.Fatalf("chores = %+v; want only the location's", chores)

+ }

+ if loc := chores[f.location]; loc.Incarnation != f.inc || !loc.EvidenceAt.Equal(evAt) {

+ t.Fatalf("location chore = %+v; want the close %v as its evidence", loc, evAt)

+ }

+// TestSweepScopeMembership (witness wSweepScopeMembership, M7): residue of

+// an older incarnation is never the moves/<host> chore's. Without its own

+// evidence the provider refuses the whole sweep, so nothing is queued.

+func TestSweepScopeMembership(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", "instances/"+evHost+"/"+strings.Repeat("cd", 32)+"/g-old/0000/db", []byte("late"))

+ err := f.worker(t, evNow).Run(t.Context())

+ var dep *ClaimDependencyError

+ if !errors.As(err, &dep) {

+ t.Fatalf("err = %v, want the legacy census dependency", err)

+ }

+ if len(f.chores(t)) != 0 {

+ t.Fatalf("chores %+v", f.chores(t))

+ }

+// TestFreeHostLaneProofIsClaimPlan12s (fix round 1, ruling 3): a free claim

+// names no record, so moves can prove no closed move and the claim stays

+// free. The free-host lane proof is transferred to claim Plan 12 (confirmed

+// by platform-1e); this test pins the refusal until then.

+func TestFreeHostLaneProofIsClaimPlan12s(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op, c.Touched = nil, hostclaim.Op{}, hostclaim.Touched{} })

+ f.lane(evMoveID, "move.json")

+ err := f.worker(t, evNow).Run(t.Context())

+ var dep *ClaimDependencyError

+ if !errors.As(err, &dep) {

+ t.Fatalf("err = %v, want a *ClaimDependencyError", err)

+ }

+ if c := f.observe(t).Claim; !c.Free() || len(c.Obligations) != 0 {

+ t.Fatalf("claim = %+v", c)

+ }

+// authoritative makes ie the authoritative region, with the marker at the

+// holder's current generation and objects classified by the landing's

+// layout, as Task 9a's view will.

+func (f *evFixture) authoritative(t *testing.T, w *hostbox.RegionalWorker, current string) {

+ t.Helper()

+ marker := f.location + "/generation.json"

+ ref := park.Ref{Key: marker, VersionID: f.bucket.LatestID("data", marker)}

+ location := f.location

+ w.Regional.Engine.View = func(_ context.Context, o hostclaim.Observed) (park.View, error) {

+ return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: current, Marker: ref,

+ GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil

+ }

+// TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired (fix round 1,

+// ruling 1): in the authoritative region the engine makes superseded

+// generations park-generation candidates and binds them to

+// VerifyRetirement's time. A generation the move's admission superseded

+// gets the close; when the move's own generation was superseded after the

+// close, the provider refuses and nothing is queued.

+func TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired(t *testing.T) {

+ listing := evAt.Add(time.Minute) // before the close's horizon, so chores stay queued

+ t.Run("superseded by the move", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))

+ f.bucket.Put("data", f.genKey("old"), []byte("superseded by the move"))

+ w := f.worker(t, listing)

+ f.authoritative(t, &w, evMoveID)

+ if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) {

+ t.Fatalf("the chores wait for their horizons: %v", err)

+ }

+ var gens []hostclaim.Obligation

+ for _, o := range f.observe(t).Claim.Obligations {

+ if o.Kind == "park-generation" {

+ gens = append(gens, o)

+ }

+ if len(gens) != 1 || gens[0].Generation != "old" || gens[0].Park.Location != f.location || !gens[0].Park.EvidenceAt.Equal(evAt) {

+ t.Fatalf("generation chores = %+v; want one for old at the close %v", gens, evAt)

+ }

+ if moves, ok := f.chores(t)["moves/"+evHost]; !ok || !moves.EvidenceAt.Equal(listing) {

+ t.Fatalf("chores = %+v; want the moves chore at the listing", f.chores(t))

+ }

+ })

+ t.Run("superseded after the close", func(t *testing.T) {

+ f := newEvFixture(t)

+ f.setClaim(t, func(c *hostclaim.Claim) {

+ c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3

+ })

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey("later"), []byte("live"))

+ f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("written until after the close"))

+ w := f.worker(t, listing)

+ f.authoritative(t, &w, "later")

+ err := w.Run(t.Context())

+ var ne *NotMovesEvidence

+ if !errors.As(err, &ne) || ne.Generation != evMoveID {

+ t.Fatalf("err = %v, want a *NotMovesEvidence for %s", err, evMoveID)

+ }

+ if len(f.observe(t).Claim.Obligations) != 0 {

+ t.Fatalf("chores %+v", f.observe(t).Claim.Obligations)

+ }

+ })

+// TestMovesParkLockMatchesTheEngine (fix round 2): the provider mirrors the

+// park engine's unexported lock predicate (park's isLock over locationOf),

+// so it skips exactly the lease objects Decide skips. park.Decide is the

+// oracle: over a one-key census in a non-authoritative region, where every

+// non-lock object is a candidate, it decides nothing exactly for a lock.

+func TestMovesParkLockMatchesTheEngine(t *testing.T) {

+ f := newEvFixture(t)

+ obs := f.observe(t)

+ view := park.View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "uk",

+ GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(f.location, r.Key) }}

+ other := strings.Repeat("ab", 32)

+ for _, key := range []string{

+ "instances/" + evHost + "/lock.json",

+ "instances/" + evHost + "/" + f.inc + "/lock.json",

+ "instances/" + evHost + "/" + other + "/lock.json",

+ "moves/" + evHost + "/lock.json",

+ "moves/" + evHost + "/" + evMoveID + "/lock.json",

+ "instances/" + evHost + "/g-old/lock.json",

+ "instances/" + evHost + "/" + f.inc + "/g-old/lock.json",

+ "instances/" + evHost + "/lock.json.old",

+ "instances/" + evHost + "/0000/db",

+ } {

+ d, err := park.Decide(obs, view, []lease.ObjectVersion{{Key: key, VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}, f.inc)

+ if err != nil {

+ t.Fatalf("%s: %v", key, err)

+ }

+ if engine, mirror := len(d.Candidates) == 0, movesParkLock(evHost, key); engine != mirror {

+ t.Errorf("%s: the engine skips it %v, the provider's mirror says lock %v", key, engine, mirror)

+ }

+// TestLiveLocationLeaseIsSweptNormally (fix round 2): the writer's lease,

+// <holder.location>/lock.json (lease.Acquire), stands at every live

+// location. It is the engine's to skip, never residue the close must

+// vouch for, so the sweep proceeds, scoped and bare alike.

+func TestLiveLocationLeaseIsSweptNormally(t *testing.T) {

+ for name, bare := range map[string]bool{"scoped": false, "bare": true} {

+ t.Run(name, func(t *testing.T) {

+ f := newEvFixture(t)

+ if bare {

+ f.bare(t)

+ }

+ f.lane(evMoveID, "move.json")

+ f.bucket.Put("data", f.genKey(evMoveID), []byte("live"))

+ f.bucket.Put("data", f.location+"/generation.json", []byte("marker"))

+ f.bucket.Put("data", f.location+"/lock.json", []byte("writer lease"))

+ if _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)); err != nil {

+ t.Fatalf("VerifyRetirement refused the writer's lease: %v", err)

+ }

+ listing := evAt.Add(time.Minute)

+ w := f.worker(t, listing)

+ f.authoritative(t, &w, evMoveID)

+ if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) {

+ t.Fatalf("the lane chore waits for its horizon: %v", err)

+ }

+ chores := f.chores(t)

+ if moves, ok := chores["moves/"+evHost]; !ok || len(chores) != 1 || !moves.EvidenceAt.Equal(listing) {

+ t.Fatalf("chores = %+v; want the moves chore alone", chores)

+ }

+ if vs, err := f.bucket.ListVersions(t.Context(), "data", f.location+"/lock.json"); err != nil || len(vs) != 1 {

+ t.Fatalf("the lease was touched: %v, %v", vs, err)

+ }

+ })

+ }

diff --git a/internal/host/moveslandcommit.go b/internal/host/moveslandcommit.go

index f0410e810..f866a8c06 100644

--- a/internal/host/moveslandcommit.go

+++ b/internal/host/moveslandcommit.go

@@ -101,8 +101,8 @@ func (w *movesWorker) committedTarget(ctx context.Context, s release.Store, rec

return fmt.Errorf("read the host claim: %w", err)

}

if !obs.Absent && hostclaim.RequireReleasedMove(m.ID)(obs.Claim) == nil {

- // The close released the claim and stopped before the record's

- // half (§7.5): only that is left.

+ // A binary that closed claim first released the claim and stopped

+ // before the record's half: only that is left (closeMove).

return w.closeMove(ctx, s, rec, l.fault)

}

if err := l.commitClaim(ctx); err != nil {

diff --git a/internal/host/movesparks.go b/internal/host/movesparks.go

index 8fb5df098..252b73185 100644

--- a/internal/host/movesparks.go

+++ b/internal/host/movesparks.go

@@ -31,15 +31,17 @@ import (

// Step 5, the close (closeMove). Any participant that sees opened_at and

// source_parked_at on the record and no obligation of the op in the claim

// closes: the terminal event `done` (moves.LogEvent when the hook is wired,

-// Plan 6; a log line until then), then hostclaim.ReleaseMove to serving under

-// the target (§19 K8: the claim first), then provision.MoveClose (the record

-// last: LastMove{Outcome: "done"}, index D11). A crash between the two

-// leaves a finished move on the record that admits nothing (MoveRequest

-// refuses while Move is set), and the next pass of the target finds the

-// claim recording this move's release and writes only the record's close.

-// The target calls it every committed pass; the source calls it in the pass

-// that stamps source_parked_at, which is the last pass that routes the

-// record to it. The console's recovery pass also tries (Task 7).

+// Plan 6; a log line until then), then provision.MoveClose (the record

+// first: LastMove{Outcome: "done"}, index D11), then hostclaim.ReleaseMove

+// to serving under the target (§7.5, §19 K8 as Paul's ruling M4 amends

+// them: record first, then the claim's release). A crash between the two

+// leaves the record at rest and the claim still landing/committed with

+// nothing owed; no box pass visits a record without a move again, and the

+// console's recovery pass releases from LastMove (Recovery.Moves, Plan 5

+// Task 7). A claim found already released by this move (a binary that

+// closed claim first) gets only the record's close. The target calls it

+// every committed pass; the source calls it in the pass that stamps

+// source_parked_at, which is the last pass that routes the record to it.

// Production (K4). The staging park and the close are production-live only

// after Plan 5 Task 9 (park arming) and claim Plans 10 and 11. The committed

@@ -51,9 +53,9 @@ import (

// then the tests arm everything with fakes.

// closeMove is the close (§7.9 step 5; see the file comment). fault is the

-// caller's test seam, passed "after-close-release" between the claim's

-// release and the record's close. It does nothing until the record carries

-// both stamps and the claim owes nothing under the op.

+// caller's test seam, passed "after-close-record" between the record's close

+// and the claim's release. It does nothing until the record carries both

+// stamps and the claim owes nothing under the op.

func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provision.Record, fault func(string) error) error {

host, id, inc := rec.Host, rec.Move.ID, rec.Move.Incarnation

fresh, err := provision.GetRecord(ctx, s, rec.Account, rec.App, host)

@@ -79,7 +81,11 @@ func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provis

return fmt.Errorf("move %s has no host claim", id)

}

c := obs.Claim

- if hostclaim.RequireReleasedMove(id)(c) != nil {

+ // A claim this move already released was released by a binary that

+ // closed claim first: only the record's half is left.

+ released := hostclaim.RequireReleasedMove(id)(c) == nil

+ target := ""

+ if !released {

if c.Op.ID != id || c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted || len(c.Op.Participants) != 2 {

w.notice(host+"#close", "moves: %s: move %s waits to close: the claim is %s/%q (%s), not this move's commit", host, id, c.Op.Kind, c.Op.Phase, c.Op.ID)

return nil

@@ -87,26 +93,14 @@ func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provis

if c.Holder == nil || c.Holder.Incarnation != inc {

return fmt.Errorf("move %s: the host claim names another incarnation", id)

}

- owed := 0

- for _, o := range c.Obligations {

- if o.Op == id {

- owed++

- }

- if owed > 0 {

+ // Nothing owed under the op, checked here before the record's

+ // close (M4), as the console's grant checks it: a record closed

+ // over an owed chore would leave a release ReleaseMove refuses.

+ if owed := owedUnder(c, id); owed > 0 {

w.notice(host+"#close", "moves: %s: move %s waits for %d park(s) of the move to be acknowledged before it closes", host, id, owed)

return nil

}

- target := c.Op.Participants[1]

- if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseMove(id, target)); err != nil {

- return fmt.Errorf("release the claim: %w", err)

- }

- log.Printf("moves: %s: the claim is serving under %s after move %s", host, target, id)

- if fault != nil {

- if err := fault("after-close-release"); err != nil {

- return err

- }

+ target = c.Op.Participants[1]

}

g := provision.MoveGuard{ID: id, Incarnation: inc}

var pre provision.Record

@@ -121,5 +115,52 @@ func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provis

w.logTransition(ctx, s, pre, *closed)

log.Printf("moves: %s: move %s is done; the instance is in %s", host, id, m.To)

nudgeSerial(ctx, s)

+ if released {

+ return nil

+ }

+ if fault != nil {

+ if err := fault("after-close-record"); err != nil {

+ return err

+ }

+ if err := releaseAfterClose(ctx, s, host, id, hostclaim.ReleaseMove(id, target)); err != nil {

+ return err

+ }

+ log.Printf("moves: %s: the claim is serving under %s after move %s", host, target, id)

+ return nil

+// owedUnder counts the obligations of the claim under op id.

+func owedUnder(c *hostclaim.Claim, id string) int {

+ owed := 0

+ for _, o := range c.Obligations {

+ if o.Op == id {

+ owed++

+ }

+ return owed

+// releaseAfterClose is the second half of every box close (ruling M4): the

+// record already carries the outcome in LastMove, so the claim is released

+// from a fresh read with the op's own release transition. A claim this move

+// already released (the console's pass finished it first) is success. Any

+// other failure is returned and left to the console's recovery pass, which

+// releases from LastMove: the box pass never visits a record without a move

+// again.

+func releaseAfterClose(ctx context.Context, s release.Store, host, id string, release func(*hostclaim.Claim) error) error {

+ obs, err := hostclaim.Read(ctx, s, host)

+ if err != nil {

+ return fmt.Errorf("read the host claim to release move %s: %w", id, err)

+ }

+ if obs.Absent {

+ return fmt.Errorf("move %s has no host claim to release", id)

+ }

+ if hostclaim.RequireReleasedMove(id)(obs.Claim) == nil {

+ return nil

+ }

+ if _, err := hostclaim.Update(ctx, s, obs, release); err != nil {

+ return fmt.Errorf("release the claim after the close of move %s (the console's recovery pass finishes it): %w", id, err)

+ }

return nil

}

diff --git a/internal/host/movesparks_test.go b/internal/host/movesparks_test.go

index 53f8e434a..ec4613b75 100644

--- a/internal/host/movesparks_test.go

+++ b/internal/host/movesparks_test.go

@@ -222,11 +222,15 @@ func TestCloseWaitsForBothObligations(t *testing.T) {

}

-// TestCloseIsRecordLast (§7.5, §19 K8): the close releases the claim before

-// it clears the record. A crash between leaves a finished move on the

-// record, which admits nothing (no new move starts over it), and the next

-// pass clears it without writing the claim again.

-func TestCloseIsRecordLast(t *testing.T) {

+// TestCloseIsRecordFirst (§7.5, §19 K8 as Paul's ruling M4 amends them):

+// the close clears the record before it releases the claim. A crash between

+// leaves the record at rest with LastMove done and the claim

+// landing/committed with nothing owed; no new move can start (the claim is

+// not serving), and no box pass writes the claim again: the release is the

+// console's recovery pass (TestBoxClosesAreRecordFirst; the console's

+// TestRecordFirstCloseCrashIsFinishedByThePass). Without a crash the same

+// pass releases.

+func TestCloseIsRecordFirst(t *testing.T) {

h := newCloseHarness(t)

h.land(t)

h.sourcePass(t)

@@ -234,26 +238,36 @@ func TestCloseIsRecordLast(t *testing.T) {

if err := h.regionalAck(t, h.chore(t, "ie", false)); err != nil {

t.Fatal(err)

}

- h.landAt = "after-close-release"

+ h.landAt = "after-close-record"

h.crashLand(t)

rec := h.read(t, p1Host)

- if rec.Move == nil || rec.Move.State != provision.MoveCommitted || rec.LastMove != nil {

- t.Fatalf("the record cleared before the claim released: %+v %+v", rec.Move, rec.LastMove)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "done" {

+ t.Fatalf("the record did not close first: %+v %+v", rec.Move, rec.LastMove)

}

c := h.claim(t)

- if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "uk-box" || hostclaim.RequireReleasedMove(p1ID)(c) != nil {

+ if c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted || c.Op.ID != p1ID {

t.Fatalf("the claim at the crash point: %+v", c.Op)

}

- if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRequest("ie", h.clock.Now())); !errors.Is(err, provision.ErrMoveConflict) {

- t.Fatalf("a new move started over the finished one: %v", err)

+ if err := hostclaim.StartGuard(c, []string{"uk"}, []string{"uk-box"}); err == nil {

+ t.Fatal("a new operation could start over the unreleased claim")

}

gen := c.Gen

h.freshTarget()

h.land(t)

- h.done(t)

if after := h.claim(t).Gen; after != gen {

- t.Fatalf("the record's close wrote the claim again (%d to %d)", gen, after)

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d)", gen, after)

+ }

+ // Without the crash, the pass that closes the record releases.

+ h2 := newCloseHarness(t)

+ h2.land(t)

+ h2.sourcePass(t)

+ h2.clock.Advance(24 * time.Hour)

+ if err := h2.regionalAck(t, h2.chore(t, "ie", false)); err != nil {

+ t.Fatal(err)

}

+ h2.land(t)

+ h2.done(t)

}

// Any participant that sees both stamps and nothing owed closes (§7.9 step

@@ -262,8 +276,8 @@ func TestCloseIsRecordLast(t *testing.T) {

func TestTheSourceClosesWhenItFinishesLast(t *testing.T) {

h := newCloseHarness(t)

h.landFault = func(point string) error {

- if point == "after-close-release" {

- t.Fatal("the target released before the source parked")

+ if point == "after-close-record" {

+ t.Fatal("the target closed before the source parked")

}

return nil

}

diff --git a/internal/host/movesrecover.go b/internal/host/movesrecover.go

index d7d0f8a3c..8d1530a66 100644

--- a/internal/host/movesrecover.go

+++ b/internal/host/movesrecover.go

@@ -82,10 +82,11 @@ import (

// - opened: the finalization. Once recovered_at, opened_at and

// target_parked_at are stamped and no obligation of the op remains in

// the claim, the terminal event (LogEvent, Plan 6's moves.LogEvent),

-// the claim's release to serving (ReleaseMove) and the record's close

-// (MoveRolledBackClose). Whichever actor stamps last tries; the

-// source's next pass, and Plan 5's console pass, finish what a crash

-// leaves.

+// the record's close (MoveRolledBackClose) and then the claim's

+// release to serving (ReleaseMove): record first, ruling M4. Whichever

+// actor stamps last tries; a crash before the record's close is the

+// source's next pass, and one after it the console's recovery pass

+// (Plan 5 Task 7), which releases from LastMove.

// Every record write goes through a provision transition (the record_seq

// rule); every claim write through a hostclaim transition.

@@ -771,9 +772,12 @@ func (r *recoverRun) open(ctx context.Context) (bool, error) {

// finalize (§7.8 rolled_back): with the source opened and recovered and the

// target parked, and no obligation of the op left in the claim, the

-// terminal event, the claim's release to serving and the record's close. A

-// claim already released by this move (a crash before the close) only

-// closes.

+// terminal event, the record's close, then the claim's release to serving:

+// record first (§7.5 and §19 K8 as Paul's ruling M4 amends them). A crash

+// between the halves leaves the record at rest and the claim

+// restoring/opened with nothing owed; the console's recovery pass releases

+// from LastMove. A claim already released by this move (a binary that

+// closed claim first) only closes.

func (r *recoverRun) finalize(ctx context.Context) (bool, error) {

cfg := r.cfg()

host, id, rb := r.rec.Host, r.g.ID, r.rollback()

@@ -788,30 +792,20 @@ func (r *recoverRun) finalize(ctx context.Context) (bool, error) {

if obs.Absent {

return false, fmt.Errorf("move %s has no host claim", id)

}

- if hostclaim.RequireReleasedMove(id)(obs.Claim) != nil {

+ released := hostclaim.RequireReleasedMove(id)(obs.Claim) == nil

+ if !released {

if err := r.ours(obs.Claim); err != nil {

return false, fmt.Errorf("move %s: %w", id, err)

}

if err := r.executing(obs.Claim, hostclaim.PhaseOpened); err != nil {

return false, err

}

- owed := 0

- for _, o := range obs.Claim.Obligations {

- if o.Op == id {

- owed++

- }

- if owed > 0 {

+ // Nothing owed under the op, checked before the record's close

+ // (M4), as the console's grant checks it.

+ if owed := owedUnder(obs.Claim, id); owed > 0 {

r.w.notice(host+"#rc-final", "moves: %s: rollback %s waits for %d chore(s) of the move to be acknowledged before it closes", host, r.rb, owed)

return false, nil

}

- if _, err := hostclaim.Update(ctx, r.s, obs, hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

- return false, fmt.Errorf("release the claim: %w", err)

- }

- log.Printf("moves: %s: the claim is serving again after rollback %s", host, r.rb)

- if err := r.fault("after-release-move"); err != nil {

- return false, err

- }

}

var pre provision.Record

closed, err := provision.SetRecordMove(ctx, r.s, r.rec.Account, r.rec.App, host, observedMove(&pre, provision.MoveRolledBackClose(r.g, r.now())))

@@ -825,6 +819,18 @@ func (r *recoverRun) finalize(ctx context.Context) (bool, error) {

r.w.logTransition(ctx, r.s, pre, *closed)

log.Printf("moves: %s: move %s is rolled back and closed", host, id)

nudgeSerial(ctx, r.s)

+ if !released {

+ if err := r.fault("after-close-record"); err != nil {

+ return true, err

+ }

+ if err := releaseAfterClose(ctx, r.s, host, id, hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

+ return true, err

+ }

+ log.Printf("moves: %s: the claim is serving again after rollback %s", host, r.rb)

+ if err := r.fault("after-release-move"); err != nil {

+ return true, err

+ }

if err := r.fault("after-close"); err != nil {

return true, err

}

diff --git a/internal/host/movesrecover_test.go b/internal/host/movesrecover_test.go

index 6f354333f..ea13ad5e4 100644

--- a/internal/host/movesrecover_test.go

+++ b/internal/host/movesrecover_test.go

@@ -614,10 +614,11 @@ func TestSourceRecoveryNeverReverifiesAfterProbing(t *testing.T) {

// TestRolledBackFinalizationReleasesTheClaim (§7.8 rolled_back): once the

// source has opened and the target has parked, and no obligation of the op

-// remains, the source writes the terminal event, releases the claim to

-// serving and closes the record with the rolled_back outcome. Until every

-// chore is acknowledged it waits; a crash between the claim's release and

-// the record's close is finished by the next pass.

+// remains, the source writes the terminal event, closes the record with the

+// rolled_back outcome and releases the claim to serving, in that order

+// (ruling M4). Until every chore is acknowledged it waits; a crash between

+// the record's close and the release is the console's to finish

+// (TestBoxClosesAreRecordFirst).

func TestRolledBackFinalizationReleasesTheClaim(t *testing.T) {

ctx := context.Background()

h := newRecoverHarness(t, false)

@@ -657,11 +658,7 @@ func TestRolledBackFinalizationReleasesTheClaim(t *testing.T) {

t.Fatal(err)

}

- h.rcAt = "after-release-move"

h.srcPass(t)

- if !h.rcDone {

- t.Fatalf("never reached the claim's release (%v)", h.rcHits)

- }

c := h.claim(t)

if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

t.Fatalf("the claim was not released: %+v", c.Op)

@@ -669,8 +666,6 @@ func TestRolledBackFinalizationReleasesTheClaim(t *testing.T) {

if err := hostclaim.RequireReleasedMove(p1ID)(c); err != nil {

t.Fatal(err)

}

- h.sourceAt(t, provision.RbSourceOpened)

- h.srcPass(t)

rec := h.read(t, p1Host)

if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != p1ID || rec.LastMove.Outcome != "rolled_back" || rec.Region != "ie" {

t.Fatalf("not closed: region %s move %+v last %+v", rec.Region, rec.Move, rec.LastMove)

diff --git a/internal/host/movesrevert.go b/internal/host/movesrevert.go

index 4e06954e4..3b3600a97 100644

--- a/internal/host/movesrevert.go

+++ b/internal/host/movesrevert.go

@@ -69,13 +69,15 @@ import (

// - opened: the claim waits for the lane's park the revert queued in the

// target region (§7.11), which the target region's box runs

// (laneParkTarget) and the park engine acknowledges after its horizon.

-// Then ReleaseMove to serving under this box, and MoveRevertClose:

-// LastMove rolled_back, offline from cut_at to the reopen. The source's

-// journal of the op is parked in the same pass; a crash before that

-// leaves it to the local journal sweep (sweepRevertJournal).

+// Then MoveRevertClose (LastMove rolled_back, offline from cut_at to

+// the reopen), and then ReleaseMove to serving under this box: record

+// first, ruling M4; a crash between is finished by the console's

+// recovery pass from LastMove. The source's journal of the op is parked

+// in the same pass; a crash before that leaves it to the local journal

+// sweep (sweepRevertJournal).

-// A claim this move already released (RequireReleasedMove: a crash before the

-// record's close) only closes.

+// A claim this move already released (RequireReleasedMove: a binary that

+// closed claim first crashed before the record's close) only closes.

// The target region's half (laneParkTarget). A revert never flipped, so its

// target box has neither the row nor a journal of the move; the participant

@@ -315,8 +317,9 @@ func (r *revertRun) step(ctx context.Context) (bool, error) {

}

c := obs.Claim

if hostclaim.RequireReleasedMove(id)(c) == nil {

- // A crash between the claim's release and the record's close.

- return r.close(ctx)

+ // A binary that closed claim first crashed between the claim's

+ // release and the record's close: only the record's half is left.

+ return r.close(ctx, false)

}

if c.Op.ID != id {

return false, fmt.Errorf("move %s: the claim names op %q", id, c.Op.ID)

@@ -692,37 +695,29 @@ func (r *revertRun) probed(ctx context.Context, obs hostclaim.Observed) (bool, e

}

// opened (§7.4 opened, §7.11): once no chore of the op is left in the claim

-// (the lane's park acknowledged), the claim's release to serving under this

-// box, then the record's close.

+// (the lane's park acknowledged), the record's close, then the claim's

+// release to serving under this box (record first, ruling M4).

func (r *revertRun) opened(ctx context.Context, obs hostclaim.Observed) (bool, error) {

- cfg := r.cfg()

host, id := r.rec.Host, r.g.ID

if r.rec.Move.ReopenedAt == nil {

return false, fmt.Errorf("move %s: the claim is opened and the record has no reopen", id)

}

- owed := 0

- for _, o := range obs.Claim.Obligations {

- if o.Op == id {

- owed++

- }

- if owed > 0 {

+ // Nothing owed under the op, checked before the record's close (M4), as

+ // the console's grant checks it.

+ if owed := owedUnder(obs.Claim, id); owed > 0 {

r.w.notice(host+"#rv-opened", "moves: %s: the revert of %s waits for %d chore(s) of the move, its lane's park, to be acknowledged", host, id, owed)

return false, nil

}

- if err := r.update(ctx, obs, "release the claim", hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

- return false, err

- }

- log.Printf("moves: %s: the claim is serving again after the revert of %s", host, id)

- if err := r.fault("after-release-move"); err != nil {

- return false, err

- }

- return r.close(ctx)

+ return r.close(ctx, true)

}

-// close is the record's close (MoveRevertClose), then the source's journal of

-// the op parked.

-func (r *revertRun) close(ctx context.Context) (bool, error) {

+// close is the record's close (MoveRevertClose), then, when releaseClaim is set,

+// the claim's release to serving under this box, then the source's journal

+// of the op parked. A crash between the record's close and the release

+// leaves the claim reverting/opened with nothing owed; the console's

+// recovery pass releases it from LastMove. releaseClaim is false only for a

+// claim a claim-first binary already released.

+func (r *revertRun) close(ctx context.Context, releaseClaim bool) (bool, error) {

cfg := r.cfg()

host, id := r.rec.Host, r.g.ID

ok, err := r.record(ctx, "the revert's close", provision.MoveRevertClose(r.g, r.now()))

@@ -731,6 +726,18 @@ func (r *revertRun) close(ctx context.Context) (bool, error) {

}

log.Printf("moves: %s: move %s is reverted and closed", host, id)

nudgeSerial(ctx, r.s)

+ if releaseClaim {

+ if err := r.fault("after-close-record"); err != nil {

+ return true, err

+ }

+ if err := releaseAfterClose(ctx, r.s, host, id, hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

+ return true, err

+ }

+ log.Printf("moves: %s: the claim is serving again after the revert of %s", host, id)

+ if err := r.fault("after-release-move"); err != nil {

+ return true, err

+ }

if err := r.fault("after-close"); err != nil {

return true, err

}

diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go

index 3fa6ea9b6..e1b4e0592 100644

--- a/internal/provision/plan9_b8_census_test.go

+++ b/internal/provision/plan9_b8_census_test.go

@@ -513,6 +513,12 @@ var plan9B8CensusInventory = []censusEntry{

{"cli.go", "runRouteHibernate", "key-site", "key:SidecarManifestKey", "c151cae2cbc57943e9a5d0a5dac611823d769a6b651d5dbc2fe86182f6e098b1"},

{"cli.go", "RunStatus", "key-site", "lit:instances", "3b9936cf0f29f57f06c378a334e3118096679dfdadab7aad45b1f6c7b5841562"},

{"domains_cli.go", "runDomainsList", "key-site", "lit:instances", "f7aefeff56b0be7b50ee82bd250ada08ae9a095b7a62f577ead70846e00b61ad"},

+ // Names a dependency contract in an error message; the instances/<host> words are the data-bucket namespace, and it touches no store.

+ {"internal/host/movesevidence.go", "const movesLegacyCensus", "key-site", "lit:instances", "ccc817d41b2148d07dfdd1e780b46fbf67bcf78272c297b95f4336d83487a813"},

+ // Maps a data-bucket replica key to its park location instances/<host>[/<incarnation>]; a pure string function.

+ {"internal/host/movesevidence.go", "movesInstancesLocation", "key-site", "lit:instances", "4f79083f445c5892fb4933a696bc322eb081d0dd0c0b53c2134e8544c5a9df3e"},

+ // Reads the record through MovesRecordReader (GetRecord); RecordKey only checks the claim holder's key, and the literals only compare data-bucket locations. No write.

+ {"internal/host/movesevidence.go", "movesRetirementEvidence.retired", "key-site", "key:RecordKey,lit:instances", "d6a42cfd64728e581bb123d5743d3c74b1b6eb5abddb8972571f15cb42d84cb6"},

// Copies config acknowledgements to moves/<account>/<app>/<host>/<op>/config-ack-source.json.

{"internal/host/movesfinal.go", "sourceFinal.configAck", "nonrecord", "sel:Put", "cca53109e6945a73b7f04705d8965b3b0997f2b7d2a82fa6d86cffa4b4174e93"},

// Stamps source_parked_at through SetRecordMove, updateRecord and WriteRecord.

@@ -525,10 +531,10 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/host/moveslandcommit.go", "landingRun.probed", "record-writer", "key:SetRecordMove", "d914debf1ba5d011767fc06cb430c09e39a469dfb182f5996fa3bda806499952"},

// Stamps opened_at through SetRecordMove (MoveOpened) inside the hold lock, after the proved release.

{"internal/host/moveslandcommit.go", "landingRun.open", "record-writer", "key:SetRecordMove", "febac996d3bb9caf4021cbc4bcffe1c900bddba4287ff95f083bd5e0751e9bc2"},

- // Closes a committed move through SetRecordMove (MoveClose, LastMove done) after the claim's ReleaseMove; never writes the claim after the record.

- {"internal/host/movesparks.go", "movesWorker.closeMove", "record-writer", "key:SetRecordMove", "da86a00f3ce1b8a9d885c9b533990beca7f4f9170d8ae3966702383342828984"},

- // Closes the rollback record through SetRecordMove; claim release and local journal parking use separate storage.

- {"internal/host/movesrecover.go", "recoverRun.finalize", "record-writer", "key:SetRecordMove", "7c22716ee494d5f1c83ed2ae46fe8a08af37e12a10779208d87dc2c00471e347"},

+ // Closes a committed move through SetRecordMove (MoveClose, LastMove done) once nothing is owed under the op, then releases the claim (ReleaseMove via releaseAfterClose): record first, ruling M4. A claim already released by this move gets only the record's close.

+ {"internal/host/movesparks.go", "movesWorker.closeMove", "record-writer", "key:SetRecordMove", "380a2a0f8cc450573256f359a77da5e451ad506fef84aaeceb9696e3d2904211"},

+ // Closes the rollback record through SetRecordMove once nothing is owed under the op, then releases the claim (record first, ruling M4); local journal parking uses separate storage.

+ {"internal/host/movesrecover.go", "recoverRun.finalize", "record-writer", "key:SetRecordMove", "509591bb643d58d9314314d4d7c3fd245468502317de750d9e7b28e7a3cd2673"},

// Advances the source recovery phase through SetRecordMove and its sequenced writer.

{"internal/host/movesrecover.go", "recoverRun.phase", "record-writer", "key:SetRecordMove", "be641eaba77b940ce471696720b637b9dfb9454de0893aca1fcc40750135719d"},

// Records probe inputs through SetRecordMove; phase and stuck helpers use the same writer.

@@ -660,7 +666,8 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/host/fleetbind.go", "WriteFleetBinding", "nonrecord", "sel:Put", "44ab984c2c9b30abcc938132118f61a6922ccc0cd40ed2b6cb61c7654aa0a1c3"},

{"internal/host/logship.go", "logshipFlush", "nonrecord", "sel:Put", "a2fa594c1206028af170a2d0cbedef55ba00a2f1c698a6853e1214cc3b172dd9"},

{"internal/host/metrics_loop.go", "putAlertState", "nonrecord", "sel:Put", "d5b6e8b34dc5a731d6bbafeac0b7d0c5c20706d6f413dff38088b3321b35a26c"},

- {"internal/host/movesabort.go", "movesWorker.abortSource", "record-writer", "key:SetRecordMove", "07fe8eeb9224485c313929549234e429d5a7bf62f060e07560e0072cb3d5e667"},

+ // Clears an aborting record through SetRecordMove (MoveGuardedAbortComplete) once nothing is owed under the op, then ReleaseAbort: record first, ruling M4.

+ {"internal/host/movesabort.go", "movesWorker.abortSource", "record-writer", "key:SetRecordMove", "51bafe02fe16f07c1f1e08cfbc224ba21c8277ec065fff1142349afe0dde675b"},

{"internal/host/movesarchive.go", "cutRun.publishCutArchive", "nonrecord", "sel:CopyVersioned,sel:PutIfAbsent", "6c7fcfd2d4b2a0319ac15ecbfe003ed0292fe160457fd5704be508c2ab302a1b"},

{"internal/host/movescut.go", "cutRun.cutTransition", "record-writer", "key:SetRecordMove", "3ec34659c98aa9ec12221b30363d0e79a3b2d77b99051e0484f2a923c884c0af"},

{"internal/host/movescut.go", "publishStoppedReport", "nonrecord", "sel:Put", "c710682f31d99127ad76524add246fda9f640c63934940e45c79ea25ba36eb3c"},

diff --git a/internal/sim/hostlife/console_test.go b/internal/sim/hostlife/console_test.go

index c85851365..e0cf5fea8 100644

--- a/internal/sim/hostlife/console_test.go

+++ b/internal/sim/hostlife/console_test.go

@@ -56,13 +56,24 @@ func (w *world) execBox(c *claim) *box {

// fixRecord is the recovery owner for the record halves of a move (claim

// C2): the request's `move` field after the claim's intent; the claim half

-// of a record-first transition (followRecord); and, after a release, the

-// CAS that clears `move` and names the executor's region (§7.9 step 5).

+// of a record-first transition (followRecord); the claim's release after a

+// move's record-first close, from last_move (rev 7.23 §7.5, ruling M4: the

+// closing box may stop between the halves and never comes back); and, after

+// any other release (a push, a relocation, or rev 7.22's claim-first close

+// under its mutation), the CAS that clears `move` and names the executor's

+// region.

func (w *world) fixRecord() {

c := w.claim

if c == nil || c.Holder == nil || w.rec == nil || w.rec.Inc != c.Holder.Inc {

return

}

+ if moveClose(c.Op) && w.rec.LastMove == c.Op.ID && w.rec.MoveID == "" {

+ if c.oblFor(c.Op.ID) == 0 {

+ w.stats["recovery: release from last_move"]++

+ w.release(c, closeExec(c.Op))

+ }

+ return

+ }

switch c.Op.Kind {

case "serving":

if b := w.execBox(c); b != nil && (w.rec.MoveID != "" || w.rec.Region != b.Region) {

diff --git a/internal/sim/hostlife/moves_test.go b/internal/sim/hostlife/moves_test.go

index 6e5ae501f..b0af446ce 100644

--- a/internal/sim/hostlife/moves_test.go

+++ b/internal/sim/hostlife/moves_test.go

@@ -544,7 +544,7 @@ func (w *world) moveSource(b *box, c *claim) {

}

case "aborting":

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Src)

+ w.closeMove(c, op.Src)

}

case "stopping":

if b.Row != nil {

@@ -938,7 +938,7 @@ func (w *world) revertSource(b *box, c *claim) {

}

case "opened":

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Src)

+ w.closeMove(c, op.Src)

}

@@ -1258,7 +1258,7 @@ func (w *world) landTarget(b *box, c *claim) {

return

}

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Tgt)

+ w.closeMove(c, op.Tgt)

}

@@ -1334,6 +1334,74 @@ func (w *world) installLocal(b *box, c *claim, gen string, seal ref, _ string, n

w.cas(c, func(n *claim) bool { next(n); return true })

}

+// moveClose reports whether op is a move's close: a committed landing, an

+// opened rollback or revert, or an abort. Pushes and relocations release on

+// their own terms.

+func moveClose(op opState) bool {

+ switch op.Kind {

+ case "landing", "restoring", "reverting":

+ return true

+ case "moving":

+ return op.Phase == "aborting"

+ }

+ return false

+// closeExec is the box a move's close releases to: the target after a

+// landing, the source otherwise (§19 K8).

+func closeExec(op opState) int {

+ if op.Kind == "landing" {

+ return op.Tgt

+ }

+ return op.Src

+// closeMove is a move's close by the box that runs it, record first (moves

+// rev 7.23 §7.5 and §19 K8, Paul's ruling M4). With nothing owed under the

+// op, the record's CAS clears move into last_move and names the executor's

+// region; only then does the claim release. The box may stop between the

+// halves (every third step here, a deterministic schedule that draws

+// nothing from the seed's stream). It never comes back to a record without a

+// move, so a closed record whose claim still holds the op is the recovery

+// pass's to release, from last_move (fixRecord).

+func (w *world) closeMove(c *claim, execBox int) {

+ op := c.Op

+ if c.oblFor(op.ID) != 0 {

+ return

+ }

+ if !on("record-first-close") {

+ // rev 7.22: the claim first; the recovery pass wrote the record

+ // half after the release

+ w.release(c, execBox)

+ return

+ }

+ if !w.closeRecord(c, execBox) {

+ return // closed already (last_move): not the box's any more

+ }

+ if w.now%3 == 0 {

+ w.stats["close: stopped between the record and the release"]++

+ return

+ }

+ w.release(c, execBox)

+// closeRecord is the close's record half: the CAS that clears move into

+// last_move and names the executor's region, conditional on the record still

+// carrying this move for the claim's incarnation.

+func (w *world) closeRecord(c *claim, execBox int) bool {

+ r, op := w.rec, c.Op

+ if r == nil || c.Holder == nil || r.Inc != c.Holder.Inc || r.MoveID != op.ID {

+ return false

+ }

+ if b := w.boxByID(execBox); b != nil {

+ r.Region = b.Region

+ }

+ r.MoveID, r.MoveState, r.RollbackID, r.LastMove = "", "", "", op.ID

+ r.Ver = w.newVer()

+ w.logf("close %s: record closed (last_move)", op.ID)

+ return true

func (w *world) release(c *claim, execBox int) {

op := c.Op

if w.cas(c, func(n *claim) bool {

@@ -1347,8 +1415,15 @@ func (w *world) release(c *claim, execBox int) {

return true

}) {

w.logf("released %s to serving on box %d", op.ID, execBox)

+ if moveClose(op) && (w.rec == nil || w.rec.LastMove != op.ID || w.rec.MoveID == op.ID) {

+ // K8 as rev 7.23 amends it: never a serving claim over a live

+ // move. Judged only for a release CAS that succeeded: one the

+ // op-id guard refused released nothing (fix round 2)

+ w.violate("K8: %s released the claim before its record closed (record first, rev 7.23)", op.ID)

+ }

w.opsDone++

- // the record half (CAS-clear move) follows on a later step (§7.9)

+ // a move's record closed before this (closeMove, rev 7.23); a

+ // push's or a relocation's record half follows on a later step

w.checkDurable(w.claim, "at the release of "+op.ID)

if r := w.residueAtEnd(); len(r) > 0 {

w.violate("INV-10: %s ended with residue %v", op.ID, r)

@@ -1639,7 +1714,7 @@ func (w *world) recoverSource(b *box, c *claim) {

}

case "opened":

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Src)

+ w.closeMove(c, op.Src)

}

diff --git a/internal/sim/hostlife/rules_test.go b/internal/sim/hostlife/rules_test.go

index 803320fad..1ba023d6c 100644

--- a/internal/sim/hostlife/rules_test.go

+++ b/internal/sim/hostlife/rules_test.go

@@ -107,6 +107,7 @@ var allRules = []string{

"sweep-namespace", // claim rev 6 §6.2a (R3-4): the sweep discovers locations from the namespace; the mutation reads the capped retired_incarnations

"live-set-region", // claim rev 6.5 §6.2a (R4-3): the live set is {authoritative region, location, generation}; the mutation exempts matching keys in every region

// moves spec

+ "record-first-close", // rev 7.23 §7.5, §19 K8 (ruling M4): a move's close CASes the record (move into last_move) before the claim's release, and the recovery pass releases from last_move; the mutation is rev 7.22's claim-first close, whose record half the recovery pass wrote after the release

"seal-by-version", // J-N2: landing copies sealed versions

"staging-quiet", // J-N2: staging retired after its writers' horizon

"commit-before-open", // INV-5

diff --git a/internal/sim/hostlife/sim_test.go b/internal/sim/hostlife/sim_test.go

index 1cac18606..59ac976f6 100644

--- a/internal/sim/hostlife/sim_test.go

+++ b/internal/sim/hostlife/sim_test.go

@@ -508,6 +508,19 @@ func (w *world) canStart(e *box, c *claim) string {

return ""

}

+// finalFreed is the end state of a freed host: no record, and no box holds

+// local state for it.

+func (w *world) finalFreed() {

+ if w.rec != nil {

+ w.violate("claim/record: claim free but record inc %d exists", w.rec.Inc)

+ }

+ for _, b := range w.liveBoxes() {

+ if b.DB != nil || b.Files != nil {

+ w.violate("INV-10: box %d still holds local state for a freed host", b.ID)

+ }

func (w *world) final() {

if !w.quiescent() {

if len(w.pending) == 0 && len(w.console) == 0 && w.atRest() {

@@ -519,14 +532,7 @@ func (w *world) final() {

}

c := w.claim

if c == nil || c.Holder == nil {

- if w.rec != nil {

- w.violate("claim/record: claim free but record inc %d exists", w.rec.Inc)

- }

- for _, b := range w.liveBoxes() {

- if b.DB != nil || b.Files != nil {

- w.violate("INV-10: box %d still holds local state for a freed host", b.ID)

- }

+ w.finalFreed()

return

}

e := w.execBox(c)

@@ -556,8 +562,18 @@ func (w *world) final() {

}

c = w.claim

+ if c == nil || c.Holder == nil {

+ // the final drive ran the console too: a purge may have freed

+ // the host (seed 2547 / 3,000 under admission-registration)

+ w.finalFreed()

+ return

+ }

e = w.execBox(c)

}

+ if e == nil || w.rec == nil {

+ w.violate("claim/record: the claim's holder has no record or no executing box at the end: %s", w.state())

+ return

+ }

if w.rec.Region != e.Region {

w.violate("claim/record: record region %d, claim serves from box %d in region %d", w.rec.Region, e.ID, e.Region)

}

diff --git a/internal/sim/hostlife/witness_test.go b/internal/sim/hostlife/witness_test.go

index 9600d7fd0..73515c8fd 100644

--- a/internal/sim/hostlife/witness_test.go

+++ b/internal/sim/hostlife/witness_test.go

@@ -66,7 +66,7 @@ var witnesses = []witness{

{"staging-quiet", "staging-quiet", "ended with residue",

"a staging round completes after the landing's staging park acknowledged; retired at the seal rather than after the writers' horizon, the round is residue at the release (INV-10)", wStagingQuiet},

{"claim-cas-opid", "claim-cas-opid,revalidate-before-effect", "store of app",

- "redundant alone: a stalled source goroutine with the rollback's last read resumes after a purge claimed the host; revalidation stops it, and so does the op-id condition on its release CAS; with both gone the release overwrites the purge and the store is torn down under a serving host", wClaimCasOpID},

+ "redundant alone: the source closes the rollback's record and stalls before its release (rev 7.23, record first); the recovery pass releases from last_move, a purge claims the host, and the stalled goroutine resumes into its release with the rollback's last read; revalidation stops it, and so does the op-id condition on its release CAS; with both gone the release overwrites the purge and the store is torn down under a serving host", wClaimCasOpID},

{"dirty-return", "dirty-return", "INV-2",

"the target writes during its probe and replicates; the rollback is dirty and the return lane carries the write to the source; treated as clean with the source's replica covering the stop, the source's admission is covered clean-return, declares nothing, and the serving database lacks the target's write (INV-2 local at the final recovery, rev 7.12; rev 7.11 saw INV-15 at the declaration, rev 7.10 INV-2 on the serving database)", wDirtyReturn},

{"commit-before-open", "commit-before-open", "INV-5",

@@ -905,28 +905,47 @@ func wStagingQuiet(t *testing.T, w *world) {

w.finish(t)

}

-// wClaimCasOpID (claim C2): a clean rollback; a second goroutine of the

-// source read the claim just before the release and stalls; the host is

-// released, a purge claims it, and the goroutine resumes.

+// wClaimCasOpID (claim C2): a clean rollback; the source's close writes the

+// record (record first, rev 7.23) and its goroutine stalls before the

+// release, holding the claim it read; the recovery pass releases from

+// last_move, a purge claims the host, and the goroutine resumes into its

+// release.

func wClaimCasOpID(t *testing.T, w *world) {

src, tgt := w.boxes[0], w.boxes[1]

w.serve(src, 1, app0)

w.landProbed(t, src, tgt)

w.rollback(w.claim, "cut deadline")

- w.reach(t, 1500, "source about to release", func() bool {

+ w.reach(t, 1500, "source about to close", func() bool {

c := w.claim

return c.Op.Kind == "restoring" && c.Op.SrcPhase == "opened" && c.oblFor(c.Op.ID) == 0

}, w.act(src, tgt), w.recoveryPass)

old := w.claim.clone() // the stalled goroutine's read

- w.reach(t, 50, "released", func() bool { return w.claim.Op.Kind == "serving" }, w.act(src, tgt), w.recoveryPass)

+ if !w.closeRecord(old, src.ID) {

+ t.Fatalf("witness: the record did not close: %s", w.state())

+ }

+ w.reach(t, 50, "released from last_move", func() bool { return w.claim.Op.Kind == "serving" }, w.recoveryPass)

pg := &consolePlan{Kind: "purge", App: app0}

w.reach(t, 20, "purge claimed the host", func() bool { return w.claim.Op.Kind == "purging" }, func() { w.runPlan(pg) })

- w.resumeStale(src, old) // the goroutine wakes with the rollback's last read

+ w.resumeStaleRelease(src, old) // the goroutine wakes into its release

w.tick()

w.tick(func() { w.runPlan(pg) }) // the purge tears the store down

w.tick(func() { w.runPlan(pg) })

}

+// resumeStaleRelease is a close's goroutine that stalled between the

+// record's close and the claim's release, resuming with the claim it read:

+// it revalidates (claim C3), then writes the release half.

+func (w *world) resumeStaleRelease(b *box, old *claim) {

+ c, ok := w.readClaim(b)

+ if !ok {

+ return

+ }

+ if on("revalidate-before-effect") && !sameStep(old, c) {

+ return

+ }

+ w.release(old, closeExec(old.Op))

// wDirtyReturn (INV-2; also commit-before-open's witness): the target

// writes during its probe and is rolled back.

func wDirtyReturn(t *testing.T, w *world) {

@@ -3552,3 +3571,42 @@ func TestProbeWindow(t *testing.T) {

t.Logf("k=%d: accepted=%v tries=%d probe start %d ready %d (failed before the start %d, at the deadline unready %d)", k, accepted, tries, p.startedAt, p.readyAt, beforeStart, atEnv)

}

+// TestK8JudgesOnlyASucceededRelease (Task 7 fix round 2): the K8 oracle

+// judges a release CAS that succeeded, never one the op-id guard refused.

+// The source closes a clean rollback's record and its goroutine stalls

+// before the release; the recovery pass releases from last_move; a purge

+// runs to the end and removes the record; the stalled release resumes with

+// only revalidate-before-effect dropped. Its CAS is conditional on the

+// rollback's op, which the claim no longer holds: refused, the claim is

+// untouched, and nothing is violated.

+func TestK8JudgesOnlyASucceededRelease(t *testing.T) {

+ saved := mutated

+ mutated = "revalidate-before-effect"

+ defer func() { mutated = saved }()

+ w := newWitness()

+ src, tgt := w.boxes[0], w.boxes[1]

+ w.serve(src, 1, app0)

+ w.landProbed(t, src, tgt)

+ w.rollback(w.claim, "cut deadline")

+ w.reach(t, 1500, "source about to close", func() bool {

+ c := w.claim

+ return c.Op.Kind == "restoring" && c.Op.SrcPhase == "opened" && c.oblFor(c.Op.ID) == 0

+ }, w.act(src, tgt), w.recoveryPass)

+ old := w.claim.clone() // the stalled goroutine's read

+ if !w.closeRecord(old, src.ID) {

+ t.Fatalf("the record did not close: %s", w.state())

+ }

+ w.reach(t, 50, "released from last_move", func() bool { return w.claim.Op.Kind == "serving" }, w.recoveryPass)

+ pg := &consolePlan{Kind: "purge", App: app0}

+ w.reach(t, 2000, "the purge removed the record", func() bool { return w.rec == nil }, func() { w.runPlan(pg) }, w.actAll(), w.recoveryPass)

+ before := w.claim.clone()

+ w.resumeStaleRelease(src, old)

+ if len(w.violations) > 0 {

+ dump(t, w)

+ t.Fatalf("a refused stale release was judged: %v", w.violations)

+ }

+ if w.claim.Ver != before.Ver || w.claim.Op.Kind == "serving" && w.claim.Op.Exec == src.ID && w.claim.Holder != nil {

+ t.Fatalf("the stale release was not refused: before %s after %s", before.Op.Kind, w.state())

+ }

diff --git a/internal/sim/hostlife/world_test.go b/internal/sim/hostlife/world_test.go

index d0a6813e1..16ba94654 100644

--- a/internal/sim/hostlife/world_test.go

+++ b/internal/sim/hostlife/world_test.go

@@ -937,6 +937,10 @@ type record struct {

MoveState string

RollbackID string

Reason string

+ // LastMove names the move whose close cleared MoveID: the close is record

+ // first (moves rev 7.23 §7.5, Paul's ruling M4), so a record can say a

+ // move is over while the claim still holds its op, until the release.

+ LastMove string

}

// ---------- the world ----------