carlos / platform Public

moves-phase1-5c

Squashed from moves-phase1-5c into main.

claude 2 weeks ago e79d71b59aff4815f04973fd92b3f4a95cb18b52 parent d562d7f

31 files changed, +2078 −180

approved_moves_copy_test.go +6 −0
cmd/console/main.go +16 −0
cmd/console/movesswitch_test.go +57 −0
docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +15 −0
docs/superpowers/specs/2026-10-02-move-any-app-design.md +51 −17
internal/alerting/alerting.go +32 −0
internal/alerting/alerting_test.go +33 −0
internal/console/claim_authority_census_test.go +3 −0
internal/console/claim_invariant_moves_test.go +19 −0
internal/console/claim_runtime.go +15 −0
internal/console/movesrecovery.go +516 −0
internal/console/movesrecovery_test.go +721 −0
internal/console/server.go +12 −3
internal/console/store.go +31 −0
internal/console/store_gate.go +1 −0
internal/console/testdata/claim-authority-sites.txt +8 −0
internal/host/movesabort.go +43 −25
internal/host/movesclose_order_test.go +135 −0
internal/host/moveslandcommit.go +2 −2
internal/host/movesparks.go +71 −30
internal/host/movesparks_test.go +29 −15
internal/host/movesrecover.go +28 −22
internal/host/movesrecover_test.go +5 −10
internal/host/movesrevert.go +36 −29
internal/provision/plan9_b8_census_test.go +6 −5
internal/sim/hostlife/console_test.go +13 −2
internal/sim/hostlife/moves_test.go +80 −5
internal/sim/hostlife/rules_test.go +1 −0
internal/sim/hostlife/sim_test.go +24 −8
internal/sim/hostlife/witness_test.go +65 −7
internal/sim/hostlife/world_test.go +4 −0

CI passed — run details

CI log

=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	314.323s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.083s
ok  	github.com/carlosframework/platform/internal/activator	134.557s
ok  	github.com/carlosframework/platform/internal/agentsock	0.017s
ok  	github.com/carlosframework/platform/internal/alerting	0.003s
ok  	github.com/carlosframework/platform/internal/analytics	0.018s
ok  	github.com/carlosframework/platform/internal/appconfig	2.160s
ok  	github.com/carlosframework/platform/internal/apperr	0.077s
ok  	github.com/carlosframework/platform/internal/applog	0.125s
ok  	github.com/carlosframework/platform/internal/awsinv	0.004s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.004s
ok  	github.com/carlosframework/platform/internal/boxid	0.638s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.008s
ok  	github.com/carlosframework/platform/internal/caps	0.011s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.196s
ok  	github.com/carlosframework/platform/internal/changelog	0.239s
ok  	github.com/carlosframework/platform/internal/clamd	0.054s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	318.966s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.395s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.009s
ok  	github.com/carlosframework/platform/internal/dbverify	0.005s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.003s
ok  	github.com/carlosframework/platform/internal/docsite	0.004s
ok  	github.com/carlosframework/platform/internal/domains	1.962s
ok  	github.com/carlosframework/platform/internal/doorkey	0.007s
ok  	github.com/carlosframework/platform/internal/economics	0.004s
ok  	github.com/carlosframework/platform/internal/edge	25.765s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.007s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.130s
ok  	github.com/carlosframework/platform/internal/gate	1.655s
ok  	github.com/carlosframework/platform/internal/genselect	0.015s
ok  	github.com/carlosframework/platform/internal/geo	0.018s
ok  	github.com/carlosframework/platform/internal/geoip	0.032s
ok  	github.com/carlosframework/platform/internal/host	44.142s
ok  	github.com/carlosframework/platform/internal/hostactors	0.350s
ok  	github.com/carlosframework/platform/internal/hostbox	1.183s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.218s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.007s
ok  	github.com/carlosframework/platform/internal/hostexec	0.312s
ok  	github.com/carlosframework/platform/internal/hostfence	0.041s
ok  	github.com/carlosframework/platform/internal/hostfiles	4.074s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.022s
ok  	github.com/carlosframework/platform/internal/hostguard	0.148s
ok  	github.com/carlosframework/platform/internal/hostops	1.243s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.016s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.145s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.003s
ok  	github.com/carlosframework/platform/internal/infraview	0.002s
ok  	github.com/carlosframework/platform/internal/instancectl	1.700s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.685s
ok  	github.com/carlosframework/platform/internal/instancefs	0.003s
ok  	github.com/carlosframework/platform/internal/layout	1.285s
ok  	github.com/carlosframework/platform/internal/lease	9.434s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.005s
ok  	github.com/carlosframework/platform/internal/ledger	0.012s
ok  	github.com/carlosframework/platform/internal/mail	0.167s
ok  	github.com/carlosframework/platform/internal/maintenance	0.086s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.496s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.074s
ok  	github.com/carlosframework/platform/internal/metrics	0.006s
ok  	github.com/carlosframework/platform/internal/moves	20.852s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.757s
ok  	github.com/carlosframework/platform/internal/objstore	0.162s
ok  	github.com/carlosframework/platform/internal/ops	0.151s
ok  	github.com/carlosframework/platform/internal/park	0.184s
ok  	github.com/carlosframework/platform/internal/privatefile	0.003s
ok  	github.com/carlosframework/platform/internal/provision	2.995s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.010s
ok  	github.com/carlosframework/platform/internal/registry	3.523s
ok  	github.com/carlosframework/platform/internal/relay	7.222s
ok  	github.com/carlosframework/platform/internal/release	18.347s
ok  	github.com/carlosframework/platform/internal/rollout	1.464s
ok  	github.com/carlosframework/platform/internal/rootaws	0.007s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.003s
ok  	github.com/carlosframework/platform/internal/schedule	2.180s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	2.345s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.227s
ok  	github.com/carlosframework/platform/internal/status	0.038s
ok  	github.com/carlosframework/platform/internal/steering	0.005s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.129s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.003s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.002s
ok  	github.com/carlosframework/platform/internal/updatewords	0.002s
ok  	github.com/carlosframework/platform/internal/usage	0.004s
ok  	github.com/carlosframework/platform/internal/version	0.001s
ok  	github.com/carlosframework/platform/internal/vet	1.521s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.023s
=== RUN   TestT4StorageProbeMinio
2026/10/11 03:41:04 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DD5C9EF4403D7D, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.015s
minio contract: PASS

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index ae295e66b..f0e8f1820 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -75,6 +75,7 @@ var movesCopyIDs = []string{

"MV-N307", "MV-N308", "MV-N309", "MV-N310", "MV-N311", "MV-N312",

"MV-N313", "MV-N314",

"MV-N315", "MV-N316", "MV-N317", "MV-N318",

+ "MV-N325", "MV-N326", "MV-N327", "MV-N328", "MV-N329", "MV-N330", "MV-N331", "MV-N332", "MV-N333", "MV-N334",

}

// copyRow is one row of the record. The File cell is the file the sentence is

@@ -543,6 +544,9 @@ func TestMovesTextAppliedPlaceholders(t *testing.T) {

// copy" task appends the files it touched; TestMovesScanFilesCoverTheDiff

// fails when the list falls behind.

var movesScanFiles = []string{

+ "internal/console/movesrecovery.go",

+ "internal/console/claim_runtime.go",

+ "internal/alerting/alerting.go",

"internal/hostops/move_recovery_grant.go",

"internal/hostops/move_loss_owner.go",

"internal/host/movesrecoveryprobe.go",

@@ -746,6 +750,8 @@ var movesReturnedSentences = map[string]map[string]bool{

"internal/host/movesreport.go": {"t1Below": true, "t3Differs": true, "t3Shadowed": true, "t4Failed": true,

"t6NoRoom": true, "t8Count": true, "t12NotQuiet": true, "replicaReason": true},

"internal/host/movesrollback.go": {"stuckPage": true},

+ // The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N334).

+ "internal/console/movesrecovery.go": {"moveWaitingPage": true, "movePreparingPageText": true, "moveOrphanPage": true, "moveCloseOverduePage": true, "moveAbortOverduePage": true},

}

// movesLiteralFuncs are functions whose every string literal is a sentence

diff --git a/cmd/console/main.go b/cmd/console/main.go

index ab1ef043d..f4b243819 100644

--- a/cmd/console/main.go

+++ b/cmd/console/main.go

@@ -88,6 +88,7 @@ import (

"unicode/utf8"

"github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/dnsrec"

@@ -419,6 +420,9 @@ func serve() {

if os.Getenv("CARLOS_MAIL_LEGACY_SWEEP") == "1" {

go sweepLegacyMail(cfg.HostClaim, rel, st)

}

+ // The move pass's pages (moves Plan 5 Task 7): the console's alert sinks,

+ // armed before the claim runtime binds the pass, so no pass runs unarmed.

+ armMovesPaging(context.Background(), cfg, os.Getenv)

if err := srv.ConfigureClaimRuntime(context.Background()); err != nil {

log.Fatalf("console: claim recovery: %v", err)

}

@@ -1345,6 +1349,18 @@ func applyMovesSwitch(cfg *console.Config, getenv func(string) string) {

cfg.MovesEnabled = moves.Enabled(getenv)

}

+// armMovesPaging arms alerting's sinks from CARLOS_ALERT_WEBHOOK and

+// CARLOS_ALERT_SNS_TOPIC_ARN, the agent's variables, for the console's move

+// pass (moves Plan 5 Task 7, delta M14). Only a host-claim console with the

+// moves switch on runs that pass, so only it arms; it reports whether it did.

+func armMovesPaging(ctx context.Context, cfg console.Config, getenv func(string) string) bool {

+ if !cfg.HostClaim || !cfg.MovesEnabled {

+ return false

+ }

+ alerting.Arm(alerting.FromEnv(ctx, getenv, nil))

+ return true

func hostClaimEnabled(getenv func(string) string) bool {

return strings.TrimSpace(getenv("CARLOS_HOST_CLAIM")) != ""

}

diff --git a/cmd/console/movesswitch_test.go b/cmd/console/movesswitch_test.go

index 5fc3f903d..6173a520a 100644

--- a/cmd/console/movesswitch_test.go

+++ b/cmd/console/movesswitch_test.go

@@ -1,11 +1,13 @@

package main

import (

+ "context"

"go/ast"

"go/parser"

"go/token"

"testing"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/moves"

)

@@ -98,3 +100,58 @@ func isIdent(e ast.Expr, name string) bool {

id, ok := e.(*ast.Ident)

return ok && id.Name == name

}

+// The console pages from its move pass (moves Plan 5 Task 7, delta M14):

+// armMovesPaging arms alerting's sinks from the agent's variables on a

+// host-claim console with the moves switch on, and nowhere else; serve()

+// calls it with os.Getenv before ConfigureClaimRuntime, so the recovery

+// pass never runs unarmed.

+func TestArmMovesPagingOnlyWithBothSwitches(t *testing.T) {

+ env := func(k string) string {

+ if k == "CARLOS_ALERT_WEBHOOK" {

+ return "http://example.test/hook"

+ }

+ return ""

+ }

+ for _, tc := range []struct{ claim, moves, want bool }{{false, false, false}, {true, false, false}, {false, true, false}, {true, true, true}} {

+ cfg := console.Config{HostClaim: tc.claim, MovesEnabled: tc.moves}

+ if got := armMovesPaging(context.Background(), cfg, env); got != tc.want {

+ t.Errorf("claim %v moves %v: armed %v, want %v", tc.claim, tc.moves, got, tc.want)

+ }

+ t.Cleanup(func() { alerting.Arm(nil) })

+ f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)

+ if err != nil {

+ t.Fatal(err)

+ }

+ var armed, configured token.Pos

+ for _, d := range f.Decls {

+ fn, ok := d.(*ast.FuncDecl)

+ if !ok || fn.Name.Name != "serve" {

+ continue

+ }

+ ast.Inspect(fn.Body, func(n ast.Node) bool {

+ call, ok := n.(*ast.CallExpr)

+ if !ok {

+ return true

+ }

+ switch fun := call.Fun.(type) {

+ case *ast.Ident:

+ if fun.Name == "armMovesPaging" && len(call.Args) == 3 {

+ if sel, ok := call.Args[2].(*ast.SelectorExpr); ok && isIdent(sel.X, "os") && sel.Sel.Name == "Getenv" {

+ armed = call.Pos()

+ }

+ case *ast.SelectorExpr:

+ if fun.Sel.Name == "ConfigureClaimRuntime" {

+ configured = call.Pos()

+ }

+ return true

+ })

+ }

+ if armed == 0 || configured == 0 || armed > configured {

+ t.Fatalf("serve() must call armMovesPaging(ctx, cfg, os.Getenv) before ConfigureClaimRuntime (armed %v, configured %v)", armed, configured)

+ }

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index 94f979f72..4651f5fb7 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -271,6 +271,21 @@ Columns: id, file, text, status.

| MV-N314 | internal/host/movesrollback.go | The move <id> of <host> stopped for an operator on <box>, its <role>: <reason>. Run carlos move resolve --host <host> --retry once the cause is fixed. | pending |

+Task 7's pages, the console's move pass (`internal/console/movesrecovery.go`, §13). MV-N319 to MV-N324 are Plan 6 Task 2's.

+|---|---|---|---|

+| MV-N326 | internal/console/movesrecovery.go | The move <id> of <host> is rolling back, and <to> has not released it by its deadline. <host> stays offline until a box in <to> confirms it has stopped, or that box is declared lost. | pending |

+| MV-N330 | internal/console/movesrecovery.go | The host claim of <host> is held by the move <id>, and the instance record carries no move. Nothing releases it on its own. An operator must check the record. | pending |

+| MV-N334 | internal/console/movesrecovery.go | The abort of the move <id> of <host> has nothing left to park in <to>, and <box> has not closed it. <box> closes it once it has removed the move's snapshots from its own bucket. Check <box>. | pending |

## Floor gate fix (#267 review)

(root's floor gate for the system units, `carlos system floor-allows-prev`; its stderr reaches the unit's journal)

diff --git a/docs/superpowers/specs/2026-10-02-move-any-app-design.md b/docs/superpowers/specs/2026-10-02-move-any-app-design.md

index c8a59c34d..fc6d8c40e 100644

--- a/docs/superpowers/specs/2026-10-02-move-any-app-design.md

+++ b/docs/superpowers/specs/2026-10-02-move-any-app-design.md

@@ -1,8 +1,32 @@

# Move any app or instance to another region: Phase 1, a safe single-instance move

**Date:** 2026-10-06

-**Revision:** 7.22.

+**Revision:** 7.23.

+- **Rev 7.23** records Paul's ruling M4 (2026-10-10, moves Plan 5 delta

+ pre-flight after claim Plan 10): **every close is record first, then

+ the claim's release.** Rev 7.22 released the claim to `serving` first

+ and cleared `move` second (§7.5, §19 K8). Claim Plan 10's move-recovery

+ grant closes the record first, and a serving claim with a live move

+ record can never acquire that grant, so the two orders could not both

+ stand. Each close (the commit's, a rollback's, a revert's, an abort's)

+ now checks that no obligation of the op remains, CAS-clears `move` into

+ `last_move`, and then releases. A crash between the halves leaves the

+ record at rest and the claim still under the op with nothing owed; the

+ box passes are driven by records with a live move and do not come

+ back, so the console's recovery pass releases from `last_move`.

+ Amended: §7.5, §7.8 finalization, §7.9 step 5, §7.11, the `reverting`

+ table's `opened` row, §14.2.2's model row and §19 K8. **The model**

+ (`internal/sim/hostlife`, moves Plan 5 Task 7 fix round 1) closes the

+ same way: a box's close CASes the record (`last_move`), and on every

+ third step stops before the release; the recovery pass releases from

+ `last_move` (`fixRecord`). A new invariant, K8, fails any move's release

+ whose record still carries the move, and a new rule,

+ `record-first-close`, whose mutation is rev 7.22's claim-first close, is

+ caught by it (1,060 / 3,000 seeds, first seed 1; no witness). The

+ 3,000-seed sweep of every rule otherwise matches its expected outcomes. The `claim-cas-opid` witness is

+ restaged: its stalled goroutine is now the source between the record's

+ close and its release (§14.2.5).

- **Rev 7.22** answers **joint round 19** (`CODEX-FINDINGS-JOINT-R19.md`,

"R19-1", the round's one finding, P2) and fixes the install

manifest's keys. **R19-1:** rev 7.21 bound the prepared files manifest

@@ -2607,7 +2631,7 @@ itself as execution holder, holds the row as `restoring` with

| `verify` | entered from `stopped`, `staged` or `archived`: check files and timers against the digests recorded at `stopped` | replayable **until `probing` is journaled** |

| `probing` | render held config and materialize the release (§7.7), then probe | restarts only the probe. **Never re-runs `verify`**: the probe's process may have written, and those writes are now the authoritative state (INV-2) |

| `probed` | conditional open on `hold_op = move id` | replayable |

-| `opened` | the claim waits for the staging obligation (§7.11), then releases to `serving` | |

+| `opened` | the claim waits for the staging obligation (§7.11); then the record clears (`last_move` = `rolled_back`) and the claim releases to `serving` (record first, §7.5) | |

Rev 3 compared digests again on re-entry, so a crash after the recovery

probe had written left the move stuck on a "mismatch" that was really the

@@ -2657,11 +2681,19 @@ with crashes, console loss and the recovery pass between them (§14.2.2).

The source row stays held (`moving`); reconcile keeps it because the claim

names this box as a participant (§9.6).

-**The close is the other way round.** Releasing to `serving` is a claim

-CAS first (§19 K8), then the record CAS that clears `move` and names the

-executor's region; the recovery pass finishes the second half after a

-crash. Until it lands, the record still carries a finished `move`, which

-admits nothing.

+**The close is record first too (rev 7.23, Paul's ruling M4).** The

+actor that closes first checks that no obligation this op created

+remains, then CASes the record: `move` is cleared and `last_move` names

+the move and its outcome (`done`, `rolled_back` or `aborted`). The claim's

+release to `serving` (§19 K8) is the second CAS, determined by

+`last_move`: the op id, its kind and phase, and the executor the outcome

+names. Until it lands, the claim still holds the op, so nothing else

+starts on the host. After a crash between the two, no box revisits the

+host (the box passes are driven by records with a live move), and the

+console's recovery pass writes the release from `last_move`. Rev 7.22

+had this the other way round, claim first; claim Plan 10's recovery

+grant can act only while the claim holds the move op, so a claim

+released over a live `move` could never be finished by it.

**Late record writes.** Every move write to the record is a conditional

CAS. One that completes after its client gave up either still matches,

@@ -2873,9 +2905,9 @@ install supersedes. These are the claim spec's durable chores (§2 there);

**Finalization:** when `recovered_at`, `opened_at` and `target_parked_at`

are all stamped, and no obligation this op created remains, whichever

actor stamps last writes the terminal log event (`rolled_back`, with every

-timing, digest and reason), releases the claim to `serving` (§19 K8), and

-CAS-clears `move`. Until then `move` and the claim stay, so nothing else

-can start on the host (codex 6). The source serves from `opened` on; the

+timing, digest and reason), CAS-clears `move`, and then releases the

+claim to `serving` (§19 K8; record first, §7.5). Until then `move` and

+the claim stay, so nothing else can start on the host (codex 6). The source serves from `opened` on; the

held claim blocks only other operations.

Nine failed recovery probe attempts (three times the landing's budget of

@@ -2917,8 +2949,9 @@ runs anywhere. It converges when the target returns (INV-3).

`moves/<host>/`): `moves/<host>/<move id>/` into

`superseded/<move id>/<host>/`; remove its obligation.

5. **Close** (when both obligations are gone): terminal log event `done`;

- release the claim to `serving` with the target as execution holder

- (§19 K8); CAS-clear `move`.

+ CAS-clear `move` (`last_move` = `done`); then release the claim to

+ `serving` with the target as execution holder (§19 K8; record first,

+ §7.5).

### 7.10 Stuck and declare-lost (M16)

@@ -3068,7 +3101,8 @@ source retires them from its own bucket, every version of

`move-snapshots/<host>/<op>/` through `DeleteVersion`; that prefix is outside

the park scope grammar, and F3's retention of snapshots covers completed

moves only.) The claim stays `moving` until every obligation

-has acknowledged, then releases to `serving`. The model gives the source

+has acknowledged; then the record clears (`last_move` = `aborted`) and

+the claim releases to `serving` (record first, §7.5). The model gives the source

the target's park and records an IAM violation on every abort.

### 7.12 Deletes and parks under the claim

@@ -3824,7 +3858,7 @@ the text. What changed, and what did not:

|---|---|

| restore stronger than the spec (an exact base pointer, historical lineage selection) | restore reads `holder.location` and `holder.generation` and replays every object of that generation (§5.1). The mutation restores the newest object under the location |

| the start check stronger than the spec (compared with the committed lineage) | the start check compares the box's own three markers with each other and, as a separate rule, with the claim's generation (§5.1); rev 5's check is the mutation |

-| claim transitions followed by an atomic record write | record-first flip, commit and rollback, then the claim half as a separate step, written by the actor, the other participant or the recovery pass; the close the other way round (§7.5) |

+| claim transitions followed by an atomic record write | record-first flip, commit and rollback, then the claim half as a separate step, written by the actor, the other participant or the recovery pass; the close record first too, then the claim's release, which the recovery pass finishes from `last_move` after a crash (§7.5, rev 7.23) |

| every request completes by `issue + H` | one long-tail request in 20 completes up to 11 `H` late, after publication and after retirement; the restore contract keeps them out of data, the sweep retires them (§5.1, §6.4) |

| check and sign, check and local write, atomic | staging writers sign at the check and may then pause, a frozen box without bound (§7.3); host-guard writers are independent processes (`carlos-config-sync`, `carlos-adopt`) that survive an agent crash, pause before the guard and re-validate under it (claim §6.4); door listings read O1, pause, select, read O2 (claim §6.5) |

| atomic `localPark` | one artifact per step, the retired marker first |

@@ -5914,7 +5948,7 @@ witness to hold there; each witness violates with the failure it names.

| `dirty-return` (INV-2) | the target writes during its probe and replicates; the rollback is dirty and the return lane carries the write | **violated**: treated as clean with the source's replica covering the stop, the return is covered `clean-return`, nothing is declared and the serving database lacks the target's write (INV-2 local at the final recovery; rev 7.11 saw INV-15 at the declaration) | | 1 / 3,000 |

+| `claim-cas-opid` (C2) | the source closes a rollback's record and its goroutine stalls before the release, holding the rollback's last read (rev 7.23: before it, a goroutine stalled before a claim-first release); the recovery pass releases from `last_move`, a purge claims the host, and the goroutine resumes into its release | held | with `revalidate-before-effect`: **violated**, the store is torn down under a serving host | 0 / 3,000 |

| `exec-admission`, `hold-op-open`, `start-check-claim-gen`, `exec-writer` (J2-R2-8; claim rev 6.8.1) | a paused open from an earlier operation lands after the commit while the source's row is still held; a request wakes the source | held, each, and under every proper subset | all four: **violated**, INV-1 (the non-exec source is admitted cold into a fresh generation restored from its own region's old files and serves beside the target) | 0 / 3,000 |

| `start-check` (INV-4) | the same, dropping the whole start check | retired in rev 7.10: with every restore from the box's own region, the stale source finds nothing of the target's generation to restore and cannot start; subsumed (`expected-outcomes.txt`) | | 0 |

@@ -6350,7 +6384,7 @@ The db door refuses on every move kind (claim spec §8).

| **K5 Revalidation** | Before each destructive step the actor re-reads the claim and requires op id, kind, phase and `exec` to be what the step expects (claim spec C3); it holds the lease of every location it writes, acting before `SafeUntil`, with the exception in §19.3 D4. A staging writer checks `SafeUntil` immediately before it signs each request, after any pause (§7.3, model 14). |

| **K6 Wake admission** | `doWake` reads the claim fresh (claim spec §8) and applies K1. A box that reads a claim whose `exec` does not name it kills that host's process tree (§5.3 step 1) before anything else. |

| **K7 Other operations** | Delete, purge, park and push start only from `serving` (claim spec §4); while a move op holds the claim they are refused. Create requires free or absent. |

-| **K8 Release** | Only the move releases to `serving`, naming `exec` (target after `done`; source after `rolled_back` or `aborted`), and only when every obligation this op created is gone (§7.8, §7.9). |

+| **K8 Release** | Only the move releases to `serving`, naming `exec` (target after `done`; source after `rolled_back` or `aborted`), and only when every obligation this op created is gone (§7.8, §7.9). The release follows the record's close: the closing actor checks that nothing is owed under the op, CAS-clears `move` into `last_move`, then releases; the console's recovery pass finishes the release from `last_move` after a crash (§7.5; rev 7.23, Paul's ruling M4). |

| **K9 Other passes while a move op holds the claim** | Reconcile mints participant rows held and never `ready`, and never removes or orphan-retires a participant row; the schedule converger neither upserts nor prunes the host's timers; `SyncExec` and release adoption skip the held rows (they already skip non-`ready`). |

| **K10 Version floor** | The claim spec's floor (§9 there) and `moves_protocol` (§11) are one object and one check. |

| **K11 Declare-lost** | §7.10; §19.3 D6. |

diff --git a/internal/alerting/alerting.go b/internal/alerting/alerting.go

index 8956e2fec..22ceb2df0 100644

--- a/internal/alerting/alerting.go

+++ b/internal/alerting/alerting.go

@@ -187,6 +187,38 @@ func Page(ctx context.Context, level, subject, body string) error {

return s.Page(ctx, level, subject, body)

}

+// FromEnv builds a process's sinks from the agent's two variables,

+// CARLOS_ALERT_WEBHOOK (posted through PostJSON) and

+// CARLOS_ALERT_SNS_TOPIC_ARN (published from the topic's own region through

+// newClient, NewSNSClient when nil). The console arms these for its move

+// pass (moves Plan 5 Task 7). A topic that does not parse, or whose client

+// cannot be built, leaves SNS off and says so; the webhook is unaffected.

+// It never returns nil: with neither set, Page logs that no sink is

+// configured.

+func FromEnv(ctx context.Context, getenv func(string) string, newClient func(context.Context, string) (Publisher, error)) *Sinks {

+ if newClient == nil {

+ newClient = NewSNSClient

+ }

+ s := &Sinks{}

+ if url := strings.TrimSpace(getenv("CARLOS_ALERT_WEBHOOK")); url != "" {

+ s.WebhookURL, s.Post = url, PostJSON

+ }

+ if topic := strings.TrimSpace(getenv("CARLOS_ALERT_SNS_TOPIC_ARN")); topic != "" {

+ region, err := ParseTopicARN(topic)

+ if err != nil {

+ log.Printf("alerting: CARLOS_ALERT_SNS_TOPIC_ARN is not usable, SNS sink off: %v", err)

+ return s

+ }

+ pub, err := newClient(ctx, region)

+ if err != nil {

+ log.Printf("alerting: build the SNS client, SNS sink off: %v", err)

+ return s

+ }

+ s.TopicARN, s.Pub = topic, pub

+ }

+ return s

// PostJSON is the production webhook post: a 5 second POST with a JSON

// content type. Any non-2xx status is an error.

func PostJSON(url string, body []byte) error {

diff --git a/internal/alerting/alerting_test.go b/internal/alerting/alerting_test.go

index 90df95149..c84a93f87 100644

--- a/internal/alerting/alerting_test.go

+++ b/internal/alerting/alerting_test.go

@@ -206,3 +206,36 @@ func TestParseTopicARN(t *testing.T) {

}

+// FromEnv is the console's sinks (moves Plan 5 Task 7, delta M14): the

+// agent's two variables, the webhook through PostJSON and the topic's own

+// region for SNS. A topic that does not parse, or whose client cannot be

+// built, leaves SNS off and the webhook on; nothing set is no sink.

+func TestFromEnvBuildsTheConsoleSinks(t *testing.T) {

+ env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } }

+ var asked []string

+ client := func(_ context.Context, region string) (Publisher, error) {

+ asked = append(asked, region)

+ return &fakeSNS{}, nil

+ }

+ s := FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_WEBHOOK": " http://example.test/hook ", "CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), client)

+ if !s.HasWebhook() || s.WebhookURL != "http://example.test/hook" || !s.HasSNS() || s.TopicARN != testTopicARN {

+ t.Fatalf("both sinks: %+v", s)

+ }

+ if len(asked) != 1 || asked[0] != "eu-west-1" {

+ t.Fatalf("the SNS client was built for %v, want the topic's region", asked)

+ }

+ buf := captureLog(t)

+ s = FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_WEBHOOK": "http://example.test/hook", "CARLOS_ALERT_SNS_TOPIC_ARN": "arn:aws:sns:eu-west-1:1:x:sub"}), client)

+ if !s.HasWebhook() || s.HasSNS() || !strings.Contains(buf.String(), "SNS sink off") {

+ t.Fatalf("a bad topic: %+v (log %q)", s, buf.String())

+ }

+ s = FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), func(context.Context, string) (Publisher, error) { return nil, errors.New("no credentials") })

+ if s.HasWebhook() || s.HasSNS() {

+ t.Fatalf("a client that cannot be built: %+v", s)

+ }

+ s = FromEnv(context.Background(), env(nil), client)

+ if s == nil || s.HasWebhook() || s.HasSNS() {

+ t.Fatalf("nothing set: %+v", s)

+ }

diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go

index 1a702ccfc..455e34e1f 100644

--- a/internal/console/claim_authority_census_test.go

+++ b/internal/console/claim_authority_census_test.go

@@ -26,6 +26,9 @@ func claimAuthoritySites(t *testing.T) []string {

"ResumeExclusive": true, "RegisterStart": true, "FinishWakeRetryReset": true, "ReservePurge": true,

"ConsoleMove": true, "InspectMoveLoss": true, "DeclareMoveLoss": true,

"RunRecovery": true, "runMoveRecovery": true, "captureMoveStartBase": true, "resolveUncertain": true, "ConvertMovingToRelocating": true,

+ // The console's move pass (moves Plan 5 Task 7): every write of the

+ // move-recovery grant, and the lost target's stamp.

+ "ResolveStart": true, "Convert": true, "Claim": true, "Record": true, "StampLostMoveTargetParked": true, "stampLostMoveTarget": true,

"BeginPurge": true, "SavePurgeJournal": true, "RemovePurgeJournal": true,

"AcquireMailGate": true, "ReleaseMailGate": true, "EnrollNewApp": true,

"EnrollBackfillApp": true, "OpenGated": true, "WithAppMutation": true,

diff --git a/internal/console/claim_invariant_moves_test.go b/internal/console/claim_invariant_moves_test.go

index 51df1b2a3..94f14449f 100644

--- a/internal/console/claim_invariant_moves_test.go

+++ b/internal/console/claim_invariant_moves_test.go

@@ -132,3 +132,22 @@ func TestClaimPublishMovesConsoleIsInertWithTheSwitchOff(t *testing.T) {

t.Fatalf("the worker wrote with the moves switch off: %v", events)

}

+// TestClaimMovesConsoleBindsTheMovePass (moves Plan 5 Task 7): a moves-on

+// claim console binds its move pass as claim Plan 10's Recovery.Moves hook,

+// with the lost target's stamp; it adds no route and no worker (the pass runs

+// inside StartClaimRecoveryPass). A moves-off console binds neither, so the

+// recovery pass skips move claims as main does.

+func TestClaimMovesConsoleBindsTheMovePass(t *testing.T) {

+ off := newClaimCase(t).srv

+ if off.cfg.ClaimMoveRecovery != nil || off.cfg.HostClaimRecovery == nil || off.cfg.HostClaimRecovery.Moves != nil || off.stampLostMoveTarget != nil {

+ t.Fatal("a moves-off console bound the move pass")

+ }

+ on := newClaimCaseWith(t, claimMovesOn).srv

+ if on.cfg.ClaimMoveRecovery == nil || on.cfg.HostClaimRecovery == nil || on.cfg.HostClaimRecovery.Moves == nil || on.stampLostMoveTarget == nil {

+ t.Fatal("a moves-on console did not bind the move pass")

+ }

+ if on.cfg.HostClaimRecovery.MoveFenceAuthority == nil {

+ t.Fatal("the move pass has no fence authority for its conversions")

+ }

diff --git a/internal/console/claim_runtime.go b/internal/console/claim_runtime.go

index 6a41d1e10..c1bc23eb5 100644

--- a/internal/console/claim_runtime.go

+++ b/internal/console/claim_runtime.go

@@ -109,6 +109,21 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {

s.cfg.ClaimPlacement = placement

s.claimOwner = owner

s.claimRuntimeConfigured = true

+ if s.cfg.MovesEnabled && s.cfg.ClaimMoveRecovery == nil {

+ // The console's move pass (moves Plan 5 Task 7): the moves body of

+ // claim Plan 10's Recovery.Moves hook, bound only with the moves

+ // switch on, and only now that every box close is record first

+ // (ruling M4, P3-6; TestClaimMoveRecoveryWiringWaitsForRecordFirstOwners).

+ s.cfg.ClaimMoveRecovery = s.movesRecovery

+ // The lost target's target_parked_at (p10 Task G) is written

+ // outside the grant by the lifecycle record writer, which the

+ // conditional record store carries and the data boundary (an app

+ // permit's) does not. The pass gets this one transition, never

+ // the store.

+ s.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

+ return hostops.StampLostMoveTargetParked(ctx, records, host, now)

+ }

s.cfg.HostClaimRecovery = &hostops.Recovery{

Moves: s.cfg.ClaimMoveRecovery,

MoveFenceAuthority: hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: c.Now},

diff --git a/internal/console/movesrecovery.go b/internal/console/movesrecovery.go

new file mode 100644

index 000000000..db883ea40

--- /dev/null

+++ b/internal/console/movesrecovery.go

@@ -0,0 +1,516 @@

+package console

+import (

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "errors"

+ "fmt"

+ "log"

+ "os"

+ "slices"

+ "time"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostfence"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+// The console's move pass (moves Plan 5 Task 7, spec §7.5, §7.10, §7.11,

+// §13; Paul's ruling M4). It is the moves body of claim Plan 10's

+// Recovery.Moves hook: the recovery pass (ControlWriter.RecoveryPass) calls

+// movesRecovery once per claim of a move kind, with a MoveRecoveryGrant bound

+// to that claim, its move registration and the exact record it observed.

+// ConfigureClaimRuntime binds it only with the moves switch on.

+//

+// Each call reads the claim and the record again through the console's store,

+// gives up if either moved on since the grant's observation (the next pass

+// observes afresh), decides at most one action (decideMoveRecovery) and takes

+// it. The grant is spent by its first write, so one observation is never two

+// writes. In the order the decision tries them:

+//

+// 1. An orphan: a claim of a move kind whose record carries neither the move

+// nor its outcome. A moving claim before the seal (the request door's

+// record write was lost) is left for ten minutes, then its start effect

+// is settled without the request's handle (ResolveStart, K3), and on a

+// later pass, with the start settled not applied, released

+// (release-not-applied). Any other orphan is paged, once, and left: an

+// operation past the seal with no record is an incident.

+// 2. A source terminated before the flip (K13): Convert with the source's

+// termination fence, which writes the record's relocation decision on

+// one pass and converts the claim to relocating on the next.

+// 3. The six-hour abort (K18): a move still preparing moves.AbortAfter

+// after its request is aborted through ConsoleMove(ConsoleMoveAbort);

+// the source's own abort (movesabort.go) takes it from there.

+// 4. The claim half the record decided (§7.5, Review Focus 5): flip,

+// commit or rollback. Anything else is left alone.

+// 5. The deadline: a landing past the earlier of its cut and landing

+// deadlines is rolled back by the record (J11, J9's recorded reason).

+// 6. A lost target released with nothing of its box or region owed:

+// target_parked_at, through hostops.StampLostMoveTargetParked over the

+// conditional record store (stampLostMoveTarget), outside the grant

+// (p10 Task G).

+// 7. A close whose stamps are in and whose op owes nothing: the record's

+// close (close, rolled-back-close or revert-close). The boxes close on

+// their own in the pass the close becomes due; the console writes it

+// only when it has been due for moveCloseGrace, and pages that it did.

+// An abort is never closed here (fix round 1, ruling 1): only the

+// source closes it, after it retired its own snapshots, receipts,

+// ledger and journal (movesabort.go), and nothing on the record shows

+// that it did. An abort due past the lane's horizon (the claim's

+// quiet_after for the target region) plus moveCloseGrace is paged

+// once, and left to the source.

+// 8. The release: the record closed (LastMove names the op) and the claim

+// not yet released. This is the second half of every record-first close

+// (ruling M4); after a box crashed between the halves, only this pass

+// writes it, because no box visits a record without a move.

+//

+// Pages (§13) go through alerting.Page on the sinks cmd/console arms, best

+// effort: a rollback waiting past its release deadline (an incident, again

+// only a full hour after the last attempt), a move preparing for over an

+// hour, an orphan past the seal (an incident), an overdue close the console

+// wrote, and an overdue abort the console leaves to its source (each once).

+// The rate limit is durable: the last attempt's time is kept per finding in

+// the console's store (move_pages, fix round 1 ruling 3), written whether or

+// not delivery worked, so a restarted console keeps the spacing.

+//

+// A relocating claim is region HA's, and the pass does nothing for it.

+const (

+ // moveOrphanAfter is how old, by Op.StartedAt, a moving claim with no

+ // record move must be before the pass settles its start.

+ moveOrphanAfter = 10 * time.Minute

+ // movePreparingPage is how long a move may prepare before it pages

+ // (§13: 60 minutes with no promotable round).

+ movePreparingPage = time.Hour

+ // moveCloseGrace is how long a close must have been due before the

+ // console writes it instead of the boxes.

+ moveCloseGrace = 10 * time.Minute

+type moveRecoveryAction string

+const (

+ moveActNone moveRecoveryAction = ""

+ moveActResolveStart moveRecoveryAction = "resolve-start"

+ moveActConvert moveRecoveryAction = "convert"

+ moveActAbort moveRecoveryAction = "abort"

+ moveActClaim moveRecoveryAction = "claim"

+ moveActRecord moveRecoveryAction = "record"

+ moveActStampLost moveRecoveryAction = "stamp-lost"

+ moveActClose moveRecoveryAction = "close"

+ // moveActAbortOverdue writes nothing: an abort due to close, which only

+ // its source may close, is checked for being overdue and paged.

+ moveActAbortOverdue moveRecoveryAction = "abort-overdue"

+type movePageKind string

+const (

+ movePageReleaseDeadline movePageKind = "release-deadline"

+ movePagePreparing movePageKind = "preparing"

+ movePageOrphan movePageKind = "orphan"

+ movePageCloseOverdue movePageKind = "close-overdue"

+ movePageAbortOverdue movePageKind = "abort-overdue"

+// moveRecoveryInput is what one observation saw. Start is the op's move-start

+// effect, read only for a moving claim with no record move; Fence is the

+// source's termination fence, read only for a moving claim.

+type moveRecoveryInput struct {

+ Now time.Time

+ Claim *hostclaim.Claim

+ Rec provision.Record

+ Start *hostops.Effect

+ Fence *hostclaim.FenceRef

+// moveRecoveryDecision is at most one action, with the grant action it

+// names, and the pages the observation calls for.

+type moveRecoveryDecision struct {

+ Action moveRecoveryAction

+ Claim hostops.MoveRecoveryClaimAction

+ Record hostops.MoveRecoveryRecordAction

+ Pages []movePageKind

+// decideMoveRecovery is the pass's whole policy, on values (see the file

+// comment for the order).

+func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision {

+ var d moveRecoveryDecision

+ c := in.Claim

+ if c == nil || c.Holder == nil || c.Op.ID == "" || c.Op.Kind == hostclaim.KindRelocating {

+ return d

+ }

+ op, rec, m := c.Op, in.Rec, in.Rec.Move

+ sealed := op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseSealed

+ if m == nil || m.Legacy() || m.ID != op.ID {

+ if m != nil {

+ return d // another move's record: not this op's to settle

+ }

+ if rec.LastMove != nil && rec.LastMove.ID == op.ID {

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRelease

+ return d

+ }

+ if op.Kind != hostclaim.KindMoving || op.Phase != hostclaim.PhaseMoving {

+ d.Pages = append(d.Pages, movePageOrphan)

+ return d

+ }

+ if op.StartedAt.IsZero() || in.Now.Before(op.StartedAt.Add(moveOrphanAfter)) || in.Start == nil {

+ return d

+ }

+ switch {

+ case in.Start.State == "uncertain":

+ d.Action = moveActResolveStart

+ case in.Start.State == "complete" && in.Start.ResultReference == "not-applied":

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveReleaseNotApplied

+ }

+ return d

+ }

+ if m.Incarnation != c.Holder.Incarnation {

+ return d

+ }

+ rb := m.Rollback

+ if m.State == provision.MoveRollingBack && rb != nil && rb.ReleaseDeadline != nil && in.Now.After(*rb.ReleaseDeadline) && rb.TargetPhase == provision.RbTargetReleasing {

+ d.Pages = append(d.Pages, movePageReleaseDeadline)

+ }

+ if m.State == provision.MovePreparing && !m.RequestedAt.IsZero() && !in.Now.Before(preparingSince(m).Add(movePreparingPage)) {

+ d.Pages = append(d.Pages, movePagePreparing)

+ }

+ owed := owedUnderOp(c, op.ID, nil)

+ switch {

+ case op.Kind == hostclaim.KindMoving && in.Fence != nil && (m.State == provision.MovePreparing || m.State == provision.MoveReady || m.State == provision.MoveCutting || m.State == provision.MoveRelocating):

+ d.Action = moveActConvert

+ case op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseMoving && moves.AbortDue(rec, in.Now):

+ d.Action = moveActAbort

+ case m.State == provision.MoveLanding && m.FlippedAt != nil && rec.Region == m.To && sealed:

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveFlip

+ case m.State == provision.MoveCommitted && m.CommittedAt != nil && m.FlippedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseLandingProbed:

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveCommit

+ case m.State == provision.MoveRollingBack && rb != nil && rec.Region == m.From && (op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted || sealed):

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRollback

+ case m.State == provision.MoveLanding && op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted && moveDeadlinePassed(m, in.Now):

+ d.Action, d.Record = moveActRecord, hostops.RecoveryMoveDeadlineRollback

+ case m.State == provision.MoveRollingBack && rb != nil && rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty && rb.TargetParkedAt == nil &&

+ (op.Kind == hostclaim.KindRestoring || op.Kind == hostclaim.KindLanding) && m.To != "" && op.Intent["target"] != "" && slices.Contains(c.Lost, op.Intent["target"]) &&

+ owedUnderOp(c, op.ID, func(o hostclaim.Obligation) bool {

+ t := op.Intent["target"]

+ return o.Box == t || o.Owner == "box:"+t || o.Region == m.To || o.Owner == "region:"+m.To

+ }) == 0:

+ d.Action = moveActStampLost

+ case owed > 0:

+ case m.State == provision.MoveCommitted && m.OpenedAt != nil && m.SourceParkedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseCommitted:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveClose

+ case m.State == provision.MoveRollingBack && rb != nil && rb.RecoveredAt != nil && rb.OpenedAt != nil && rb.TargetParkedAt != nil && rec.Region == m.From && op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseOpened:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveRolledBackClose

+ case m.State == provision.MoveReverting && m.ReopenedAt != nil && rec.Region == m.From && op.Kind == hostclaim.KindReverting && op.Phase == hostclaim.PhaseOpened:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveRevertClose

+ case m.State == provision.MoveAborting && op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseAborting:

+ d.Action = moveActAbortOverdue

+ }

+ return d

+// preparingSince is where a preparing move's clocks run from: the later of

+// its request and its readiness (moves.AbortDue's base, K18).

+func preparingSince(m *provision.MoveState) time.Time {

+ from := m.RequestedAt

+ if m.ReadyAt != nil && m.ReadyAt.After(from) {

+ from = *m.ReadyAt

+ }

+ return from

+// moveDeadlinePassed is the grant's deadline rule: the earlier of the cut and

+// landing deadlines (J11) is behind now.

+func moveDeadlinePassed(m *provision.MoveState, now time.Time) bool {

+ deadline := m.CutDeadline

+ if l := m.LandingDeadline; l != nil && (deadline == nil || l.Before(*deadline)) {

+ deadline = l

+ }

+ return deadline != nil && now.After(*deadline)

+// owedUnderOp counts the claim's obligations under op id, those match

+// accepts when it is set.

+func owedUnderOp(c *hostclaim.Claim, id string, match func(hostclaim.Obligation) bool) int {

+ n := 0

+ for _, o := range c.Obligations {

+ if o.Op == id && (match == nil || match(o)) {

+ n++

+ }

+ return n

+func (s *Server) movesNow() time.Time {

+ if s.movesClock != nil {

+ return s.movesClock().UTC()

+ }

+ return time.Now().UTC()

+// movesRecovery is the Recovery.Moves hook (see the file comment).

+func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant) error {

+ a := s.cfg.ClaimAuthority

+ store := s.cfg.Releases

+ if a == nil || a.Control == nil || store == nil || g == nil {

+ return hostops.ErrContract

+ }

+ o := g.Observation()

+ obs, err := hostclaim.Read(ctx, store, o.Host)

+ if err != nil {

+ return err

+ }

+ if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.ETag != o.ClaimETag || obs.Claim.Op.ID != o.OpID {

+ return nil // moved on since the observation: the next pass observes again

+ }

+ c := obs.Claim

+ r, err := hostops.ReadRecord(ctx, store, c.Holder.Key)

+ if err != nil {

+ return err

+ }

+ if r.ETag != o.RecordETag {

+ return nil

+ }

+ rec, err := provision.DecodeRecord(r.Body)

+ if err != nil {

+ return err

+ }

+ now := s.movesNow()

+ in := moveRecoveryInput{Now: now, Claim: c, Rec: rec}

+ ours := rec.Move != nil && rec.Move.ID == o.OpID

+ if c.Op.Kind == hostclaim.KindMoving && c.Op.Phase == hostclaim.PhaseMoving && !ours {

+ adm, err := hostops.ReadAdmission(ctx, store, o.Account, o.App, o.AppID)

+ if err != nil {

+ return err

+ }

+ if e, ok := adm.Value.Effects[o.OpID]; ok {

+ in.Start = &e

+ }

+ if c.Op.Kind == hostclaim.KindMoving && ours {

+ if in.Fence, err = s.sourceFence(ctx, c); err != nil {

+ return err

+ }

+ d := decideMoveRecovery(in)

+ for _, kind := range d.Pages {

+ s.pageMove(ctx, kind, o, c, rec, now)

+ }

+ switch d.Action {

+ case moveActResolveStart:

+ out, err := g.ResolveStart(ctx)

+ if errors.Is(err, hostops.ErrMoveStartFenceUnproven) {

+ log.Printf("console: moves: %s: the start of move %s is uncertain and its record has not advanced past it; left", o.Host, o.OpID)

+ return nil

+ }

+ if err == nil {

+ log.Printf("console: moves: %s: the start of move %s settled %s", o.Host, o.OpID, out)

+ }

+ return err

+ case moveActConvert:

+ return g.Convert(ctx, *in.Fence)

+ case moveActAbort:

+ co, err := hostops.ObserveConsoleMove(ctx, store, o.Account, o.App, o.Host)

+ if err != nil {

+ return err

+ }

+ if co != o {

+ return nil

+ }

+ if _, err := a.Control.ConsoleMove(ctx, co, hostops.ConsoleMoveAbort); err != nil {

+ return err

+ }

+ log.Printf("console: moves: %s: move %s is still preparing after %s; aborted", o.Host, o.OpID, moves.AbortAfter)

+ return nil

+ case moveActClaim:

+ return g.Claim(ctx, d.Claim)

+ case moveActRecord:

+ return g.Record(ctx, d.Record)

+ case moveActStampLost:

+ if s.stampLostMoveTarget == nil {

+ return hostops.ErrContract

+ }

+ return s.stampLostMoveTarget(ctx, o.Host, now)

+ case moveActClose:

+ if !s.moveCloseOverdue(o, d.Record, now) {

+ return nil

+ }

+ if err := g.Record(ctx, d.Record); err != nil {

+ return err

+ }

+ log.Printf("console: moves: %s: move %s closed by the console (%s); the release follows", o.Host, o.OpID, d.Record)

+ s.pageMove(ctx, movePageCloseOverdue, o, c, rec, now)

+ return nil

+ case moveActAbortOverdue:

+ if s.moveAbortOverdue(o, c.QuietAfter[rec.Move.To], now) {

+ s.pageMove(ctx, movePageAbortOverdue, o, c, rec, now)

+ }

+ return nil

+ }

+ return nil

+// sourceFence is the moving source's termination fence, when one exists:

+// the reference to control/box-fences/<source enrollment>.json and its exact

+// bytes. Convert verifies it; nothing here trusts it.

+func (s *Server) sourceFence(ctx context.Context, c *hostclaim.Claim) (*hostclaim.FenceRef, error) {

+ enrollment := c.Op.Intent["source_enrollment"]

+ if enrollment == "" {

+ return nil, nil

+ }

+ key := hostfence.FenceKey(enrollment)

+ body, _, err := s.cfg.Releases.Get(ctx, key)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, nil

+ }

+ if err != nil {

+ return nil, err

+ }

+ sum := sha256.Sum256(body)

+ return &hostclaim.FenceRef{Enrollment: enrollment, Key: key, SHA256: hex.EncodeToString(sum[:])}, nil

+// moveDueSince is when the pass first saw this close due, remembering now

+// if it had not. The memory is the process's: a restarted console waits the

+// grace again, which only delays the console's close or page. Entries a day

+// old are dropped.

+func (s *Server) moveDueSince(key string, now time.Time) time.Time {

+ s.moveCloseMu.Lock()

+ defer s.moveCloseMu.Unlock()

+ if s.moveCloseSeen == nil {

+ s.moveCloseSeen = map[string]time.Time{}

+ }

+ for k, at := range s.moveCloseSeen {

+ if now.Sub(at) > 24*time.Hour {

+ delete(s.moveCloseSeen, k)

+ }

+ first, ok := s.moveCloseSeen[key]

+ if !ok {

+ s.moveCloseSeen[key] = now

+ return now

+ }

+ return first

+// moveCloseOverdue reports whether this close has been due for

+// moveCloseGrace; when it has, the memory is cleared for the close.

+func (s *Server) moveCloseOverdue(o hostops.ConsoleMoveObservation, action hostops.MoveRecoveryRecordAction, now time.Time) bool {

+ key := o.Host + "|" + o.OpID + "|" + string(action)

+ if now.Sub(s.moveDueSince(key, now)) < moveCloseGrace {

+ return false

+ }

+ s.moveCloseMu.Lock()

+ delete(s.moveCloseSeen, key)

+ s.moveCloseMu.Unlock()

+ return true

+// moveAbortOverdue reports whether an abort with nothing owed has stayed

+// unclosed past its lane's horizon (quiet, the claim's quiet_after for the

+// target region, when the claim has one) plus moveCloseGrace, and has been

+// seen due for moveCloseGrace itself. The page it leads to is once per move

+// (pageMove), so the memory is kept.

+func (s *Server) moveAbortOverdue(o hostops.ConsoleMoveObservation, quiet, now time.Time) bool {

+ since := s.moveDueSince(o.Host+"|"+o.OpID+"|abort", now)

+ if now.Sub(since) < moveCloseGrace {

+ return false

+ }

+ return quiet.IsZero() || !now.Before(quiet.Add(moveCloseGrace))

+// pageMove sends one page for a finding (its kind, host and move) unless

+// the finding was paged before: ever, for a once-per-move page, or within

+// the last full hour, for the release deadline. The attempt's time is

+// recorded whether or not delivery worked (pages are best effort).

+func (s *Server) pageMove(ctx context.Context, kind movePageKind, o hostops.ConsoleMoveObservation, c *hostclaim.Claim, rec provision.Record, now time.Time) {

+ rule := string(kind) + " " + o.Host + " " + o.OpID

+ level := alerting.LevelWarning

+ every := time.Duration(0) // once per move

+ var subject, body string

+ switch kind {

+ case movePageReleaseDeadline:

+ level, every = alerting.LevelIncident, time.Hour

+ subject, body = moveWaitingPage(o.Host, o.OpID, rec.Move.To)

+ case movePagePreparing:

+ subject, body = movePreparingPageText(o.Host, o.OpID, rec.Move.To)

+ case movePageOrphan:

+ level = alerting.LevelIncident

+ subject, body = moveOrphanPage(o.Host, o.OpID)

+ case movePageCloseOverdue:

+ subject, body = moveCloseOverduePage(o.Host, o.OpID)

+ case movePageAbortOverdue:

+ subject, body = moveAbortOverduePage(o.Host, o.OpID, rec.Move.To, c.Op.Intent["source"])

+ default:

+ return

+ }

+ if s.store == nil {

+ return

+ }

+ last, paged, err := s.store.MovePageAttempt(rule)

+ if err != nil {

+ log.Printf("console: moves: %s: the page record for %s: %v", o.Host, rule, err)

+ return

+ }

+ if paged && (every == 0 || now.Sub(last) < every) {

+ return

+ }

+ if err := s.store.SetMovePageAttempt(rule, now); err != nil {

+ log.Printf("console: moves: %s: record the page %s: %v", o.Host, rule, err)

+ return

+ }

+ if err := alerting.Page(ctx, level, subject, body); err != nil {

+ log.Printf("console: moves: %s: page %s: %v", o.Host, kind, err)

+ }

+// moveWaitingPage is a rollback waiting past its release deadline (§7.8

+// waiting): the subject (MV-N325) and the body (MV-N326).

+func moveWaitingPage(host, id, to string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move waiting for %s to release: %s", to, host)

+ body = fmt.Sprintf("The move %s of %s is rolling back, and %s has not released it by its deadline. %s stays offline until a box in %s confirms it has stopped, or that box is declared lost.",

+ id, host, to, host, to)

+ return subject, body

+// movePreparingPageText is a move preparing for over an hour (§13): the

+// subject (MV-N327) and the body (MV-N328).

+func movePreparingPageText(host, id, to string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move preparing for over an hour: %s", host)

+ body = fmt.Sprintf("The move %s of %s to %s has been preparing for over an hour. It aborts itself six hours after the request if %s is still not ready.",

+ id, host, to, to)

+ return subject, body

+// moveOrphanPage is a claim held by a move the record does not carry (K3):

+// the subject (MV-N329) and the body (MV-N330).

+func moveOrphanPage(host, id string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move without a record: %s", host)

+ body = fmt.Sprintf("The host claim of %s is held by the move %s, and the instance record carries no move. Nothing releases it on its own. An operator must check the record.",

+ host, id)

+ return subject, body

+// moveCloseOverduePage is a close the console wrote because no box did: the

+// subject (MV-N331) and the body (MV-N332).

+func moveCloseOverduePage(host, id string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move close overdue: %s", host)

+ body = fmt.Sprintf("The move %s of %s was ready to close for more than 10 minutes and no box closed it. The console has closed it.", id, host)

+ return subject, body

+// moveAbortOverduePage is an abort that owes nothing and that its source has

+// not closed (fix round 1, ruling 1): the subject (MV-N333) and the body

+// (MV-N334).

+func moveAbortOverduePage(host, id, to, box string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move abort not closed: %s", host)

+ body = fmt.Sprintf("The abort of the move %s of %s has nothing left to park in %s, and %s has not closed it. %s closes it once it has removed the move's snapshots from its own bucket. Check %s.",

+ id, host, to, box, box, box)

+ return subject, body

diff --git a/internal/console/movesrecovery_test.go b/internal/console/movesrecovery_test.go

new file mode 100644

index 000000000..0e43f0ce7

--- /dev/null

+++ b/internal/console/movesrecovery_test.go

@@ -0,0 +1,721 @@

+package console

+import (

+ "context"

+ "crypto/rand"

+ "encoding/json"

+ "strings"

+ "sync"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+// The console's move pass (moves Plan 5 Task 7; claim Plan 10's

+// Recovery.Moves hook under the move-recovery grant). The decision table is

+// tested on values; every action is then driven end to end through the real

+// recovery pass (ControlWriter.RecoveryPass over hostops.Recovery.Pass) on a

+// host-claim console whose move the door started.

+// ---- the decision, on values ----

+var mvT0 = time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)

+func mvClaim(kind, phase string, mut ...func(*hostclaim.Claim)) *hostclaim.Claim {

+ c := &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"},

+ Op: hostclaim.Op{Kind: kind, Phase: phase, ID: "mv", Exec: "ie-box", Participants: []string{"ie-box", "uk-box"}, StartedAt: mvT0,

+ Intent: map[string]string{"source": "ie-box", "target": "uk-box"}}}

+ for _, m := range mut {

+ m(c)

+ }

+ return c

+func mvRecord(state string, mut ...func(*provision.Record)) provision.Record {

+ at := mvT0

+ r := provision.Record{Region: "ie", Move: &provision.MoveState{ID: "mv", Incarnation: "inc", From: "ie", To: "uk", State: state, RequestedAt: mvT0}}

+ switch state {

+ case provision.MoveLanding, provision.MoveCommitted:

+ r.Region = "uk"

+ r.Move.FlippedAt = &at

+ }

+ if state == provision.MoveCommitted {

+ r.Move.CommittedAt = &at

+ }

+ if state == provision.MoveRollingBack {

+ r.Move.FlippedAt = &at

+ r.Move.Rollback = &provision.MoveRollback{ID: "rb", TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceWaiting}

+ }

+ for _, m := range mut {

+ m(&r)

+ }

+ return r

+func mvAt(d time.Duration) *time.Time { t := mvT0.Add(d); return &t }

+func TestRecoveryPassWritesOnlyDecidedHalves(t *testing.T) {

+ now := mvT0.Add(time.Minute)

+ for _, c := range []struct {

+ name string

+ claim *hostclaim.Claim

+ rec provision.Record

+ action moveRecoveryAction

+ half hostops.MoveRecoveryClaimAction

+ }{

+ {"record landing, claim moving/sealed: the flip half", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding), moveActClaim, hostops.RecoveryMoveFlip},

+ {"record committed, claim landing/probed: the commit half", mvClaim(hostclaim.KindLanding, hostclaim.PhaseLandingProbed), mvRecord(provision.MoveCommitted), moveActClaim, hostops.RecoveryMoveCommit},

+ {"record rolling_back, claim landing: the rollback half", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack), moveActClaim, hostops.RecoveryMoveRollback},

+ {"record rolling_back, claim moving/sealed: RollBackFlipped (K22)", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveRollingBack), moveActClaim, hostops.RecoveryMoveRollback},

+ {"record landing, claim landing: in step", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveLanding), moveActNone, ""},

+ {"record cutting, claim moving/sealed: no decision", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveCutting), moveActNone, ""},

+ {"record committed, claim landing/probing: the claim is behind the probe", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveCommitted), moveActNone, ""},

+ {"record committed, claim landing/committed, stamps missing", mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), mvRecord(provision.MoveCommitted), moveActNone, ""},

+ {"record landing without its flip stamp", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.FlippedAt = nil }), moveActNone, ""},

+ {"record rolling_back in the target's region", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack, func(r *provision.Record) { r.Region = "uk" }), moveActNone, ""},

+ {"record preparing, claim moving: nothing yet", mvClaim(hostclaim.KindMoving, hostclaim.PhaseMoving), mvRecord(provision.MovePreparing), moveActNone, ""},

+ {"another move's record", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.ID = "other" }), moveActNone, ""},

+ {"another incarnation's record", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.Incarnation = "old" }), moveActNone, ""},

+ {"a relocating claim is region HA's", mvClaim(hostclaim.KindRelocating, "pinning"), mvRecord(provision.MoveRelocating), moveActNone, ""},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ got := decideMoveRecovery(moveRecoveryInput{Now: now, Claim: c.claim, Rec: c.rec})

+ if got.Action != c.action || got.Claim != c.half {

+ t.Fatalf("decided %+v, want %s %s", got, c.action, c.half)

+ }

+ })

+ }

+// One eligible action per observation, in the brief's order: a pending

+// claim half comes before the deadline, the deadline before a close, the

+// lost target's stamp before the close it enables.

+func TestRecoveryPassOneActionPerObservation(t *testing.T) {

+ past := mvT0.Add(-time.Hour)

+ // Flip half pending and the landing deadline passed: the half first.

+ d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })})

+ if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveFlip {

+ t.Fatalf("flip and deadline: %+v", d)

+ }

+ // With the claim in step, the deadline.

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })})

+ if d.Action != moveActRecord || d.Record != hostops.RecoveryMoveDeadlineRollback {

+ t.Fatalf("the deadline: %+v", d)

+ }

+ // The earlier of the cut and landing deadlines (J11).

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.CutDeadline = &past; r.Move.LandingDeadline = mvAt(time.Hour) })})

+ if d.Action != moveActRecord {

+ t.Fatalf("the cut deadline: %+v", d)

+ }

+ // A lost, released target: the stamp, never the close it enables.

+ lost := func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"} }

+ rb := func(r *provision.Record) {

+ r.Move.Rollback.TargetPhase, r.Move.Rollback.SourcePhase = provision.RbTargetReleased, provision.RbSourceOpened

+ r.Move.Rollback.RecoveredAt, r.Move.Rollback.OpenedAt = mvAt(0), mvAt(0)

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened, lost), Rec: mvRecord(provision.MoveRollingBack, rb)})

+ if d.Action != moveActStampLost {

+ t.Fatalf("the lost target: %+v", d)

+ }

+ // A chore of the target's region still owed: neither.

+ owed := func(c *hostclaim.Claim) {

+ c.Obligations = []hostclaim.Obligation{{ID: "o", Op: "mv", Kind: "park-replica", Region: "uk", Owner: "region:uk"}}

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened, lost, owed), Rec: mvRecord(provision.MoveRollingBack, rb)})

+ if d.Action != moveActNone {

+ t.Fatalf("the lost target with its region's chore owed: %+v", d)

+ }

+ // Every close waits for the op's chores (the grant's rule, M4).

+ stamped := func(r *provision.Record) { r.Move.OpenedAt, r.Move.SourceParkedAt = mvAt(0), mvAt(0) }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted, owed), Rec: mvRecord(provision.MoveCommitted, stamped)})

+ if d.Action != moveActNone {

+ t.Fatalf("a close with a chore owed: %+v", d)

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), Rec: mvRecord(provision.MoveCommitted, stamped)})

+ if d.Action != moveActClose || d.Record != hostops.RecoveryMoveClose {

+ t.Fatalf("the committed close: %+v", d)

+ }

+ for _, c := range []struct {

+ claim *hostclaim.Claim

+ rec provision.Record

+ want hostops.MoveRecoveryRecordAction

+ }{

+ {mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened), mvRecord(provision.MoveRollingBack, rb, func(r *provision.Record) { r.Move.Rollback.TargetParkedAt = mvAt(0) }), hostops.RecoveryMoveRolledBackClose},

+ {mvClaim(hostclaim.KindReverting, hostclaim.PhaseOpened), mvRecord(provision.MoveReverting, func(r *provision.Record) { r.Move.ReopenedAt = mvAt(0) }), hostops.RecoveryMoveRevertClose},

+ } {

+ if d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: c.claim, Rec: c.rec}); d.Action != moveActClose || d.Record != c.want {

+ t.Fatalf("%s/%s: %+v, want close %s", c.claim.Op.Kind, c.claim.Op.Phase, d, c.want)

+ }

+ // An abort the console never closes (fix round 1, ruling 1): only the

+ // source closes it, after it retired its own state. Due, it is at most

+ // a page.

+ if d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindMoving, hostclaim.PhaseAborting), Rec: mvRecord(provision.MoveAborting)}); d.Action != moveActAbortOverdue || d.Record != "" {

+ t.Fatalf("a due abort: %+v, want only the overdue check", d)

+ }

+ // The record closed, the claim not released: the release.

+ closed := func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: "mv", Outcome: "done", At: mvT0}

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), Rec: mvRecord(provision.MoveCommitted, closed)})

+ if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveRelease {

+ t.Fatalf("the release from LastMove: %+v", d)

+ }

+// ---- the pass, end to end ----

+// movePassWorld is a host-claim console whose move the request door

+// started (j17StartedMove), with the console's own move pass bound as the

+// Recovery.Moves hook, pages captured from the armed alert sinks, and the

+// pass's clock under the test's control.

+type movePassWorld struct {

+ fx *consoleFixture

+ id string

+ auth hostclaim.FenceAuthority

+ clock time.Time

+ mu sync.Mutex

+ pages []pagePayload

+type pagePayload struct{ Kind, Level, Subject, Body string }

+func newMovePassWorld(t *testing.T, fx *consoleFixture, id string) *movePassWorld {

+ t.Helper()

+ w := &movePassWorld{fx: fx, id: id, clock: time.Now().UTC()}

+ fx.srv.movesClock = func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.clock }

+ // As ConfigureClaimRuntime binds it: the lost target's stamp over the

+ // conditional record store under the fixture's data boundary.

+ records, err := hostops.NewConditionalRecordStore(fx.rel, fx.pub)

+ if err != nil {

+ t.Fatal(err)

+ }

+ fx.srv.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

+ return hostops.StampLostMoveTargetParked(ctx, records, host, now)

+ }

+ alerting.Arm(&alerting.Sinks{WebhookURL: "http://pages.test/hook", Post: func(_ string, body []byte) error {

+ var p pagePayload

+ if err := json.Unmarshal(body, &p); err != nil {

+ return err

+ }

+ w.mu.Lock()

+ w.pages = append(w.pages, p)

+ w.mu.Unlock()

+ return nil

+ }})

+ t.Cleanup(func() { alerting.Arm(nil) })

+ return w

+func (w *movePassWorld) advance(d time.Duration) {

+ w.mu.Lock()

+ w.clock = w.clock.Add(d)

+ w.mu.Unlock()

+// pass runs one console recovery pass with only the moves hook live.

+func (w *movePassWorld) pass(t *testing.T) {

+ t.Helper()

+ a := w.fx.srv.cfg.ClaimAuthority

+ nop := func(context.Context) error { return nil }

+ nopObs := func(context.Context, hostclaim.Observed) error { return nil }

+ r := hostops.Recovery{Moves: w.fx.srv.movesRecovery, MoveFenceAuthority: w.auth,

+ Controller: hostops.Controller{Store: w.fx.srv.cfg.Releases, Control: a.Control, Authority: a, Now: time.Now, NewID: claimOperationID},

+ Purge: nopObs, Placement: nopObs, SweepRecords: nop, SweepPurges: nop, SweepAdmissions: nop}

+ if err := a.Control.RecoveryPass(context.Background(), r); err != nil {

+ t.Fatalf("the recovery pass: %v", err)

+ }

+func (w *movePassWorld) claim(t *testing.T) *hostclaim.Claim {

+ t.Helper()

+ o, err := hostclaim.Read(context.Background(), w.fx.rel, "acme.go.tito.io")

+ if err != nil || o.Absent {

+ t.Fatalf("read the claim: %v", err)

+ }

+ return o.Claim

+func (w *movePassWorld) record(t *testing.T) provision.Record {

+ t.Helper()

+ rec, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "titogo", "acme.go.tito.io")

+ if err != nil || rec == nil {

+ t.Fatalf("read the record: %v", err)

+ }

+ return *rec

+// setClaim rewrites the claim under the raw bucket, as a box would.

+func (w *movePassWorld) setClaim(t *testing.T, mut func(*hostclaim.Claim)) {

+ t.Helper()

+ ctx := context.Background()

+ obs, err := hostclaim.Read(ctx, w.fx.rel, "acme.go.tito.io")

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(ctx, w.fx.rel, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil {

+ t.Fatalf("rewrite the claim: %v", err)

+ }

+func (w *movePassWorld) startEffect(t *testing.T) hostops.Effect {

+ t.Helper()

+ a, err := hostops.ReadAdmission(context.Background(), w.fx.rel, "bab", "titogo", claimMoveAppID)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return a.Value.Effects[w.id]

+// setStartEffect rewrites the move-start effect of the admission object.

+func (w *movePassWorld) setStartEffect(t *testing.T, mut func(*hostops.Effect)) {

+ t.Helper()

+ ctx := context.Background()

+ body, etag, err := w.fx.rel.Get(ctx, "apps/bab/titogo/admission.json")

+ if err != nil {

+ t.Fatal(err)

+ }

+ var a hostops.AppAdmission

+ if err := json.Unmarshal(body, &a); err != nil {

+ t.Fatal(err)

+ }

+ e := a.Effects[w.id]

+ mut(&e)

+ a.Effects[w.id] = e

+ if body, err = json.Marshal(a); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := w.fx.rel.Put(ctx, "apps/bab/titogo/admission.json", body, etag); err != nil {

+ t.Fatal(err)

+ }

+func (w *movePassWorld) paged(t *testing.T) []pagePayload {

+ t.Helper()

+ w.mu.Lock()

+ defer w.mu.Unlock()

+ return append([]pagePayload(nil), w.pages...)

+func startedMovePass(t *testing.T, code string) *movePassWorld {

+ t.Helper()

+ fx, _, _, id := j17StartedMove(t, code)

+ return newMovePassWorld(t, fx, id)

+func landingMovePass(t *testing.T, code string) *movePassWorld {

+ t.Helper()

+ fx, _, _ := j4LandingMove(t, code)

+ return newMovePassWorld(t, fx, j17MoveState(t, fx).ID)

+func newRollbackID(t *testing.T) string {

+ t.Helper()

+ id, err := moves.NewRollbackID(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return id

+// TestRecoveryPassRollsBackAtTheLandingDeadline: a landing past the earlier

+// of its deadlines is rolled back by the record (one pass), and the claim

+// follows on the next (the rollback half). Each pass writes one object.

+func TestRecoveryPassRollsBackAtTheLandingDeadline(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-deadline")

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveLanding {

+ t.Fatalf("rolled back before the deadline: %+v", m)

+ }

+ past := time.Now().UTC().Add(-time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move.LandingDeadline = &past })

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move.State != provision.MoveRollingBack || rec.Move.Rollback == nil || rec.Move.Rollback.Reason != provision.RollbackLandingDeadline || rec.Region != "ie" {

+ t.Fatalf("the deadline's rollback: region %s %+v", rec.Region, rec.Move)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the pass that wrote the record wrote the claim too")

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindRestoring || c.Op.ID != w.id {

+ t.Fatalf("the claim did not follow the rollback: %+v", c.Op)

+ }

+ if after := w.record(t); after.Move.Rollback.ID != rec.Move.Rollback.ID || after.RecordSeq != rec.RecordSeq {

+ t.Fatalf("the claim's pass wrote the record: %+v", after.Move)

+ }

+// TestRecoveryPassSettlesAnOrphanStartThenReleases (K3): the request door's

+// record write was lost, so the claim is moving/"" and the record carries no

+// move, with the start effect uncertain. Under ten minutes the pass leaves

+// it. After, it settles the start not-applied from record_seq (no handle),

+// and on the next pass releases the claim to serving under the source. A

+// landing orphan is paged once and never released.

+func TestRecoveryPassSettlesAnOrphanStartThenReleases(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-orphan")

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move = nil })

+ w.setStartEffect(t, func(e *hostops.Effect) { e.State, e.ResultReference = "uncertain", "" })

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving || w.startEffect(t).State != "uncertain" {

+ t.Fatalf("acted inside ten minutes: claim %+v effect %+v", c.Op, w.startEffect(t))

+ }

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.StartedAt = time.Now().UTC().Add(-11 * time.Minute) })

+ w.pass(t)

+ if e := w.startEffect(t); e.State != "complete" || e.ResultReference != "not-applied" {

+ t.Fatalf("the start was not settled: %+v", e)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving {

+ t.Fatalf("the settling pass released too: %+v", c.Op)

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(c) != nil {

+ t.Fatalf("the orphan was not released: %+v", c.Op)

+ }

+ if p := w.paged(t); len(p) != 0 {

+ t.Fatalf("a moving orphan paged: %+v", p)

+ }

+ // A landing orphan pages once, and is left.

+ l := landingMovePass(t, "code-mvpass-orphan-landing")

+ j17SetRecord(t, l.fx, func(r *provision.Record) { r.Move = nil })

+ before := claimBytes(t, l.fx, "acme.go.tito.io")

+ l.pass(t)

+ if claimBytes(t, l.fx, "acme.go.tito.io") != before {

+ t.Fatal("a landing orphan's claim was written")

+ }

+ p := l.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelIncident || !strings.Contains(p[0].Subject, "acme.go.tito.io") || !strings.Contains(p[0].Body, l.id) {

+ t.Fatalf("the landing orphan's pages: %+v", p)

+ }

+// oneBoxFenceAuthority is c7MoveAuthority with the one-box-region check the

+// grant's Convert needs (the console's hostfence.ClaimFenceAuthority has

+// both).

+type oneBoxFenceAuthority struct{ c7MoveAuthority }

+func (oneBoxFenceAuthority) CheckOneBoxRegion(context.Context, string, string) error { return nil }

+// TestRecoveryPassConvertsAFencedSourceInTwoPasses (K13): a source

+// terminated before the flip. The first pass writes the record's relocation

+// decision, the second the claim's conversion to relocating.

+func TestRecoveryPassConvertsAFencedSourceInTwoPasses(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-convert")

+ w.auth = oneBoxFenceAuthority{c7MoveAuthority{region: "ie"}}

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MovePreparing {

+ t.Fatalf("converted with no fence: %+v", m)

+ }

+ terminatedSourceFence(t, w.fx, "ie-box", "uk-box")

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveRelocating {

+ t.Fatalf("the decision pass: %+v", m)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving {

+ t.Fatalf("the decision pass converted the claim: %+v", c.Op)

+ }

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != "pinning" || c.Op.ID != w.id {

+ t.Fatalf("the conversion pass: %+v", c.Op)

+ }

+ w.pass(t) // a relocating claim is region HA's: nothing

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRelocating {

+ t.Fatalf("after the conversion: %+v", c.Op)

+ }

+// lostTargetRollback puts the landing world into a rollback whose target

+// box was lost and released: the source recovered and opened, the claim

+// restoring/opened under the source.

+func lostTargetRollback(t *testing.T, w *movePassWorld) {

+ t.Helper()

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box"

+ c.Lost = []string{"uk-box"}

+ c.Obligations = nil

+ })

+ rb, at := newRollbackID(t), time.Now().UTC().Add(-time.Hour)

+ deadline := at.Add(15 * time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Region = "ie"

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: rb, RequestedAt: &at, Reason: provision.RollbackDeclaredLost, TargetPhase: provision.RbTargetReleased,

+ SourcePhase: provision.RbSourceOpened, RecoveredAt: &at, OpenedAt: &at, ReleaseDeadline: &deadline}

+ })

+// TestRecoveryPassStampsALostTargetThenCloses (p10 Task G): with the target

+// lost and released and nothing of its box or region owed, the pass stamps

+// target_parked_at outside the grant; the close follows once overdue, then

+// the release.

+func TestRecoveryPassStampsALostTargetThenCloses(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-lost")

+ lostTargetRollback(t, w)

+ w.pass(t)

+ rb := w.record(t).Move.Rollback

+ if rb.TargetParkedAt == nil || rb.TargetPhase != provision.RbTargetCleaned {

+ t.Fatalf("the lost target was not stamped: %+v", rb)

+ }

+ w.pass(t)

+ if w.record(t).Move == nil {

+ t.Fatal("closed inside the boxes' grace")

+ }

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "rolled_back" {

+ t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRestoring {

+ t.Fatalf("the close's pass released too: %+v", c.Op)

+ }

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

+ t.Fatalf("not released: %+v", c.Op)

+ }

+// committedAndOpened puts the landing world at a committed, opened and

+// source-parked move with nothing owed: the close is due.

+func committedAndOpened(t *testing.T, w *movePassWorld) {

+ t.Helper()

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseCommitted, nil })

+ at := time.Now().UTC().Add(-time.Hour)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move.State, r.Move.LandingPhase = provision.MoveCommitted, provision.LandProbed

+ r.Move.CommittedAt, r.Move.OpenedAt, r.Move.SourceParkedAt = &at, &at, &at

+ })

+// TestRecoveryPassFinishesTheClose: a close the boxes did not write within

+// moveCloseGrace is overdue. The pass pages it once, closes the record

+// (LastMove done) and, on the next pass, releases under the target.

+func TestRecoveryPassFinishesTheClose(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-close")

+ committedAndOpened(t, w)

+ w.pass(t)

+ if w.record(t).Move == nil || len(w.paged(t)) != 0 {

+ t.Fatalf("closed or paged inside the grace: %+v", w.paged(t))

+ }

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != w.id || rec.LastMove.Outcome != "done" {

+ t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted {

+ t.Fatalf("released in the close's pass: %+v", c.Op)

+ }

+ if p := w.paged(t); len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Body, w.id) {

+ t.Fatalf("the overdue close's page: %+v", p)

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "uk-box" || hostclaim.RequireReleasedMove(w.id)(c) != nil {

+ t.Fatalf("not released under the target: %+v", c.Op)

+ }

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again: %+v", w.paged(t))

+ }

+// TestRecordFirstCloseCrashIsFinishedByThePass (ruling M4): a box closed the

+// record and crashed before the claim's release. The record carries

+// LastMove and no move; the claim is still in the move's closing op. The

+// console's pass releases from LastMove, under the executor the outcome

+// names, with no grace (the release is the record's decision).

+func TestRecordFirstCloseCrashIsFinishedByThePass(t *testing.T) {

+ for _, c := range []struct {

+ name, outcome, exec string

+ setup func(*testing.T, *movePassWorld)

+ }{

+ {"committed", "done", "uk-box", func(t *testing.T, w *movePassWorld) {

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseCommitted, nil })

+ }},

+ {"rolled back", "rolled_back", "ie-box", func(t *testing.T, w *movePassWorld) {

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec, c.Obligations = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box", nil

+ })

+ }},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-crash-"+strings.ReplaceAll(c.name, " ", "-"))

+ c.setup(t, w)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ if c.outcome == "rolled_back" {

+ r.Region = "ie"

+ }

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: c.outcome, At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != c.exec || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+ }

+ t.Run("reverted", func(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-crash-reverted")

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Obligations = hostclaim.KindReverting, hostclaim.PhaseOpened, nil

+ if c.Op.Journal == nil {

+ c.Op.Journal = map[string]string{}

+ }

+ c.Op.Journal[hostclaim.JournalRevPhase] = hostclaim.PhaseOpened

+ })

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: "rolled_back", At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+ t.Run("aborted", func(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-crash-aborted")

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseAborting, nil })

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: "aborted", At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+// TestRecoveryPassAbortsAfterSixHours (K18): a move still preparing

+// moves.AbortAfter after its request is aborted through ConsoleMove; before

+// that the pass pages once at an hour and writes nothing.

+func TestRecoveryPassAbortsAfterSixHours(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-abort")

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ requested := w.record(t).Move.RequestedAt

+ w.advance(30 * time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 0 {

+ t.Fatalf("paged at 30 minutes: %+v", w.paged(t))

+ }

+ w.advance(time.Hour)

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MovePreparing {

+ t.Fatalf("aborted early: %+v", m)

+ }

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Subject, "acme.go.tito.io") {

+ t.Fatalf("the hour's page: %+v", p)

+ }

+ w.mu.Lock()

+ w.clock = requested.Add(moves.AbortAfter + time.Minute)

+ w.mu.Unlock()

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveAborting {

+ t.Fatalf("not aborted at six hours: %+v", m)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the abort wrote the claim (the source's abort takes it to aborting)")

+ }

+// TestRecoveryPassNeverClosesAnAbort (fix round 1, ruling 1): the source's

+// retirement of its own state is blocked, so it never closes the aborting

+// record, though nothing is owed under the op. The console does not close

+// it either: once the abort has been due past the lane's horizon (the

+// claim's quiet_after for the target region) plus moveCloseGrace, it pages

+// once, and the record stays aborting.

+func TestRecoveryPassNeverClosesAnAbort(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-abort-blocked")

+ horizon := w.clock.Add(20 * time.Minute)

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Phase, c.Obligations = hostclaim.PhaseAborting, nil

+ if c.QuietAfter == nil {

+ c.QuietAfter = map[string]time.Time{}

+ }

+ c.QuietAfter["uk"] = horizon

+ })

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move.State = provision.MoveAborting })

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ w.pass(t)

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 0 {

+ t.Fatalf("paged before the lane's horizon plus the grace: %+v", w.paged(t))

+ }

+ w.advance(20 * time.Minute) // past quiet_after + the grace

+ w.pass(t)

+ w.advance(2 * time.Hour)

+ w.pass(t)

+ if m := w.record(t).Move; m == nil || m.State != provision.MoveAborting {

+ t.Fatalf("the console closed the abort: %+v %+v", m, w.record(t).LastMove)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the console wrote the aborting claim")

+ }

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Body, w.id) || !strings.Contains(p[0].Body, "ie-box") {

+ t.Fatalf("the overdue abort's pages: %+v", p)

+ }

+// TestConsolePagesThroughAlerting (§13, M14; fix round 1, ruling 3): the

+// pass pages through alerting.Page on the armed sinks. A rollback waiting

+// past its release deadline is an incident, paged again only a full hour

+// after the last attempt: two passes either side of a whole hour since the

+// deadline do not both page. The last attempt's time is durable, in the

+// console's store, so a restarted console keeps the spacing.

+func TestConsolePagesThroughAlerting(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-pages")

+ rb, at := newRollbackID(t), w.clock.Add(-74*time.Minute)

+ deadline := at.Add(15 * time.Minute) // 59 minutes ago

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Region = "ie"

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: rb, RequestedAt: &at, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetReleasing,

+ SourcePhase: provision.RbSourceWaiting, ReleaseDeadline: &deadline}

+ })

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseClaimed })

+ first := w.clock

+ w.pass(t)

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Kind != "page" || p[0].Level != alerting.LevelIncident || !strings.Contains(p[0].Subject, "uk") || !strings.Contains(p[0].Body, w.id) {

+ t.Fatalf("the release deadline's page: %+v", p)

+ }

+ // Across the whole hour since the deadline (59 to 61 minutes): no page.

+ w.advance(2 * time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again two minutes later, across the hour since the deadline: %+v", w.paged(t))

+ }

+ // A restarted console: the in-memory state is gone, the store is not.

+ w.fx.srv.moveCloseSeen = nil

+ if last, ok, err := w.fx.st.MovePageAttempt("release-deadline acme.go.tito.io " + w.id); err != nil || !ok || !last.Equal(first.Truncate(time.Second)) {

+ t.Fatalf("the last attempt's durable time: %v %v %v, want %v", last, ok, err, first)

+ }

+ w.advance(56 * time.Minute) // 58 minutes after the first page

+ w.pass(t)

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again inside the hour: %+v", w.paged(t))

+ }

+ w.advance(3 * time.Minute) // 61 minutes after the first page

+ w.pass(t)

+ if len(w.paged(t)) != 2 {

+ t.Fatalf("not paged again a full hour after the last attempt: %+v", w.paged(t))

+ }

diff --git a/internal/console/server.go b/internal/console/server.go

index 89eb5c44e..1e5c08d1d 100644

--- a/internal/console/server.go

+++ b/internal/console/server.go

@@ -828,9 +828,18 @@ type Server struct {

claimBackendObserver func(backendEvent) // fixture recorder only

claimEntries claimInventory

claimWorkers map[string]func(context.Context)

- dbHandleKey []byte

- cfg Config

- store *Store

+ // The console's move pass (movesrecovery.go): its clock (nil is

+ // time.Now; a TEST SEAM) and when each close was first seen due.

+ movesClock func() time.Time

+ moveCloseMu sync.Mutex

+ moveCloseSeen map[string]time.Time

+ // stampLostMoveTarget is hostops.StampLostMoveTargetParked over the

+ // conditional record store, bound by ConfigureClaimRuntime with the

+ // moves switch on; nil otherwise.

+ stampLostMoveTarget func(ctx context.Context, host string, now time.Time) error

+ dbHandleKey []byte

+ cfg Config

+ store *Store

// appRegionOf and beforeClaimInsert are TEST SEAMS (nil in

// production): a store failure on the app-region read (C20), and a

// competing insert landing after the region was resolved.

diff --git a/internal/console/store.go b/internal/console/store.go

index f005f1ddc..51d0619d6 100644

--- a/internal/console/store.go

+++ b/internal/console/store.go

@@ -350,6 +350,14 @@ func (s *Store) migrate() error {

`CREATE TABLE IF NOT EXISTS edge_proof_seen (

id INTEGER PRIMARY KEY CHECK (id = 1),

seen_at INTEGER NOT NULL

+);`,

+ // The console's move pass pages (movesrecovery.go): the last

+ // attempted page per finding, so a repeating page keeps a full

+ // hour between attempts across restarts, and a once-only page is

+ // never repeated.

+ `CREATE TABLE IF NOT EXISTS move_pages (

+ rule TEXT PRIMARY KEY,

+ attempted_at INTEGER NOT NULL

);`,

} {

if _, err := s.db.Exec(stmt); err != nil {

@@ -2182,6 +2190,29 @@ ON CONFLICT(box, rule, address) DO UPDATE SET firing = excluded.firing`, box, ru

return err

}

+// ---- move pass pages (movesrecovery.go) -------------------------------------

+// MovePageAttempt is when the move pass last attempted the page for rule,

+// to the second, and whether it ever did.

+func (s *Store) MovePageAttempt(rule string) (time.Time, bool, error) {

+ var at int64

+ err := s.db.QueryRow(`SELECT attempted_at FROM move_pages WHERE rule = ?`, rule).Scan(&at)

+ if err == sql.ErrNoRows {

+ return time.Time{}, false, nil

+ }

+ if err != nil {

+ return time.Time{}, false, err

+ }

+ return time.Unix(at, 0).UTC(), true, nil

+// SetMovePageAttempt records an attempt at the page for rule at.

+func (s *Store) SetMovePageAttempt(rule string, at time.Time) error {

+ _, err := s.db.Exec(`INSERT INTO move_pages (rule, attempted_at) VALUES (?, ?)

+ON CONFLICT(rule) DO UPDATE SET attempted_at = excluded.attempted_at`, rule, at.Unix())

+ return err

// ---- grants (the console's Keymail access tokens) ---------------------------

// SetGrantToken stores the Keymail access token the user's OAuth approval

diff --git a/internal/console/store_gate.go b/internal/console/store_gate.go

index aac845c62..c2d72d223 100644

--- a/internal/console/store_gate.go

+++ b/internal/console/store_gate.go

@@ -209,6 +209,7 @@ var claimSQLTables = map[string]claimSQLTable{

"memberships": {NonApp: "account membership, keyed by account and address; names no app"},

"invites": {NonApp: "account invites, keyed by account and address; names no app"},

"alert_seen": {NonApp: "operator alert memory, keyed by box, rule and address"},

+ "move_pages": {NonApp: "the move pass's last page attempt per finding (rule names the host and the move); no app column"},

"edge_proof_seen": {NonApp: "single-row edge proof marker"},

"cli_auth_requests": {NonApp: "device-code logins in flight, keyed by code hash"},

"step_ups": {NonApp: "passkey step-up marks, keyed by session hash"},

diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt

index 96aaeeb23..8e7d07046 100644

--- a/internal/console/testdata/claim-authority-sites.txt

+++ b/internal/console/testdata/claim-authority-sites.txt

@@ -92,6 +92,7 @@ claim_runtime.go|ConfigureClaimRuntime|hostops.NewControlWriter(raw, s.cfg.Relea

claim_runtime.go|ConfigureClaimRuntime|hostops.NewObservedMutationBoundary(true, records, check, s.claimMutationObserver)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewProductionMutationCheck(raw, control)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewVersionedBoundary(true, s.cfg.DBVersions, raw, control, s.cfg.Bucket, dataBuckets, s.claimMutationObserver)

+claim_runtime.go|ConfigureClaimRuntime|hostops.StampLostMoveTargetParked(ctx, records, host, now)

claim_runtime.go|sweepOneClaimAdmission|c.Control.RecoverSQLTransaction(ctx, app.Account, app.Name, appID, id, s.store)

claim_sql_auto.go|withAutoAppSQL|s.WithAppMutation(pctx, []sqlGrant{permitGrant(pctx, s.claimBucket, acct.Sqid, scope)}, call)

claim_sql_grants.go|WithAppMutation|p.BindTransaction(database)

@@ -105,6 +106,13 @@ geoippublish.go|Commit|os.Open(st.path)

hostclaimbackfill.go|RunClaimRecoveryPass|a.Control.RunRecovery(ctx, s.runClaimRecoveryPassBody)

hostwakereset.go|registerWakeRetryReset|a.Control.RegisterStart(ctx, h.Account, h.App, reg.AppID, id, reg, registeredAt)

hostwakereset.go|registerWakeRetryReset|start.FinishWakeRetryReset(cleanup)

+movesrecovery.go|movesRecovery|a.Control.ConsoleMove(ctx, co, hostops.ConsoleMoveAbort)

+movesrecovery.go|movesRecovery|g.Claim(ctx, d.Claim)

+movesrecovery.go|movesRecovery|g.Convert(ctx, *in.Fence)

+movesrecovery.go|movesRecovery|g.Record(ctx, d.Record)

+movesrecovery.go|movesRecovery|g.ResolveStart(ctx)

+movesrecovery.go|movesRecovery|s.stampLostMoveTarget(ctx, o.Host, now)

scanpass.go|storeLister|objstore.S3ClientFor(ctx, p.Region, p.Endpoint)

store.go|Open|sql.Open("sqlite", path)

store_gate.go|Connect|k.g.d.Open(k.g.dsn)

diff --git a/internal/host/movesabort.go b/internal/host/movesabort.go

index 7f1ecd3e1..23a9a25f9 100644

--- a/internal/host/movesabort.go

+++ b/internal/host/movesabort.go

@@ -49,13 +49,16 @@ import (

// release, and is a no-op once everything is gone: nothing infers from

// a missing journal that it already ran, and any error (a listing, a

// delete, the journal directory) ends the pass unreleased;

-// 5. it waits until no obligation under the op remains, then releases with

-// hostclaim.ReleaseAbort (K7: from moving/aborting only, so never before

-// the lane park was queued), and last clears the record with

-// provision.MoveGuardedAbortComplete (§7.5: the claim releases first,

-// the record clears second). A round that finds the claim already

-// serving and recording this move's release completes the record; any

-// other serving claim is left alone.

+// 5. it waits until no obligation under the op remains, then clears the

+// record with provision.MoveGuardedAbortComplete, and last releases

+// with hostclaim.ReleaseAbort (K7: from moving/aborting only, so never

+// before the lane park was queued). Record first, then the claim's

+// release (§7.5 and §19 K8 as Paul's ruling M4 amends them): a crash

+// between leaves the record at rest and the claim moving/aborting with

+// nothing owed, and the console's recovery pass releases from LastMove

+// (Plan 5 Task 7). A round that finds the claim already serving and

+// recording this move's release (a binary that closed claim first)

+// completes the record; any other serving claim is left alone.

// The target region's box, for a record aborting whose To is its region:

// movesParticipantRecord answers it as the target on the state alone (the

@@ -105,6 +108,7 @@ func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec prov

}

op := obs.Claim.Op

mine := op.Kind == hostclaim.KindMoving && op.ID == id

+ releaseAfter := false

switch {

case mine && op.Exec != cfg.Box:

w.notice(rec.Host+"#abort", "moves: %s: aborting move %s: the claim's executor is %q, not this box; left for the console pass", rec.Host, id, op.Exec)

@@ -122,19 +126,23 @@ func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec prov

}

fallthrough

case mine && op.Phase == hostclaim.PhaseAborting:

- // Every pass, before the release: the retirement is idempotent, and

- // a release never follows from inferring that it already ran (fix

+ // Every pass, before the close: the retirement is idempotent, and

+ // a close never follows from inferring that it already ran (fix

// round 1). Any error, the journal directory's included, ends the

- // pass with the claim still aborting.

+ // pass with the record and the claim still aborting.

if err := w.retireOwnMoveState(ctx, rec); err != nil {

return err

}

- if released, err := w.releaseAbort(ctx, s, rec, obs); err != nil || !released {

- return err

+ // Nothing owed under the op, checked before the record's close

+ // (ruling M4), as the console's grant checks it.

+ if w.abortOwes(rec, obs) {

+ return nil

}

+ releaseAfter = true

case hostclaim.RequireReleasedMove(id)(obs.Claim) == nil:

- // This abort released the claim before a crash: finish. The own

- // state went before the release; retiring it again is a no-op.

+ // A binary that closed claim first released before a crash:

+ // finish the record. The own state went before the release;

+ // retiring it again is a no-op.

if err := w.retireOwnMoveState(ctx, rec); err != nil {

return err

}

@@ -152,7 +160,13 @@ func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec prov

}

nudgeSerial(ctx, s)

log.Printf("moves: %s: move %s aborted; the instance stays in %q", rec.Host, id, rec.Region)

- return nil

+ if !releaseAfter {

+ return nil

+ }

+ if err := w.abFault("after-abort-record"); err != nil {

+ return err

+ }

+ return w.releaseAbort(ctx, s, rec)

}

// abortClaim is steps 1 to 3: the staging lease, the journalled bound and

@@ -225,23 +239,27 @@ func (w *movesWorker) abortClaim(ctx context.Context, s release.Store, rec provi

return true, nil

}

-// releaseAbort is step 5: the release once no obligation of the op remains.

-// released is false while the lane park is still owed.

-func (w *movesWorker) releaseAbort(ctx context.Context, s release.Store, rec provision.Record, obs hostclaim.Observed) (bool, error) {

+// abortOwes is step 5's wait: true, with a notice, while an obligation of

+// the op remains (the lane park the target region has not acknowledged).

+func (w *movesWorker) abortOwes(rec provision.Record, obs hostclaim.Observed) bool {

id := rec.Move.ID

for _, o := range obs.Claim.Obligations {

if o.Op == id {

w.notice(rec.Host+"#abort-wait", "moves: %s: aborting move %s waits for %s to park the staging lane (%s %s)", rec.Host, id, rec.Move.To, o.Kind, o.Region+o.Box)

- return false, nil

+ return true

}

- if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseAbort(id, w.cfg.Box)); err != nil {

- return false, fmt.Errorf("release the host claim: %w", err)

- }

- if err := w.abFault("after-release-abort"); err != nil {

- return false, err

+ return false

+// releaseAbort is step 5's second half: the record already says aborted, so

+// hostclaim.ReleaseAbort from a fresh read. A failure is left to the

+// console's recovery pass, which releases from LastMove.

+func (w *movesWorker) releaseAbort(ctx context.Context, s release.Store, rec provision.Record) error {

+ if err := releaseAfterClose(ctx, s, rec.Host, rec.Move.ID, hostclaim.ReleaseAbort(rec.Move.ID, w.cfg.Box)); err != nil {

+ return err

}

- return true, nil

+ return w.abFault("after-release-abort")

}

// retireOwnMoveState is step 4: the move's snapshots in this box's own

diff --git a/internal/host/movesclose_order_test.go b/internal/host/movesclose_order_test.go

new file mode 100644

index 000000000..5182afef2

--- /dev/null

+++ b/internal/host/movesclose_order_test.go

@@ -0,0 +1,135 @@

+package host

+import (

+ "context"

+ "strings"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/provision"

+// Paul's ruling M4 (2026-10-10; spec §7.5, §19 K8): every move close is

+// record first, then the claim's release. Each box close checks that nothing

+// is owed under the op, clears the record's move into LastMove, and only then

+// releases the claim. A crash between the two halves leaves the record closed

+// and the claim still under the move; the box pass is driven by records with

+// a live move, so it never comes back, and the console's recovery pass

+// releases from LastMove (internal/console, TestRecordFirstCloseCrashIsFinishedByThePass).

+//

+// Each case crashes at "after-close-record", the seam between the halves,

+// and asserts the crash state: the record at rest with the outcome, the

+// claim still in the move's closing op with nothing owed, and a further

+// pass of every box writes nothing.

+// closedButHeld asserts the state between the halves of a record-first

+// close: the record closed with outcome, the claim still at kind/phase under

+// the move with nothing owed under it.

+func closedButHeld(t *testing.T, rec provision.Record, c *hostclaim.Claim, outcome, kind, phase string) {

+ t.Helper()

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != p1ID || rec.LastMove.Outcome != outcome {

+ t.Fatalf("the record between the halves: move %+v last %+v, want closed %s", rec.Move, rec.LastMove, outcome)

+ }

+ if c.Op.ID != p1ID || c.Op.Kind != kind || c.Op.Phase != phase {

+ t.Fatalf("the claim between the halves: %+v, want %s/%q under %s (the release is second)", c.Op, kind, phase, p1ID)

+ }

+ for _, o := range c.Obligations {

+ if o.Op == p1ID {

+ t.Fatalf("the record closed while %s %s%s was still owed under the op", o.Kind, o.Region, o.Box)

+ }

+func TestBoxClosesAreRecordFirst(t *testing.T) {

+ t.Run("committed close (movesparks.go)", func(t *testing.T) {

+ h := newCloseHarness(t)

+ h.land(t)

+ h.sourcePass(t)

+ h.clock.Advance(24 * time.Hour)

+ if err := h.regionalAck(t, h.chore(t, "ie", false)); err != nil {

+ t.Fatal(err)

+ }

+ h.landAt = "after-close-record"

+ h.crashLand(t)

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "done", hostclaim.KindLanding, hostclaim.PhaseCommitted)

+ gen := h.claim(t).Gen

+ h.freshTarget()

+ h.land(t)

+ h.sourcePass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

+ t.Run("abort (movesabort.go)", func(t *testing.T) {

+ h := newAbortHarness(t)

+ h.abort(t)

+ h.abAt = "after-abort-record"

+ for i := 0; i < 6 && !h.abDone; i++ {

+ movesPass(context.Background(), h.s, h.w())

+ h.tPass(t)

+ if o := h.laneChore(t); o != nil {

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ }

+ if !h.abDone {

+ t.Fatalf("never reached after-abort-record (%v)", h.abHits)

+ }

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "aborted", hostclaim.KindMoving, hostclaim.PhaseAborting)

+ h.ownStateGone(t)

+ gen := h.claim(t).Gen

+ h.worker1 = nil

+ h.pass(t)

+ h.tPass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

+ t.Run("recovery close (movesrecover.go)", func(t *testing.T) {

+ h := newRecoverHarness(t, false)

+ h.released(t)

+ h.srcPass(t)

+ h.opened(t)

+ h.rcAt = "after-close-record"

+ h.closeOut(t)

+ if !h.rcDone {

+ t.Fatalf("never reached after-close-record (%v)", h.rcHits)

+ }

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "rolled_back", hostclaim.KindRestoring, hostclaim.PhaseOpened)

+ if n := strings.Count(strings.Join(h.sEvents, " "), "rolled_back"); n != 1 {

+ t.Fatalf("the terminal event %d times: %v", n, h.sEvents)

+ }

+ gen := h.claim(t).Gen

+ h.sw = nil

+ h.srcPass(t)

+ h.rollbackPass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

+ t.Run("revert close (movesrevert.go)", func(t *testing.T) {

+ h := newRevertHarness(t)

+ h.fromStopped(t)

+ h.arm(t)

+ h.pass(t)

+ h.reopened(t)

+ h.rvAt = "after-close-record"

+ o := h.laneChore(t)

+ h.tPass(t)

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ h.pass(t)

+ if !h.rvDone {

+ t.Fatalf("never reached after-close-record (%v)", h.rvHits)

+ }

+ closedButHeld(t, h.read(t, p1Host), h.claim(t), "rolled_back", hostclaim.KindReverting, hostclaim.PhaseOpened)

+ gen := h.claim(t).Gen

+ h.freshWorker()

+ h.pass(t)

+ h.tPass(t)

+ if c := h.claim(t); c.Gen != gen {

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d): %+v", gen, c.Gen, c.Op)

+ }

+ })

diff --git a/internal/host/moveslandcommit.go b/internal/host/moveslandcommit.go

index 2a564e71b..721073acf 100644

--- a/internal/host/moveslandcommit.go

+++ b/internal/host/moveslandcommit.go

@@ -97,8 +97,8 @@ func (w *movesWorker) committedTarget(ctx context.Context, s release.Store, rec

return fmt.Errorf("read the host claim: %w", err)

}

if !obs.Absent && hostclaim.RequireReleasedMove(m.ID)(obs.Claim) == nil {

- // The close released the claim and stopped before the record's

- // half (§7.5): only that is left.

+ // A binary that closed claim first released the claim and stopped

+ // before the record's half: only that is left (closeMove).

return w.closeMove(ctx, s, rec, l.fault)

}

if err := l.commitClaim(ctx); err != nil {

diff --git a/internal/host/movesparks.go b/internal/host/movesparks.go

index 0e515bf47..9ec7aa2d6 100644

--- a/internal/host/movesparks.go

+++ b/internal/host/movesparks.go

@@ -31,15 +31,17 @@ import (

// Step 5, the close (closeMove). Any participant that sees opened_at and

// source_parked_at on the record and no obligation of the op in the claim

// closes: the terminal event `done` (moves.LogEvent when the hook is wired,

-// Plan 6; a log line until then), then hostclaim.ReleaseMove to serving under

-// the target (§19 K8: the claim first), then provision.MoveClose (the record

-// last: LastMove{Outcome: "done"}, index D11). A crash between the two

-// leaves a finished move on the record that admits nothing (MoveRequest

-// refuses while Move is set), and the next pass of the target finds the

-// claim recording this move's release and writes only the record's close.

-// The target calls it every committed pass; the source calls it in the pass

-// that stamps source_parked_at, which is the last pass that routes the

-// record to it. The console's recovery pass also tries (Task 7).

+// Plan 6; a log line until then), then provision.MoveClose (the record

+// first: LastMove{Outcome: "done"}, index D11), then hostclaim.ReleaseMove

+// to serving under the target (§7.5, §19 K8 as Paul's ruling M4 amends

+// them: record first, then the claim's release). A crash between the two

+// leaves the record at rest and the claim still landing/committed with

+// nothing owed; no box pass visits a record without a move again, and the

+// console's recovery pass releases from LastMove (Recovery.Moves, Plan 5

+// Task 7). A claim found already released by this move (a binary that

+// closed claim first) gets only the record's close. The target calls it

+// every committed pass; the source calls it in the pass that stamps

+// source_parked_at, which is the last pass that routes the record to it.

// Production (K4). The staging park and the close are production-live only

// after Plan 5 Task 9 (park arming) and claim Plans 10 and 11. The committed

@@ -51,9 +53,9 @@ import (

// then the tests arm everything with fakes.

// closeMove is the close (§7.9 step 5; see the file comment). fault is the

-// caller's test seam, passed "after-close-release" between the claim's

-// release and the record's close. It does nothing until the record carries

-// both stamps and the claim owes nothing under the op.

+// caller's test seam, passed "after-close-record" between the record's close

+// and the claim's release. It does nothing until the record carries both

+// stamps and the claim owes nothing under the op.

func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provision.Record, fault func(string) error) error {

host, id, inc := rec.Host, rec.Move.ID, rec.Move.Incarnation

fresh, err := provision.GetRecord(ctx, s, rec.Account, rec.App, host)

@@ -79,7 +81,11 @@ func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provis

return fmt.Errorf("move %s has no host claim", id)

}

c := obs.Claim

- if hostclaim.RequireReleasedMove(id)(c) != nil {

+ // A claim this move already released was released by a binary that

+ // closed claim first: only the record's half is left.

+ released := hostclaim.RequireReleasedMove(id)(c) == nil

+ target := ""

+ if !released {

if c.Op.ID != id || c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted || len(c.Op.Participants) != 2 {

w.notice(host+"#close", "moves: %s: move %s waits to close: the claim is %s/%q (%s), not this move's commit", host, id, c.Op.Kind, c.Op.Phase, c.Op.ID)

return nil

@@ -87,27 +93,15 @@ func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provis

if c.Holder == nil || c.Holder.Incarnation != inc {

return fmt.Errorf("move %s: the host claim names another incarnation", id)

}

- owed := 0

- for _, o := range c.Obligations {

- if o.Op == id {

- owed++

- }

- if owed > 0 {

+ // Nothing owed under the op, checked here before the record's

+ // close (M4), as the console's grant checks it: a record closed

+ // over an owed chore would leave a release ReleaseMove refuses.

+ if owed := owedUnder(c, id); owed > 0 {

w.notice(host+"#close", "moves: %s: move %s waits for %d park(s) of the move to be acknowledged before it closes", host, id, owed)

return nil

}

+ target = c.Op.Participants[1]

w.logEvent(ctx, *fresh, "done")

- target := c.Op.Participants[1]

- if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseMove(id, target)); err != nil {

- return fmt.Errorf("release the claim: %w", err)

- }

- log.Printf("moves: %s: the claim is serving under %s after move %s", host, target, id)

- if fault != nil {

- if err := fault("after-close-release"); err != nil {

- return err

- }

}

g := provision.MoveGuard{ID: id, Incarnation: inc}

if _, err := provision.SetRecordMove(ctx, s, fresh.Account, fresh.App, host, provision.MoveClose(g, w.now())); err != nil {

@@ -119,5 +113,52 @@ func (w *movesWorker) closeMove(ctx context.Context, s release.Store, rec provis

}

log.Printf("moves: %s: move %s is done; the instance is in %s", host, id, m.To)

nudgeSerial(ctx, s)

+ if released {

+ return nil

+ }

+ if fault != nil {

+ if err := fault("after-close-record"); err != nil {

+ return err

+ }

+ if err := releaseAfterClose(ctx, s, host, id, hostclaim.ReleaseMove(id, target)); err != nil {

+ return err

+ }

+ log.Printf("moves: %s: the claim is serving under %s after move %s", host, target, id)

+ return nil

+// owedUnder counts the obligations of the claim under op id.

+func owedUnder(c *hostclaim.Claim, id string) int {

+ owed := 0

+ for _, o := range c.Obligations {

+ if o.Op == id {

+ owed++

+ }

+ return owed

+// releaseAfterClose is the second half of every box close (ruling M4): the

+// record already carries the outcome in LastMove, so the claim is released

+// from a fresh read with the op's own release transition. A claim this move

+// already released (the console's pass finished it first) is success. Any

+// other failure is returned and left to the console's recovery pass, which

+// releases from LastMove: the box pass never visits a record without a move

+// again.

+func releaseAfterClose(ctx context.Context, s release.Store, host, id string, release func(*hostclaim.Claim) error) error {

+ obs, err := hostclaim.Read(ctx, s, host)

+ if err != nil {

+ return fmt.Errorf("read the host claim to release move %s: %w", id, err)

+ }

+ if obs.Absent {

+ return fmt.Errorf("move %s has no host claim to release", id)

+ }

+ if hostclaim.RequireReleasedMove(id)(obs.Claim) == nil {

+ return nil

+ }

+ if _, err := hostclaim.Update(ctx, s, obs, release); err != nil {

+ return fmt.Errorf("release the claim after the close of move %s (the console's recovery pass finishes it): %w", id, err)

+ }

return nil

}

diff --git a/internal/host/movesparks_test.go b/internal/host/movesparks_test.go

index 53f8e434a..ec4613b75 100644

--- a/internal/host/movesparks_test.go

+++ b/internal/host/movesparks_test.go

@@ -222,11 +222,15 @@ func TestCloseWaitsForBothObligations(t *testing.T) {

}

-// TestCloseIsRecordLast (§7.5, §19 K8): the close releases the claim before

-// it clears the record. A crash between leaves a finished move on the

-// record, which admits nothing (no new move starts over it), and the next

-// pass clears it without writing the claim again.

-func TestCloseIsRecordLast(t *testing.T) {

+// TestCloseIsRecordFirst (§7.5, §19 K8 as Paul's ruling M4 amends them):

+// the close clears the record before it releases the claim. A crash between

+// leaves the record at rest with LastMove done and the claim

+// landing/committed with nothing owed; no new move can start (the claim is

+// not serving), and no box pass writes the claim again: the release is the

+// console's recovery pass (TestBoxClosesAreRecordFirst; the console's

+// TestRecordFirstCloseCrashIsFinishedByThePass). Without a crash the same

+// pass releases.

+func TestCloseIsRecordFirst(t *testing.T) {

h := newCloseHarness(t)

h.land(t)

h.sourcePass(t)

@@ -234,26 +238,36 @@ func TestCloseIsRecordLast(t *testing.T) {

if err := h.regionalAck(t, h.chore(t, "ie", false)); err != nil {

t.Fatal(err)

}

- h.landAt = "after-close-release"

+ h.landAt = "after-close-record"

h.crashLand(t)

rec := h.read(t, p1Host)

- if rec.Move == nil || rec.Move.State != provision.MoveCommitted || rec.LastMove != nil {

- t.Fatalf("the record cleared before the claim released: %+v %+v", rec.Move, rec.LastMove)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "done" {

+ t.Fatalf("the record did not close first: %+v %+v", rec.Move, rec.LastMove)

}

c := h.claim(t)

- if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "uk-box" || hostclaim.RequireReleasedMove(p1ID)(c) != nil {

+ if c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted || c.Op.ID != p1ID {

t.Fatalf("the claim at the crash point: %+v", c.Op)

}

- if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRequest("ie", h.clock.Now())); !errors.Is(err, provision.ErrMoveConflict) {

- t.Fatalf("a new move started over the finished one: %v", err)

+ if err := hostclaim.StartGuard(c, []string{"uk"}, []string{"uk-box"}); err == nil {

+ t.Fatal("a new operation could start over the unreleased claim")

}

gen := c.Gen

h.freshTarget()

h.land(t)

- h.done(t)

if after := h.claim(t).Gen; after != gen {

- t.Fatalf("the record's close wrote the claim again (%d to %d)", gen, after)

+ t.Fatalf("a box pass after the record's close wrote the claim (%d to %d)", gen, after)

+ }

+ // Without the crash, the pass that closes the record releases.

+ h2 := newCloseHarness(t)

+ h2.land(t)

+ h2.sourcePass(t)

+ h2.clock.Advance(24 * time.Hour)

+ if err := h2.regionalAck(t, h2.chore(t, "ie", false)); err != nil {

+ t.Fatal(err)

}

+ h2.land(t)

+ h2.done(t)

}

// Any participant that sees both stamps and nothing owed closes (§7.9 step

@@ -262,8 +276,8 @@ func TestCloseIsRecordLast(t *testing.T) {

func TestTheSourceClosesWhenItFinishesLast(t *testing.T) {

h := newCloseHarness(t)

h.landFault = func(point string) error {

- if point == "after-close-release" {

- t.Fatal("the target released before the source parked")

+ if point == "after-close-record" {

+ t.Fatal("the target closed before the source parked")

}

return nil

}

diff --git a/internal/host/movesrecover.go b/internal/host/movesrecover.go

index db4e59706..a1fd78401 100644

--- a/internal/host/movesrecover.go

+++ b/internal/host/movesrecover.go

@@ -82,10 +82,11 @@ import (

// - opened: the finalization. Once recovered_at, opened_at and

// target_parked_at are stamped and no obligation of the op remains in

// the claim, the terminal event (LogEvent, Plan 6's moves.LogEvent),

-// the claim's release to serving (ReleaseMove) and the record's close

-// (MoveRolledBackClose). Whichever actor stamps last tries; the

-// source's next pass, and Plan 5's console pass, finish what a crash

-// leaves.

+// the record's close (MoveRolledBackClose) and then the claim's

+// release to serving (ReleaseMove): record first, ruling M4. Whichever

+// actor stamps last tries; a crash before the record's close is the

+// source's next pass, and one after it the console's recovery pass

+// (Plan 5 Task 7), which releases from LastMove.

// Every record write goes through a provision transition (the record_seq

// rule); every claim write through a hostclaim transition.

@@ -768,9 +769,12 @@ func (r *recoverRun) open(ctx context.Context) (bool, error) {

// finalize (§7.8 rolled_back): with the source opened and recovered and the

// target parked, and no obligation of the op left in the claim, the

-// terminal event, the claim's release to serving and the record's close. A

-// claim already released by this move (a crash before the close) only

-// closes.

+// terminal event, the record's close, then the claim's release to serving:

+// record first (§7.5 and §19 K8 as Paul's ruling M4 amends them). A crash

+// between the halves leaves the record at rest and the claim

+// restoring/opened with nothing owed; the console's recovery pass releases

+// from LastMove. A claim already released by this move (a binary that

+// closed claim first) only closes.

func (r *recoverRun) finalize(ctx context.Context) (bool, error) {

cfg := r.cfg()

host, id, rb := r.rec.Host, r.g.ID, r.rollback()

@@ -785,31 +789,21 @@ func (r *recoverRun) finalize(ctx context.Context) (bool, error) {

if obs.Absent {

return false, fmt.Errorf("move %s has no host claim", id)

}

- if hostclaim.RequireReleasedMove(id)(obs.Claim) != nil {

+ released := hostclaim.RequireReleasedMove(id)(obs.Claim) == nil

+ if !released {

if err := r.ours(obs.Claim); err != nil {

return false, fmt.Errorf("move %s: %w", id, err)

}

if err := r.executing(obs.Claim, hostclaim.PhaseOpened); err != nil {

return false, err

}

- owed := 0

- for _, o := range obs.Claim.Obligations {

- if o.Op == id {

- owed++

- }

- if owed > 0 {

+ // Nothing owed under the op, checked before the record's close

+ // (M4), as the console's grant checks it.

+ if owed := owedUnder(obs.Claim, id); owed > 0 {

r.w.notice(host+"#rc-final", "moves: %s: rollback %s waits for %d chore(s) of the move to be acknowledged before it closes", host, r.rb, owed)

return false, nil

}

r.w.logEvent(ctx, r.rec, "rolled_back")

- if _, err := hostclaim.Update(ctx, r.s, obs, hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

- return false, fmt.Errorf("release the claim: %w", err)

- }

- log.Printf("moves: %s: the claim is serving again after rollback %s", host, r.rb)

- if err := r.fault("after-release-move"); err != nil {

- return false, err

- }

}

if _, err := provision.SetRecordMove(ctx, r.s, r.rec.Account, r.rec.App, host, provision.MoveRolledBackClose(r.g, r.now())); err != nil {

if errors.Is(err, provision.ErrMoveConflict) {

@@ -820,6 +814,18 @@ func (r *recoverRun) finalize(ctx context.Context) (bool, error) {

}

log.Printf("moves: %s: move %s is rolled back and closed", host, id)

nudgeSerial(ctx, r.s)

+ if !released {

+ if err := r.fault("after-close-record"); err != nil {

+ return true, err

+ }

+ if err := releaseAfterClose(ctx, r.s, host, id, hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

+ return true, err

+ }

+ log.Printf("moves: %s: the claim is serving again after rollback %s", host, r.rb)

+ if err := r.fault("after-release-move"); err != nil {

+ return true, err

+ }

if err := r.fault("after-close"); err != nil {

return true, err

}

diff --git a/internal/host/movesrecover_test.go b/internal/host/movesrecover_test.go

index 928368cb6..f0174bb47 100644

--- a/internal/host/movesrecover_test.go

+++ b/internal/host/movesrecover_test.go

@@ -612,10 +612,11 @@ func TestSourceRecoveryNeverReverifiesAfterProbing(t *testing.T) {

// TestRolledBackFinalizationReleasesTheClaim (§7.8 rolled_back): once the

// source has opened and the target has parked, and no obligation of the op

-// remains, the source writes the terminal event, releases the claim to

-// serving and closes the record with the rolled_back outcome. Until every

-// chore is acknowledged it waits; a crash between the claim's release and

-// the record's close is finished by the next pass.

+// remains, the source writes the terminal event, closes the record with the

+// rolled_back outcome and releases the claim to serving, in that order

+// (ruling M4). Until every chore is acknowledged it waits; a crash between

+// the record's close and the release is the console's to finish

+// (TestBoxClosesAreRecordFirst).

func TestRolledBackFinalizationReleasesTheClaim(t *testing.T) {

ctx := context.Background()

h := newRecoverHarness(t, false)

@@ -655,11 +656,7 @@ func TestRolledBackFinalizationReleasesTheClaim(t *testing.T) {

t.Fatal(err)

}

- h.rcAt = "after-release-move"

h.srcPass(t)

- if !h.rcDone {

- t.Fatalf("never reached the claim's release (%v)", h.rcHits)

- }

c := h.claim(t)

if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

t.Fatalf("the claim was not released: %+v", c.Op)

@@ -667,8 +664,6 @@ func TestRolledBackFinalizationReleasesTheClaim(t *testing.T) {

if err := hostclaim.RequireReleasedMove(p1ID)(c); err != nil {

t.Fatal(err)

}

- h.sourceAt(t, provision.RbSourceOpened)

- h.srcPass(t)

rec := h.read(t, p1Host)

if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != p1ID || rec.LastMove.Outcome != "rolled_back" || rec.Region != "ie" {

t.Fatalf("not closed: region %s move %+v last %+v", rec.Region, rec.Move, rec.LastMove)

diff --git a/internal/host/movesrevert.go b/internal/host/movesrevert.go

index 406b0294e..dfa54e91d 100644

--- a/internal/host/movesrevert.go

+++ b/internal/host/movesrevert.go

@@ -69,13 +69,15 @@ import (

// - opened: the claim waits for the lane's park the revert queued in the

// target region (§7.11), which the target region's box runs

// (laneParkTarget) and the park engine acknowledges after its horizon.

-// Then ReleaseMove to serving under this box, and MoveRevertClose:

-// LastMove rolled_back, offline from cut_at to the reopen. The source's

-// journal of the op is parked in the same pass; a crash before that

-// leaves it to the local journal sweep (sweepRevertJournal).

+// Then MoveRevertClose (LastMove rolled_back, offline from cut_at to

+// the reopen), and then ReleaseMove to serving under this box: record

+// first, ruling M4; a crash between is finished by the console's

+// recovery pass from LastMove. The source's journal of the op is parked

+// in the same pass; a crash before that leaves it to the local journal

+// sweep (sweepRevertJournal).

-// A claim this move already released (RequireReleasedMove: a crash before the

-// record's close) only closes.

+// A claim this move already released (RequireReleasedMove: a binary that

+// closed claim first crashed before the record's close) only closes.

// The target region's half (laneParkTarget). A revert never flipped, so its

// target box has neither the row nor a journal of the move; the participant

@@ -310,8 +312,9 @@ func (r *revertRun) step(ctx context.Context) (bool, error) {

}

c := obs.Claim

if hostclaim.RequireReleasedMove(id)(c) == nil {

- // A crash between the claim's release and the record's close.

- return r.close(ctx)

+ // A binary that closed claim first crashed between the claim's

+ // release and the record's close: only the record's half is left.

+ return r.close(ctx, false)

}

if c.Op.ID != id {

return false, fmt.Errorf("move %s: the claim names op %q", id, c.Op.ID)

@@ -688,38 +691,30 @@ func (r *revertRun) probed(ctx context.Context, obs hostclaim.Observed) (bool, e

}

// opened (§7.4 opened, §7.11): once no chore of the op is left in the claim

-// (the lane's park acknowledged), the claim's release to serving under this

-// box, then the record's close.

+// (the lane's park acknowledged), the record's close, then the claim's

+// release to serving under this box (record first, ruling M4).

func (r *revertRun) opened(ctx context.Context, obs hostclaim.Observed) (bool, error) {

- cfg := r.cfg()

host, id := r.rec.Host, r.g.ID

if r.rec.Move.ReopenedAt == nil {

return false, fmt.Errorf("move %s: the claim is opened and the record has no reopen", id)

}

- owed := 0

- for _, o := range obs.Claim.Obligations {

- if o.Op == id {

- owed++

- }

- if owed > 0 {

+ // Nothing owed under the op, checked before the record's close (M4), as

+ // the console's grant checks it.

+ if owed := owedUnder(obs.Claim, id); owed > 0 {

r.w.notice(host+"#rv-opened", "moves: %s: the revert of %s waits for %d chore(s) of the move, its lane's park, to be acknowledged", host, id, owed)

return false, nil

}

r.w.logEvent(ctx, r.rec, "rolled_back")

- if err := r.update(ctx, obs, "release the claim", hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

- return false, err

- }

- log.Printf("moves: %s: the claim is serving again after the revert of %s", host, id)

- if err := r.fault("after-release-move"); err != nil {

- return false, err

- }

- return r.close(ctx)

+ return r.close(ctx, true)

}

-// close is the record's close (MoveRevertClose), then the source's journal of

-// the op parked.

-func (r *revertRun) close(ctx context.Context) (bool, error) {

+// close is the record's close (MoveRevertClose), then, when releaseClaim is set,

+// the claim's release to serving under this box, then the source's journal

+// of the op parked. A crash between the record's close and the release

+// leaves the claim reverting/opened with nothing owed; the console's

+// recovery pass releases it from LastMove. releaseClaim is false only for a

+// claim a claim-first binary already released.

+func (r *revertRun) close(ctx context.Context, releaseClaim bool) (bool, error) {

cfg := r.cfg()

host, id := r.rec.Host, r.g.ID

ok, err := r.record(ctx, "the revert's close", provision.MoveRevertClose(r.g, r.now()))

@@ -728,6 +723,18 @@ func (r *revertRun) close(ctx context.Context) (bool, error) {

}

log.Printf("moves: %s: move %s is reverted and closed", host, id)

nudgeSerial(ctx, r.s)

+ if releaseClaim {

+ if err := r.fault("after-close-record"); err != nil {

+ return true, err

+ }

+ if err := releaseAfterClose(ctx, r.s, host, id, hostclaim.ReleaseMove(id, cfg.Box)); err != nil {

+ return true, err

+ }

+ log.Printf("moves: %s: the claim is serving again after the revert of %s", host, id)

+ if err := r.fault("after-release-move"); err != nil {

+ return true, err

+ }

if err := r.fault("after-close"); err != nil {

return true, err

}

diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go

index e5620231e..7b38a2d85 100644

--- a/internal/provision/plan9_b8_census_test.go

+++ b/internal/provision/plan9_b8_census_test.go

@@ -531,10 +531,10 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/host/moveslandcommit.go", "landingRun.probed", "record-writer", "key:SetRecordMove", "6f961790d8fe169af68e2f70790c400d37c2388a4f62c26921f22b8b47c859de"},

// Stamps opened_at through SetRecordMove (MoveOpened) inside the hold lock, after the proved release.

{"internal/host/moveslandcommit.go", "landingRun.open", "record-writer", "key:SetRecordMove", "65e6908056c3055d5d0df2c27bb88943e31c08691a2173a090a9edb94c8c0b49"},

- // Closes a committed move through SetRecordMove (MoveClose, LastMove done) after the claim's ReleaseMove; never writes the claim after the record.

- {"internal/host/movesparks.go", "movesWorker.closeMove", "record-writer", "key:SetRecordMove", "849d114cf80a45004e9d01c66fcf11516381b046fe0f7c40294a628a3b922b90"},

- // Closes the rollback record through SetRecordMove; claim release and local journal parking use separate storage.

- {"internal/host/movesrecover.go", "recoverRun.finalize", "record-writer", "key:SetRecordMove", "fff2e28c78db68bccb16da6ab70ca877b51243b0aa56b9cb2c513c6f5660f7f7"},

+ // Closes a committed move through SetRecordMove (MoveClose, LastMove done) once nothing is owed under the op, then releases the claim (ReleaseMove via releaseAfterClose): record first, ruling M4. A claim already released by this move gets only the record's close.

+ {"internal/host/movesparks.go", "movesWorker.closeMove", "record-writer", "key:SetRecordMove", "efb6fa290d9eb2ee6b64a67f853d690ff1b259bd565b6191327822d99a9c0771"},

+ // Closes the rollback record through SetRecordMove once nothing is owed under the op, then releases the claim (record first, ruling M4); local journal parking uses separate storage.

+ {"internal/host/movesrecover.go", "recoverRun.finalize", "record-writer", "key:SetRecordMove", "2986525fbe297ed1f5cccc58957c10c63156301df6700ab2484b2d7837664de5"},

// Advances the source recovery phase through SetRecordMove and its sequenced writer.

{"internal/host/movesrecover.go", "recoverRun.phase", "record-writer", "key:SetRecordMove", "d67132b6e797a88a9677d962c5c7aa30fc7f85fdbdff06ce8318562397a9ee78"},

// Records probe inputs through SetRecordMove; phase and stuck helpers use the same writer.

@@ -663,7 +663,8 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/host/fleetbind.go", "WriteFleetBinding", "nonrecord", "sel:Put", "44ab984c2c9b30abcc938132118f61a6922ccc0cd40ed2b6cb61c7654aa0a1c3"},

{"internal/host/logship.go", "logshipFlush", "nonrecord", "sel:Put", "a2fa594c1206028af170a2d0cbedef55ba00a2f1c698a6853e1214cc3b172dd9"},

{"internal/host/metrics_loop.go", "putAlertState", "nonrecord", "sel:Put", "d5b6e8b34dc5a731d6bbafeac0b7d0c5c20706d6f413dff38088b3321b35a26c"},

- {"internal/host/movesabort.go", "movesWorker.abortSource", "record-writer", "key:SetRecordMove", "70f2cd7ec7709045043081c078f1abe63ef822325eef7016ab9f595a16cabad5"},

+ // Clears an aborting record through SetRecordMove (MoveGuardedAbortComplete) once nothing is owed under the op, then ReleaseAbort: record first, ruling M4.

+ {"internal/host/movesabort.go", "movesWorker.abortSource", "record-writer", "key:SetRecordMove", "c846c24a039e490f430bd41bb9672d00220a1d50fe5b2721b77fdeeafe98029c"},

{"internal/host/movesarchive.go", "cutRun.publishCutArchive", "nonrecord", "sel:CopyVersioned,sel:PutIfAbsent", "6c7fcfd2d4b2a0319ac15ecbfe003ed0292fe160457fd5704be508c2ab302a1b"},

{"internal/host/movescut.go", "cutRun.cutTransition", "record-writer", "key:SetRecordMove", "d0d67399aa784036de43b3c143e88c3d85f8b3d9e6345f84908d3b58ff56191e"},

{"internal/host/movescut.go", "publishStoppedReport", "nonrecord", "sel:Put", "c710682f31d99127ad76524add246fda9f640c63934940e45c79ea25ba36eb3c"},

diff --git a/internal/sim/hostlife/console_test.go b/internal/sim/hostlife/console_test.go

index c85851365..e0cf5fea8 100644

--- a/internal/sim/hostlife/console_test.go

+++ b/internal/sim/hostlife/console_test.go

@@ -56,13 +56,24 @@ func (w *world) execBox(c *claim) *box {

// fixRecord is the recovery owner for the record halves of a move (claim

// C2): the request's `move` field after the claim's intent; the claim half

-// of a record-first transition (followRecord); and, after a release, the

-// CAS that clears `move` and names the executor's region (§7.9 step 5).

+// of a record-first transition (followRecord); the claim's release after a

+// move's record-first close, from last_move (rev 7.23 §7.5, ruling M4: the

+// closing box may stop between the halves and never comes back); and, after

+// any other release (a push, a relocation, or rev 7.22's claim-first close

+// under its mutation), the CAS that clears `move` and names the executor's

+// region.

func (w *world) fixRecord() {

c := w.claim

if c == nil || c.Holder == nil || w.rec == nil || w.rec.Inc != c.Holder.Inc {

return

}

+ if moveClose(c.Op) && w.rec.LastMove == c.Op.ID && w.rec.MoveID == "" {

+ if c.oblFor(c.Op.ID) == 0 {

+ w.stats["recovery: release from last_move"]++

+ w.release(c, closeExec(c.Op))

+ }

+ return

+ }

switch c.Op.Kind {

case "serving":

if b := w.execBox(c); b != nil && (w.rec.MoveID != "" || w.rec.Region != b.Region) {

diff --git a/internal/sim/hostlife/moves_test.go b/internal/sim/hostlife/moves_test.go

index 6e5ae501f..b0af446ce 100644

--- a/internal/sim/hostlife/moves_test.go

+++ b/internal/sim/hostlife/moves_test.go

@@ -544,7 +544,7 @@ func (w *world) moveSource(b *box, c *claim) {

}

case "aborting":

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Src)

+ w.closeMove(c, op.Src)

}

case "stopping":

if b.Row != nil {

@@ -938,7 +938,7 @@ func (w *world) revertSource(b *box, c *claim) {

}

case "opened":

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Src)

+ w.closeMove(c, op.Src)

}

@@ -1258,7 +1258,7 @@ func (w *world) landTarget(b *box, c *claim) {

return

}

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Tgt)

+ w.closeMove(c, op.Tgt)

}

@@ -1334,6 +1334,74 @@ func (w *world) installLocal(b *box, c *claim, gen string, seal ref, _ string, n

w.cas(c, func(n *claim) bool { next(n); return true })

}

+// moveClose reports whether op is a move's close: a committed landing, an

+// opened rollback or revert, or an abort. Pushes and relocations release on

+// their own terms.

+func moveClose(op opState) bool {

+ switch op.Kind {

+ case "landing", "restoring", "reverting":

+ return true

+ case "moving":

+ return op.Phase == "aborting"

+ }

+ return false

+// closeExec is the box a move's close releases to: the target after a

+// landing, the source otherwise (§19 K8).

+func closeExec(op opState) int {

+ if op.Kind == "landing" {

+ return op.Tgt

+ }

+ return op.Src

+// closeMove is a move's close by the box that runs it, record first (moves

+// rev 7.23 §7.5 and §19 K8, Paul's ruling M4). With nothing owed under the

+// op, the record's CAS clears move into last_move and names the executor's

+// region; only then does the claim release. The box may stop between the

+// halves (every third step here, a deterministic schedule that draws

+// nothing from the seed's stream). It never comes back to a record without a

+// move, so a closed record whose claim still holds the op is the recovery

+// pass's to release, from last_move (fixRecord).

+func (w *world) closeMove(c *claim, execBox int) {

+ op := c.Op

+ if c.oblFor(op.ID) != 0 {

+ return

+ }

+ if !on("record-first-close") {

+ // rev 7.22: the claim first; the recovery pass wrote the record

+ // half after the release

+ w.release(c, execBox)

+ return

+ }

+ if !w.closeRecord(c, execBox) {

+ return // closed already (last_move): not the box's any more

+ }

+ if w.now%3 == 0 {

+ w.stats["close: stopped between the record and the release"]++

+ return

+ }

+ w.release(c, execBox)

+// closeRecord is the close's record half: the CAS that clears move into

+// last_move and names the executor's region, conditional on the record still

+// carrying this move for the claim's incarnation.

+func (w *world) closeRecord(c *claim, execBox int) bool {

+ r, op := w.rec, c.Op

+ if r == nil || c.Holder == nil || r.Inc != c.Holder.Inc || r.MoveID != op.ID {

+ return false

+ }

+ if b := w.boxByID(execBox); b != nil {

+ r.Region = b.Region

+ }

+ r.MoveID, r.MoveState, r.RollbackID, r.LastMove = "", "", "", op.ID

+ r.Ver = w.newVer()

+ w.logf("close %s: record closed (last_move)", op.ID)

+ return true

func (w *world) release(c *claim, execBox int) {

op := c.Op

if w.cas(c, func(n *claim) bool {

@@ -1347,8 +1415,15 @@ func (w *world) release(c *claim, execBox int) {

return true

}) {

w.logf("released %s to serving on box %d", op.ID, execBox)

+ if moveClose(op) && (w.rec == nil || w.rec.LastMove != op.ID || w.rec.MoveID == op.ID) {

+ // K8 as rev 7.23 amends it: never a serving claim over a live

+ // move. Judged only for a release CAS that succeeded: one the

+ // op-id guard refused released nothing (fix round 2)

+ w.violate("K8: %s released the claim before its record closed (record first, rev 7.23)", op.ID)

+ }

w.opsDone++

- // the record half (CAS-clear move) follows on a later step (§7.9)

+ // a move's record closed before this (closeMove, rev 7.23); a

+ // push's or a relocation's record half follows on a later step

w.checkDurable(w.claim, "at the release of "+op.ID)

if r := w.residueAtEnd(); len(r) > 0 {

w.violate("INV-10: %s ended with residue %v", op.ID, r)

@@ -1639,7 +1714,7 @@ func (w *world) recoverSource(b *box, c *claim) {

}

case "opened":

if c.oblFor(op.ID) == 0 {

- w.release(c, op.Src)

+ w.closeMove(c, op.Src)

}

diff --git a/internal/sim/hostlife/rules_test.go b/internal/sim/hostlife/rules_test.go

index 803320fad..1ba023d6c 100644

--- a/internal/sim/hostlife/rules_test.go

+++ b/internal/sim/hostlife/rules_test.go

@@ -107,6 +107,7 @@ var allRules = []string{

"sweep-namespace", // claim rev 6 §6.2a (R3-4): the sweep discovers locations from the namespace; the mutation reads the capped retired_incarnations

"live-set-region", // claim rev 6.5 §6.2a (R4-3): the live set is {authoritative region, location, generation}; the mutation exempts matching keys in every region

// moves spec

+ "record-first-close", // rev 7.23 §7.5, §19 K8 (ruling M4): a move's close CASes the record (move into last_move) before the claim's release, and the recovery pass releases from last_move; the mutation is rev 7.22's claim-first close, whose record half the recovery pass wrote after the release

"seal-by-version", // J-N2: landing copies sealed versions

"staging-quiet", // J-N2: staging retired after its writers' horizon

"commit-before-open", // INV-5

diff --git a/internal/sim/hostlife/sim_test.go b/internal/sim/hostlife/sim_test.go

index 1cac18606..59ac976f6 100644

--- a/internal/sim/hostlife/sim_test.go

+++ b/internal/sim/hostlife/sim_test.go

@@ -508,6 +508,19 @@ func (w *world) canStart(e *box, c *claim) string {

return ""

}

+// finalFreed is the end state of a freed host: no record, and no box holds

+// local state for it.

+func (w *world) finalFreed() {

+ if w.rec != nil {

+ w.violate("claim/record: claim free but record inc %d exists", w.rec.Inc)

+ }

+ for _, b := range w.liveBoxes() {

+ if b.DB != nil || b.Files != nil {

+ w.violate("INV-10: box %d still holds local state for a freed host", b.ID)

+ }

func (w *world) final() {

if !w.quiescent() {

if len(w.pending) == 0 && len(w.console) == 0 && w.atRest() {

@@ -519,14 +532,7 @@ func (w *world) final() {

}

c := w.claim

if c == nil || c.Holder == nil {

- if w.rec != nil {

- w.violate("claim/record: claim free but record inc %d exists", w.rec.Inc)

- }

- for _, b := range w.liveBoxes() {

- if b.DB != nil || b.Files != nil {

- w.violate("INV-10: box %d still holds local state for a freed host", b.ID)

- }

+ w.finalFreed()

return

}

e := w.execBox(c)

@@ -556,8 +562,18 @@ func (w *world) final() {

}

c = w.claim

+ if c == nil || c.Holder == nil {

+ // the final drive ran the console too: a purge may have freed

+ // the host (seed 2547 / 3,000 under admission-registration)

+ w.finalFreed()

+ return

+ }

e = w.execBox(c)

}

+ if e == nil || w.rec == nil {

+ w.violate("claim/record: the claim's holder has no record or no executing box at the end: %s", w.state())

+ return

+ }

if w.rec.Region != e.Region {

w.violate("claim/record: record region %d, claim serves from box %d in region %d", w.rec.Region, e.ID, e.Region)

}

diff --git a/internal/sim/hostlife/witness_test.go b/internal/sim/hostlife/witness_test.go

index 9600d7fd0..73515c8fd 100644

--- a/internal/sim/hostlife/witness_test.go

+++ b/internal/sim/hostlife/witness_test.go

@@ -66,7 +66,7 @@ var witnesses = []witness{

{"staging-quiet", "staging-quiet", "ended with residue",

"a staging round completes after the landing's staging park acknowledged; retired at the seal rather than after the writers' horizon, the round is residue at the release (INV-10)", wStagingQuiet},

{"claim-cas-opid", "claim-cas-opid,revalidate-before-effect", "store of app",

- "redundant alone: a stalled source goroutine with the rollback's last read resumes after a purge claimed the host; revalidation stops it, and so does the op-id condition on its release CAS; with both gone the release overwrites the purge and the store is torn down under a serving host", wClaimCasOpID},

+ "redundant alone: the source closes the rollback's record and stalls before its release (rev 7.23, record first); the recovery pass releases from last_move, a purge claims the host, and the stalled goroutine resumes into its release with the rollback's last read; revalidation stops it, and so does the op-id condition on its release CAS; with both gone the release overwrites the purge and the store is torn down under a serving host", wClaimCasOpID},

{"dirty-return", "dirty-return", "INV-2",

"the target writes during its probe and replicates; the rollback is dirty and the return lane carries the write to the source; treated as clean with the source's replica covering the stop, the source's admission is covered clean-return, declares nothing, and the serving database lacks the target's write (INV-2 local at the final recovery, rev 7.12; rev 7.11 saw INV-15 at the declaration, rev 7.10 INV-2 on the serving database)", wDirtyReturn},

{"commit-before-open", "commit-before-open", "INV-5",

@@ -905,28 +905,47 @@ func wStagingQuiet(t *testing.T, w *world) {

w.finish(t)

}

-// wClaimCasOpID (claim C2): a clean rollback; a second goroutine of the

-// source read the claim just before the release and stalls; the host is

-// released, a purge claims it, and the goroutine resumes.

+// wClaimCasOpID (claim C2): a clean rollback; the source's close writes the

+// record (record first, rev 7.23) and its goroutine stalls before the

+// release, holding the claim it read; the recovery pass releases from

+// last_move, a purge claims the host, and the goroutine resumes into its

+// release.

func wClaimCasOpID(t *testing.T, w *world) {

src, tgt := w.boxes[0], w.boxes[1]

w.serve(src, 1, app0)

w.landProbed(t, src, tgt)

w.rollback(w.claim, "cut deadline")

- w.reach(t, 1500, "source about to release", func() bool {

+ w.reach(t, 1500, "source about to close", func() bool {

c := w.claim

return c.Op.Kind == "restoring" && c.Op.SrcPhase == "opened" && c.oblFor(c.Op.ID) == 0

}, w.act(src, tgt), w.recoveryPass)

old := w.claim.clone() // the stalled goroutine's read

- w.reach(t, 50, "released", func() bool { return w.claim.Op.Kind == "serving" }, w.act(src, tgt), w.recoveryPass)

+ if !w.closeRecord(old, src.ID) {

+ t.Fatalf("witness: the record did not close: %s", w.state())

+ }

+ w.reach(t, 50, "released from last_move", func() bool { return w.claim.Op.Kind == "serving" }, w.recoveryPass)

pg := &consolePlan{Kind: "purge", App: app0}

w.reach(t, 20, "purge claimed the host", func() bool { return w.claim.Op.Kind == "purging" }, func() { w.runPlan(pg) })

- w.resumeStale(src, old) // the goroutine wakes with the rollback's last read

+ w.resumeStaleRelease(src, old) // the goroutine wakes into its release

w.tick()

w.tick(func() { w.runPlan(pg) }) // the purge tears the store down

w.tick(func() { w.runPlan(pg) })

}

+// resumeStaleRelease is a close's goroutine that stalled between the

+// record's close and the claim's release, resuming with the claim it read:

+// it revalidates (claim C3), then writes the release half.

+func (w *world) resumeStaleRelease(b *box, old *claim) {

+ c, ok := w.readClaim(b)

+ if !ok {

+ return

+ }

+ if on("revalidate-before-effect") && !sameStep(old, c) {

+ return

+ }

+ w.release(old, closeExec(old.Op))

// wDirtyReturn (INV-2; also commit-before-open's witness): the target

// writes during its probe and is rolled back.

func wDirtyReturn(t *testing.T, w *world) {

@@ -3552,3 +3571,42 @@ func TestProbeWindow(t *testing.T) {

t.Logf("k=%d: accepted=%v tries=%d probe start %d ready %d (failed before the start %d, at the deadline unready %d)", k, accepted, tries, p.startedAt, p.readyAt, beforeStart, atEnv)

}

+// TestK8JudgesOnlyASucceededRelease (Task 7 fix round 2): the K8 oracle

+// judges a release CAS that succeeded, never one the op-id guard refused.

+// The source closes a clean rollback's record and its goroutine stalls

+// before the release; the recovery pass releases from last_move; a purge

+// runs to the end and removes the record; the stalled release resumes with

+// only revalidate-before-effect dropped. Its CAS is conditional on the

+// rollback's op, which the claim no longer holds: refused, the claim is

+// untouched, and nothing is violated.

+func TestK8JudgesOnlyASucceededRelease(t *testing.T) {

+ saved := mutated

+ mutated = "revalidate-before-effect"

+ defer func() { mutated = saved }()

+ w := newWitness()

+ src, tgt := w.boxes[0], w.boxes[1]

+ w.serve(src, 1, app0)

+ w.landProbed(t, src, tgt)

+ w.rollback(w.claim, "cut deadline")

+ w.reach(t, 1500, "source about to close", func() bool {

+ c := w.claim

+ return c.Op.Kind == "restoring" && c.Op.SrcPhase == "opened" && c.oblFor(c.Op.ID) == 0

+ }, w.act(src, tgt), w.recoveryPass)

+ old := w.claim.clone() // the stalled goroutine's read

+ if !w.closeRecord(old, src.ID) {

+ t.Fatalf("the record did not close: %s", w.state())

+ }

+ w.reach(t, 50, "released from last_move", func() bool { return w.claim.Op.Kind == "serving" }, w.recoveryPass)

+ pg := &consolePlan{Kind: "purge", App: app0}

+ w.reach(t, 2000, "the purge removed the record", func() bool { return w.rec == nil }, func() { w.runPlan(pg) }, w.actAll(), w.recoveryPass)

+ before := w.claim.clone()

+ w.resumeStaleRelease(src, old)

+ if len(w.violations) > 0 {

+ dump(t, w)

+ t.Fatalf("a refused stale release was judged: %v", w.violations)

+ }

+ if w.claim.Ver != before.Ver || w.claim.Op.Kind == "serving" && w.claim.Op.Exec == src.ID && w.claim.Holder != nil {

+ t.Fatalf("the stale release was not refused: before %s after %s", before.Op.Kind, w.state())

+ }

diff --git a/internal/sim/hostlife/world_test.go b/internal/sim/hostlife/world_test.go

index d0a6813e1..16ba94654 100644

--- a/internal/sim/hostlife/world_test.go

+++ b/internal/sim/hostlife/world_test.go

@@ -937,6 +937,10 @@ type record struct {

MoveState string

RollbackID string

Reason string

+ // LastMove names the move whose close cleared MoveID: the close is record

+ // first (moves rev 7.23 §7.5, Paul's ruling M4), so a record can say a

+ // move is over while the claim still holds its op, until the release.

+ LastMove string

}

// ---------- the world ----------