carlos / platform Public

console: the move pass under claim Plan 10's Recovery.Moves hook (Plan 5 Task 7)

The moves body of the recovery pass's Recovery.Moves hook, bound by
ConfigureClaimRuntime only with the moves switch on (and now that every
box close is record first, P3-6). movesRecovery rereads the claim and the
record, gives up if either moved on since the grant's observation, and
decides at most one action (decideMoveRecovery):

- an orphan moving/"" start older than ten minutes is settled without
  its handle (ResolveStart, K3), then released not-applied; any other
  orphan pages once and is left;
- a source terminated before the flip converts in two passes (Convert
  with its termination fence, K13);
- a move preparing past moves.AbortDue aborts through ConsoleMove (K18);
- the claim half the record decided: flip, commit, rollback (including
  RollBackFlipped from moving/sealed, K22);
- the landing deadline (the earlier of cut and landing, J11);
- a lost, released target gets target_parked_at through
  StampLostMoveTargetParked over the conditional record store (a closure
  ConfigureClaimRuntime binds; the data boundary refuses it);
- a close due for moveCloseGrace that no box wrote is closed and paged;
- the release from LastMove, the second half of every record-first close.

Pages go through alerting.Page (best effort), rate-limited durably in
alert_seen: the release deadline once an hour (incident), preparing over
an hour (warning), an orphan past the seal (incident), an overdue close
(warning). cmd/console arms the sinks from the agent's variables
(alerting.FromEnv) before the claim runtime, only with both switches on.

Copy rows MV-N325 to MV-N332 (pending; MV-N319 to N324 are Plan 6
Task 2's). The console authority census names the grant's writes, the
abort and the stamp.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 4952efc39b4b5961aa3d3da1d9fe47d87a1e23de parent 4416b34

13 files changed, +1355 −3

approved_moves_copy_test.go +6 −0
cmd/console/main.go +16 −0
cmd/console/movesswitch_test.go +57 −0
docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +13 −0
internal/alerting/alerting.go +32 −0
internal/alerting/alerting_test.go +33 −0
internal/console/claim_authority_census_test.go +3 −0
internal/console/claim_invariant_moves_test.go +19 −0
internal/console/claim_runtime.go +15 −0
internal/console/movesrecovery.go +470 −0
internal/console/movesrecovery_test.go +671 −0
internal/console/server.go +12 −3
internal/console/testdata/claim-authority-sites.txt +8 −0

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index ae295e66b..6616cbad8 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -75,6 +75,7 @@ var movesCopyIDs = []string{

"MV-N307", "MV-N308", "MV-N309", "MV-N310", "MV-N311", "MV-N312",

"MV-N313", "MV-N314",

"MV-N315", "MV-N316", "MV-N317", "MV-N318",

+ "MV-N325", "MV-N326", "MV-N327", "MV-N328", "MV-N329", "MV-N330", "MV-N331", "MV-N332",

}

// copyRow is one row of the record. The File cell is the file the sentence is

@@ -543,6 +544,9 @@ func TestMovesTextAppliedPlaceholders(t *testing.T) {

// copy" task appends the files it touched; TestMovesScanFilesCoverTheDiff

// fails when the list falls behind.

var movesScanFiles = []string{

+ "internal/console/movesrecovery.go",

+ "internal/console/claim_runtime.go",

+ "internal/alerting/alerting.go",

"internal/hostops/move_recovery_grant.go",

"internal/hostops/move_loss_owner.go",

"internal/host/movesrecoveryprobe.go",

@@ -746,6 +750,8 @@ var movesReturnedSentences = map[string]map[string]bool{

"internal/host/movesreport.go": {"t1Below": true, "t3Differs": true, "t3Shadowed": true, "t4Failed": true,

"t6NoRoom": true, "t8Count": true, "t12NotQuiet": true, "replicaReason": true},

"internal/host/movesrollback.go": {"stuckPage": true},

+ // The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N332).

+ "internal/console/movesrecovery.go": {"moveWaitingPage": true, "movePreparingPageText": true, "moveOrphanPage": true, "moveCloseOverduePage": true},

}

// movesLiteralFuncs are functions whose every string literal is a sentence

diff --git a/cmd/console/main.go b/cmd/console/main.go

index ab1ef043d..f4b243819 100644

--- a/cmd/console/main.go

+++ b/cmd/console/main.go

@@ -88,6 +88,7 @@ import (

"unicode/utf8"

"github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/dnsrec"

@@ -419,6 +420,9 @@ func serve() {

if os.Getenv("CARLOS_MAIL_LEGACY_SWEEP") == "1" {

go sweepLegacyMail(cfg.HostClaim, rel, st)

}

+ // The move pass's pages (moves Plan 5 Task 7): the console's alert sinks,

+ // armed before the claim runtime binds the pass, so no pass runs unarmed.

+ armMovesPaging(context.Background(), cfg, os.Getenv)

if err := srv.ConfigureClaimRuntime(context.Background()); err != nil {

log.Fatalf("console: claim recovery: %v", err)

}

@@ -1345,6 +1349,18 @@ func applyMovesSwitch(cfg *console.Config, getenv func(string) string) {

cfg.MovesEnabled = moves.Enabled(getenv)

}

+// armMovesPaging arms alerting's sinks from CARLOS_ALERT_WEBHOOK and

+// CARLOS_ALERT_SNS_TOPIC_ARN, the agent's variables, for the console's move

+// pass (moves Plan 5 Task 7, delta M14). Only a host-claim console with the

+// moves switch on runs that pass, so only it arms; it reports whether it did.

+func armMovesPaging(ctx context.Context, cfg console.Config, getenv func(string) string) bool {

+ if !cfg.HostClaim || !cfg.MovesEnabled {

+ return false

+ }

+ alerting.Arm(alerting.FromEnv(ctx, getenv, nil))

+ return true

func hostClaimEnabled(getenv func(string) string) bool {

return strings.TrimSpace(getenv("CARLOS_HOST_CLAIM")) != ""

}

diff --git a/cmd/console/movesswitch_test.go b/cmd/console/movesswitch_test.go

index 5fc3f903d..6173a520a 100644

--- a/cmd/console/movesswitch_test.go

+++ b/cmd/console/movesswitch_test.go

@@ -1,11 +1,13 @@

package main

import (

+ "context"

"go/ast"

"go/parser"

"go/token"

"testing"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/console"

"github.com/carlosframework/platform/internal/moves"

)

@@ -98,3 +100,58 @@ func isIdent(e ast.Expr, name string) bool {

id, ok := e.(*ast.Ident)

return ok && id.Name == name

}

+// The console pages from its move pass (moves Plan 5 Task 7, delta M14):

+// armMovesPaging arms alerting's sinks from the agent's variables on a

+// host-claim console with the moves switch on, and nowhere else; serve()

+// calls it with os.Getenv before ConfigureClaimRuntime, so the recovery

+// pass never runs unarmed.

+func TestArmMovesPagingOnlyWithBothSwitches(t *testing.T) {

+ env := func(k string) string {

+ if k == "CARLOS_ALERT_WEBHOOK" {

+ return "http://example.test/hook"

+ }

+ return ""

+ }

+ for _, tc := range []struct{ claim, moves, want bool }{{false, false, false}, {true, false, false}, {false, true, false}, {true, true, true}} {

+ cfg := console.Config{HostClaim: tc.claim, MovesEnabled: tc.moves}

+ if got := armMovesPaging(context.Background(), cfg, env); got != tc.want {

+ t.Errorf("claim %v moves %v: armed %v, want %v", tc.claim, tc.moves, got, tc.want)

+ }

+ t.Cleanup(func() { alerting.Arm(nil) })

+ f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0)

+ if err != nil {

+ t.Fatal(err)

+ }

+ var armed, configured token.Pos

+ for _, d := range f.Decls {

+ fn, ok := d.(*ast.FuncDecl)

+ if !ok || fn.Name.Name != "serve" {

+ continue

+ }

+ ast.Inspect(fn.Body, func(n ast.Node) bool {

+ call, ok := n.(*ast.CallExpr)

+ if !ok {

+ return true

+ }

+ switch fun := call.Fun.(type) {

+ case *ast.Ident:

+ if fun.Name == "armMovesPaging" && len(call.Args) == 3 {

+ if sel, ok := call.Args[2].(*ast.SelectorExpr); ok && isIdent(sel.X, "os") && sel.Sel.Name == "Getenv" {

+ armed = call.Pos()

+ }

+ case *ast.SelectorExpr:

+ if fun.Sel.Name == "ConfigureClaimRuntime" {

+ configured = call.Pos()

+ }

+ return true

+ })

+ }

+ if armed == 0 || configured == 0 || armed > configured {

+ t.Fatalf("serve() must call armMovesPaging(ctx, cfg, os.Getenv) before ConfigureClaimRuntime (armed %v, configured %v)", armed, configured)

+ }

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index 94f979f72..03f5e87d3 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -271,6 +271,19 @@ Columns: id, file, text, status.

| MV-N314 | internal/host/movesrollback.go | The move <id> of <host> stopped for an operator on <box>, its <role>: <reason>. Run carlos move resolve --host <host> --retry once the cause is fixed. | pending |

+Task 7's pages, the console's move pass (`internal/console/movesrecovery.go`, §13). MV-N319 to MV-N324 are Plan 6 Task 2's.

+|---|---|---|---|

+| MV-N326 | internal/console/movesrecovery.go | The move <id> of <host> is rolling back, and <to> has not released it by its deadline. <host> stays offline until a box in <to> confirms it has stopped, or that box is declared lost. | pending |

+| MV-N330 | internal/console/movesrecovery.go | The host claim of <host> is held by the move <id>, and the instance record carries no move. Nothing releases it on its own. An operator must check the record. | pending |

## Floor gate fix (#267 review)

(root's floor gate for the system units, `carlos system floor-allows-prev`; its stderr reaches the unit's journal)

diff --git a/internal/alerting/alerting.go b/internal/alerting/alerting.go

index 8956e2fec..22ceb2df0 100644

--- a/internal/alerting/alerting.go

+++ b/internal/alerting/alerting.go

@@ -187,6 +187,38 @@ func Page(ctx context.Context, level, subject, body string) error {

return s.Page(ctx, level, subject, body)

}

+// FromEnv builds a process's sinks from the agent's two variables,

+// CARLOS_ALERT_WEBHOOK (posted through PostJSON) and

+// CARLOS_ALERT_SNS_TOPIC_ARN (published from the topic's own region through

+// newClient, NewSNSClient when nil). The console arms these for its move

+// pass (moves Plan 5 Task 7). A topic that does not parse, or whose client

+// cannot be built, leaves SNS off and says so; the webhook is unaffected.

+// It never returns nil: with neither set, Page logs that no sink is

+// configured.

+func FromEnv(ctx context.Context, getenv func(string) string, newClient func(context.Context, string) (Publisher, error)) *Sinks {

+ if newClient == nil {

+ newClient = NewSNSClient

+ }

+ s := &Sinks{}

+ if url := strings.TrimSpace(getenv("CARLOS_ALERT_WEBHOOK")); url != "" {

+ s.WebhookURL, s.Post = url, PostJSON

+ }

+ if topic := strings.TrimSpace(getenv("CARLOS_ALERT_SNS_TOPIC_ARN")); topic != "" {

+ region, err := ParseTopicARN(topic)

+ if err != nil {

+ log.Printf("alerting: CARLOS_ALERT_SNS_TOPIC_ARN is not usable, SNS sink off: %v", err)

+ return s

+ }

+ pub, err := newClient(ctx, region)

+ if err != nil {

+ log.Printf("alerting: build the SNS client, SNS sink off: %v", err)

+ return s

+ }

+ s.TopicARN, s.Pub = topic, pub

+ }

+ return s

// PostJSON is the production webhook post: a 5 second POST with a JSON

// content type. Any non-2xx status is an error.

func PostJSON(url string, body []byte) error {

diff --git a/internal/alerting/alerting_test.go b/internal/alerting/alerting_test.go

index 90df95149..c84a93f87 100644

--- a/internal/alerting/alerting_test.go

+++ b/internal/alerting/alerting_test.go

@@ -206,3 +206,36 @@ func TestParseTopicARN(t *testing.T) {

}

+// FromEnv is the console's sinks (moves Plan 5 Task 7, delta M14): the

+// agent's two variables, the webhook through PostJSON and the topic's own

+// region for SNS. A topic that does not parse, or whose client cannot be

+// built, leaves SNS off and the webhook on; nothing set is no sink.

+func TestFromEnvBuildsTheConsoleSinks(t *testing.T) {

+ env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } }

+ var asked []string

+ client := func(_ context.Context, region string) (Publisher, error) {

+ asked = append(asked, region)

+ return &fakeSNS{}, nil

+ }

+ s := FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_WEBHOOK": " http://example.test/hook ", "CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), client)

+ if !s.HasWebhook() || s.WebhookURL != "http://example.test/hook" || !s.HasSNS() || s.TopicARN != testTopicARN {

+ t.Fatalf("both sinks: %+v", s)

+ }

+ if len(asked) != 1 || asked[0] != "eu-west-1" {

+ t.Fatalf("the SNS client was built for %v, want the topic's region", asked)

+ }

+ buf := captureLog(t)

+ s = FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_WEBHOOK": "http://example.test/hook", "CARLOS_ALERT_SNS_TOPIC_ARN": "arn:aws:sns:eu-west-1:1:x:sub"}), client)

+ if !s.HasWebhook() || s.HasSNS() || !strings.Contains(buf.String(), "SNS sink off") {

+ t.Fatalf("a bad topic: %+v (log %q)", s, buf.String())

+ }

+ s = FromEnv(context.Background(), env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), func(context.Context, string) (Publisher, error) { return nil, errors.New("no credentials") })

+ if s.HasWebhook() || s.HasSNS() {

+ t.Fatalf("a client that cannot be built: %+v", s)

+ }

+ s = FromEnv(context.Background(), env(nil), client)

+ if s == nil || s.HasWebhook() || s.HasSNS() {

+ t.Fatalf("nothing set: %+v", s)

+ }

diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go

index 1a702ccfc..455e34e1f 100644

--- a/internal/console/claim_authority_census_test.go

+++ b/internal/console/claim_authority_census_test.go

@@ -26,6 +26,9 @@ func claimAuthoritySites(t *testing.T) []string {

"ResumeExclusive": true, "RegisterStart": true, "FinishWakeRetryReset": true, "ReservePurge": true,

"ConsoleMove": true, "InspectMoveLoss": true, "DeclareMoveLoss": true,

"RunRecovery": true, "runMoveRecovery": true, "captureMoveStartBase": true, "resolveUncertain": true, "ConvertMovingToRelocating": true,

+ // The console's move pass (moves Plan 5 Task 7): every write of the

+ // move-recovery grant, and the lost target's stamp.

+ "ResolveStart": true, "Convert": true, "Claim": true, "Record": true, "StampLostMoveTargetParked": true, "stampLostMoveTarget": true,

"BeginPurge": true, "SavePurgeJournal": true, "RemovePurgeJournal": true,

"AcquireMailGate": true, "ReleaseMailGate": true, "EnrollNewApp": true,

"EnrollBackfillApp": true, "OpenGated": true, "WithAppMutation": true,

diff --git a/internal/console/claim_invariant_moves_test.go b/internal/console/claim_invariant_moves_test.go

index 51df1b2a3..94f14449f 100644

--- a/internal/console/claim_invariant_moves_test.go

+++ b/internal/console/claim_invariant_moves_test.go

@@ -132,3 +132,22 @@ func TestClaimPublishMovesConsoleIsInertWithTheSwitchOff(t *testing.T) {

t.Fatalf("the worker wrote with the moves switch off: %v", events)

}

+// TestClaimMovesConsoleBindsTheMovePass (moves Plan 5 Task 7): a moves-on

+// claim console binds its move pass as claim Plan 10's Recovery.Moves hook,

+// with the lost target's stamp; it adds no route and no worker (the pass runs

+// inside StartClaimRecoveryPass). A moves-off console binds neither, so the

+// recovery pass skips move claims as main does.

+func TestClaimMovesConsoleBindsTheMovePass(t *testing.T) {

+ off := newClaimCase(t).srv

+ if off.cfg.ClaimMoveRecovery != nil || off.cfg.HostClaimRecovery == nil || off.cfg.HostClaimRecovery.Moves != nil || off.stampLostMoveTarget != nil {

+ t.Fatal("a moves-off console bound the move pass")

+ }

+ on := newClaimCaseWith(t, claimMovesOn).srv

+ if on.cfg.ClaimMoveRecovery == nil || on.cfg.HostClaimRecovery == nil || on.cfg.HostClaimRecovery.Moves == nil || on.stampLostMoveTarget == nil {

+ t.Fatal("a moves-on console did not bind the move pass")

+ }

+ if on.cfg.HostClaimRecovery.MoveFenceAuthority == nil {

+ t.Fatal("the move pass has no fence authority for its conversions")

+ }

diff --git a/internal/console/claim_runtime.go b/internal/console/claim_runtime.go

index 6a41d1e10..c1bc23eb5 100644

--- a/internal/console/claim_runtime.go

+++ b/internal/console/claim_runtime.go

@@ -109,6 +109,21 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {

s.cfg.ClaimPlacement = placement

s.claimOwner = owner

s.claimRuntimeConfigured = true

+ if s.cfg.MovesEnabled && s.cfg.ClaimMoveRecovery == nil {

+ // The console's move pass (moves Plan 5 Task 7): the moves body of

+ // claim Plan 10's Recovery.Moves hook, bound only with the moves

+ // switch on, and only now that every box close is record first

+ // (ruling M4, P3-6; TestClaimMoveRecoveryWiringWaitsForRecordFirstOwners).

+ s.cfg.ClaimMoveRecovery = s.movesRecovery

+ // The lost target's target_parked_at (p10 Task G) is written

+ // outside the grant by the lifecycle record writer, which the

+ // conditional record store carries and the data boundary (an app

+ // permit's) does not. The pass gets this one transition, never

+ // the store.

+ s.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

+ return hostops.StampLostMoveTargetParked(ctx, records, host, now)

+ }

s.cfg.HostClaimRecovery = &hostops.Recovery{

Moves: s.cfg.ClaimMoveRecovery,

MoveFenceAuthority: hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: c.Now},

diff --git a/internal/console/movesrecovery.go b/internal/console/movesrecovery.go

new file mode 100644

index 000000000..ab9aed45d

--- /dev/null

+++ b/internal/console/movesrecovery.go

@@ -0,0 +1,470 @@

+package console

+import (

+ "context"

+ "crypto/sha256"

+ "encoding/hex"

+ "errors"

+ "fmt"

+ "log"

+ "os"

+ "slices"

+ "time"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostfence"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+// The console's move pass (moves Plan 5 Task 7, spec §7.5, §7.10, §7.11,

+// §13; Paul's ruling M4). It is the moves body of claim Plan 10's

+// Recovery.Moves hook: the recovery pass (ControlWriter.RecoveryPass) calls

+// movesRecovery once per claim of a move kind, with a MoveRecoveryGrant bound

+// to that claim, its move registration and the exact record it observed.

+// ConfigureClaimRuntime binds it only with the moves switch on.

+//

+// Each call reads the claim and the record again through the console's store,

+// gives up if either moved on since the grant's observation (the next pass

+// observes afresh), decides at most one action (decideMoveRecovery) and takes

+// it. The grant is spent by its first write, so one observation is never two

+// writes. In the order the decision tries them:

+//

+// 1. An orphan: a claim of a move kind whose record carries neither the move

+// nor its outcome. A moving claim before the seal (the request door's

+// record write was lost) is left for ten minutes, then its start effect

+// is settled without the request's handle (ResolveStart, K3), and on a

+// later pass, with the start settled not applied, released

+// (release-not-applied). Any other orphan is paged, once, and left: an

+// operation past the seal with no record is an incident.

+// 2. A source terminated before the flip (K13): Convert with the source's

+// termination fence, which writes the record's relocation decision on

+// one pass and converts the claim to relocating on the next.

+// 3. The six-hour abort (K18): a move still preparing moves.AbortAfter

+// after its request is aborted through ConsoleMove(ConsoleMoveAbort);

+// the source's own abort (movesabort.go) takes it from there.

+// 4. The claim half the record decided (§7.5, Review Focus 5): flip,

+// commit or rollback. Anything else is left alone.

+// 5. The deadline: a landing past the earlier of its cut and landing

+// deadlines is rolled back by the record (J11, J9's recorded reason).

+// 6. A lost target released with nothing of its box or region owed:

+// target_parked_at, through hostops.StampLostMoveTargetParked over the

+// conditional record store (stampLostMoveTarget), outside the grant

+// (p10 Task G).

+// 7. A close whose stamps are in and whose op owes nothing: the record's

+// close (close, rolled-back-close, revert-close or abort-complete). The

+// boxes close on their own in the pass the close becomes due; the

+// console writes it only when it has been due for moveCloseGrace, and

+// pages that it did.

+// 8. The release: the record closed (LastMove names the op) and the claim

+// not yet released. This is the second half of every record-first close

+// (ruling M4); after a box crashed between the halves, only this pass

+// writes it, because no box visits a record without a move.

+//

+// Pages (§13) go through alerting.Page on the sinks cmd/console arms, best

+// effort: a rollback waiting past its release deadline (an incident, once an

+// hour), a move preparing for over an hour (a warning, once), an orphan past

+// the seal (an incident, once) and an overdue close the console wrote (a

+// warning, once). The rate limit is durable: each page's rule is recorded in

+// the console's alert_seen table before the next pass, so a restarted console

+// does not repeat it.

+//

+// A relocating claim is region HA's, and the pass does nothing for it.

+const (

+ // moveOrphanAfter is how old, by Op.StartedAt, a moving claim with no

+ // record move must be before the pass settles its start.

+ moveOrphanAfter = 10 * time.Minute

+ // movePreparingPage is how long a move may prepare before it pages

+ // (§13: 60 minutes with no promotable round).

+ movePreparingPage = time.Hour

+ // moveCloseGrace is how long a close must have been due before the

+ // console writes it instead of the boxes.

+ moveCloseGrace = 10 * time.Minute

+ // movePageBox is the alert_seen box the move pass's pages are recorded

+ // under.

+ movePageBox = "moves"

+type moveRecoveryAction string

+const (

+ moveActNone moveRecoveryAction = ""

+ moveActResolveStart moveRecoveryAction = "resolve-start"

+ moveActConvert moveRecoveryAction = "convert"

+ moveActAbort moveRecoveryAction = "abort"

+ moveActClaim moveRecoveryAction = "claim"

+ moveActRecord moveRecoveryAction = "record"

+ moveActStampLost moveRecoveryAction = "stamp-lost"

+ moveActClose moveRecoveryAction = "close"

+type movePageKind string

+const (

+ movePageReleaseDeadline movePageKind = "release-deadline"

+ movePagePreparing movePageKind = "preparing"

+ movePageOrphan movePageKind = "orphan"

+ movePageCloseOverdue movePageKind = "close-overdue"

+// moveRecoveryInput is what one observation saw. Start is the op's move-start

+// effect, read only for a moving claim with no record move; Fence is the

+// source's termination fence, read only for a moving claim.

+type moveRecoveryInput struct {

+ Now time.Time

+ Claim *hostclaim.Claim

+ Rec provision.Record

+ Start *hostops.Effect

+ Fence *hostclaim.FenceRef

+// moveRecoveryDecision is at most one action, with the grant action it

+// names, and the pages the observation calls for.

+type moveRecoveryDecision struct {

+ Action moveRecoveryAction

+ Claim hostops.MoveRecoveryClaimAction

+ Record hostops.MoveRecoveryRecordAction

+ Pages []movePageKind

+// decideMoveRecovery is the pass's whole policy, on values (see the file

+// comment for the order).

+func decideMoveRecovery(in moveRecoveryInput) moveRecoveryDecision {

+ var d moveRecoveryDecision

+ c := in.Claim

+ if c == nil || c.Holder == nil || c.Op.ID == "" || c.Op.Kind == hostclaim.KindRelocating {

+ return d

+ }

+ op, rec, m := c.Op, in.Rec, in.Rec.Move

+ sealed := op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseSealed

+ if m == nil || m.Legacy() || m.ID != op.ID {

+ if m != nil {

+ return d // another move's record: not this op's to settle

+ }

+ if rec.LastMove != nil && rec.LastMove.ID == op.ID {

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRelease

+ return d

+ }

+ if op.Kind != hostclaim.KindMoving || op.Phase != hostclaim.PhaseMoving {

+ d.Pages = append(d.Pages, movePageOrphan)

+ return d

+ }

+ if op.StartedAt.IsZero() || in.Now.Before(op.StartedAt.Add(moveOrphanAfter)) || in.Start == nil {

+ return d

+ }

+ switch {

+ case in.Start.State == "uncertain":

+ d.Action = moveActResolveStart

+ case in.Start.State == "complete" && in.Start.ResultReference == "not-applied":

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveReleaseNotApplied

+ }

+ return d

+ }

+ if m.Incarnation != c.Holder.Incarnation {

+ return d

+ }

+ rb := m.Rollback

+ if m.State == provision.MoveRollingBack && rb != nil && rb.ReleaseDeadline != nil && in.Now.After(*rb.ReleaseDeadline) && rb.TargetPhase == provision.RbTargetReleasing {

+ d.Pages = append(d.Pages, movePageReleaseDeadline)

+ }

+ if m.State == provision.MovePreparing && !m.RequestedAt.IsZero() && !in.Now.Before(preparingSince(m).Add(movePreparingPage)) {

+ d.Pages = append(d.Pages, movePagePreparing)

+ }

+ owed := owedUnderOp(c, op.ID, nil)

+ switch {

+ case op.Kind == hostclaim.KindMoving && in.Fence != nil && (m.State == provision.MovePreparing || m.State == provision.MoveReady || m.State == provision.MoveCutting || m.State == provision.MoveRelocating):

+ d.Action = moveActConvert

+ case op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseMoving && moves.AbortDue(rec, in.Now):

+ d.Action = moveActAbort

+ case m.State == provision.MoveLanding && m.FlippedAt != nil && rec.Region == m.To && sealed:

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveFlip

+ case m.State == provision.MoveCommitted && m.CommittedAt != nil && m.FlippedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseLandingProbed:

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveCommit

+ case m.State == provision.MoveRollingBack && rb != nil && rec.Region == m.From && (op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted || sealed):

+ d.Action, d.Claim = moveActClaim, hostops.RecoveryMoveRollback

+ case m.State == provision.MoveLanding && op.Kind == hostclaim.KindLanding && op.Phase != hostclaim.PhaseCommitted && moveDeadlinePassed(m, in.Now):

+ d.Action, d.Record = moveActRecord, hostops.RecoveryMoveDeadlineRollback

+ case m.State == provision.MoveRollingBack && rb != nil && rb.TargetPhase == provision.RbTargetReleased && !rb.Dirty && rb.TargetParkedAt == nil &&

+ (op.Kind == hostclaim.KindRestoring || op.Kind == hostclaim.KindLanding) && m.To != "" && op.Intent["target"] != "" && slices.Contains(c.Lost, op.Intent["target"]) &&

+ owedUnderOp(c, op.ID, func(o hostclaim.Obligation) bool {

+ t := op.Intent["target"]

+ return o.Box == t || o.Owner == "box:"+t || o.Region == m.To || o.Owner == "region:"+m.To

+ }) == 0:

+ d.Action = moveActStampLost

+ case owed > 0:

+ case m.State == provision.MoveCommitted && m.OpenedAt != nil && m.SourceParkedAt != nil && rec.Region == m.To && op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseCommitted:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveClose

+ case m.State == provision.MoveRollingBack && rb != nil && rb.RecoveredAt != nil && rb.OpenedAt != nil && rb.TargetParkedAt != nil && rec.Region == m.From && op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseOpened:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveRolledBackClose

+ case m.State == provision.MoveReverting && m.ReopenedAt != nil && rec.Region == m.From && op.Kind == hostclaim.KindReverting && op.Phase == hostclaim.PhaseOpened:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveRevertClose

+ case m.State == provision.MoveAborting && op.Kind == hostclaim.KindMoving && op.Phase == hostclaim.PhaseAborting:

+ d.Action, d.Record = moveActClose, hostops.RecoveryMoveAbortComplete

+ }

+ return d

+// preparingSince is where a preparing move's clocks run from: the later of

+// its request and its readiness (moves.AbortDue's base, K18).

+func preparingSince(m *provision.MoveState) time.Time {

+ from := m.RequestedAt

+ if m.ReadyAt != nil && m.ReadyAt.After(from) {

+ from = *m.ReadyAt

+ }

+ return from

+// moveDeadlinePassed is the grant's deadline rule: the earlier of the cut and

+// landing deadlines (J11) is behind now.

+func moveDeadlinePassed(m *provision.MoveState, now time.Time) bool {

+ deadline := m.CutDeadline

+ if l := m.LandingDeadline; l != nil && (deadline == nil || l.Before(*deadline)) {

+ deadline = l

+ }

+ return deadline != nil && now.After(*deadline)

+// owedUnderOp counts the claim's obligations under op id, those match

+// accepts when it is set.

+func owedUnderOp(c *hostclaim.Claim, id string, match func(hostclaim.Obligation) bool) int {

+ n := 0

+ for _, o := range c.Obligations {

+ if o.Op == id && (match == nil || match(o)) {

+ n++

+ }

+ return n

+func (s *Server) movesNow() time.Time {

+ if s.movesClock != nil {

+ return s.movesClock().UTC()

+ }

+ return time.Now().UTC()

+// movesRecovery is the Recovery.Moves hook (see the file comment).

+func (s *Server) movesRecovery(ctx context.Context, g *hostops.MoveRecoveryGrant) error {

+ a := s.cfg.ClaimAuthority

+ store := s.cfg.Releases

+ if a == nil || a.Control == nil || store == nil || g == nil {

+ return hostops.ErrContract

+ }

+ o := g.Observation()

+ obs, err := hostclaim.Read(ctx, store, o.Host)

+ if err != nil {

+ return err

+ }

+ if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.ETag != o.ClaimETag || obs.Claim.Op.ID != o.OpID {

+ return nil // moved on since the observation: the next pass observes again

+ }

+ c := obs.Claim

+ r, err := hostops.ReadRecord(ctx, store, c.Holder.Key)

+ if err != nil {

+ return err

+ }

+ if r.ETag != o.RecordETag {

+ return nil

+ }

+ rec, err := provision.DecodeRecord(r.Body)

+ if err != nil {

+ return err

+ }

+ now := s.movesNow()

+ in := moveRecoveryInput{Now: now, Claim: c, Rec: rec}

+ ours := rec.Move != nil && rec.Move.ID == o.OpID

+ if c.Op.Kind == hostclaim.KindMoving && c.Op.Phase == hostclaim.PhaseMoving && !ours {

+ adm, err := hostops.ReadAdmission(ctx, store, o.Account, o.App, o.AppID)

+ if err != nil {

+ return err

+ }

+ if e, ok := adm.Value.Effects[o.OpID]; ok {

+ in.Start = &e

+ }

+ if c.Op.Kind == hostclaim.KindMoving && ours {

+ if in.Fence, err = s.sourceFence(ctx, c); err != nil {

+ return err

+ }

+ d := decideMoveRecovery(in)

+ for _, kind := range d.Pages {

+ s.pageMove(ctx, kind, o, rec, now)

+ }

+ switch d.Action {

+ case moveActResolveStart:

+ out, err := g.ResolveStart(ctx)

+ if errors.Is(err, hostops.ErrMoveStartFenceUnproven) {

+ log.Printf("console: moves: %s: the start of move %s is uncertain and its record has not advanced past it; left", o.Host, o.OpID)

+ return nil

+ }

+ if err == nil {

+ log.Printf("console: moves: %s: the start of move %s settled %s", o.Host, o.OpID, out)

+ }

+ return err

+ case moveActConvert:

+ return g.Convert(ctx, *in.Fence)

+ case moveActAbort:

+ co, err := hostops.ObserveConsoleMove(ctx, store, o.Account, o.App, o.Host)

+ if err != nil {

+ return err

+ }

+ if co != o {

+ return nil

+ }

+ if _, err := a.Control.ConsoleMove(ctx, co, hostops.ConsoleMoveAbort); err != nil {

+ return err

+ }

+ log.Printf("console: moves: %s: move %s is still preparing after %s; aborted", o.Host, o.OpID, moves.AbortAfter)

+ return nil

+ case moveActClaim:

+ return g.Claim(ctx, d.Claim)

+ case moveActRecord:

+ return g.Record(ctx, d.Record)

+ case moveActStampLost:

+ if s.stampLostMoveTarget == nil {

+ return hostops.ErrContract

+ }

+ return s.stampLostMoveTarget(ctx, o.Host, now)

+ case moveActClose:

+ if !s.moveCloseOverdue(o, d.Record, now) {

+ return nil

+ }

+ if err := g.Record(ctx, d.Record); err != nil {

+ return err

+ }

+ log.Printf("console: moves: %s: move %s closed by the console (%s); the release follows", o.Host, o.OpID, d.Record)

+ s.pageMove(ctx, movePageCloseOverdue, o, rec, now)

+ return nil

+ }

+ return nil

+// sourceFence is the moving source's termination fence, when one exists:

+// the reference to control/box-fences/<source enrollment>.json and its exact

+// bytes. Convert verifies it; nothing here trusts it.

+func (s *Server) sourceFence(ctx context.Context, c *hostclaim.Claim) (*hostclaim.FenceRef, error) {

+ enrollment := c.Op.Intent["source_enrollment"]

+ if enrollment == "" {

+ return nil, nil

+ }

+ key := hostfence.FenceKey(enrollment)

+ body, _, err := s.cfg.Releases.Get(ctx, key)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, nil

+ }

+ if err != nil {

+ return nil, err

+ }

+ sum := sha256.Sum256(body)

+ return &hostclaim.FenceRef{Enrollment: enrollment, Key: key, SHA256: hex.EncodeToString(sum[:])}, nil

+// moveCloseOverdue reports whether this close has been due for

+// moveCloseGrace, remembering when it was first seen due. The memory is the

+// process's: a restarted console waits the grace again, which only delays

+// the console's close.

+func (s *Server) moveCloseOverdue(o hostops.ConsoleMoveObservation, action hostops.MoveRecoveryRecordAction, now time.Time) bool {

+ key := o.Host + "|" + o.OpID + "|" + string(action)

+ s.moveCloseMu.Lock()

+ defer s.moveCloseMu.Unlock()

+ if s.moveCloseSeen == nil {

+ s.moveCloseSeen = map[string]time.Time{}

+ }

+ for k, at := range s.moveCloseSeen {

+ if now.Sub(at) > 24*time.Hour {

+ delete(s.moveCloseSeen, k)

+ }

+ first, ok := s.moveCloseSeen[key]

+ if !ok {

+ s.moveCloseSeen[key] = now

+ return false

+ }

+ if now.Sub(first) < moveCloseGrace {

+ return false

+ }

+ delete(s.moveCloseSeen, key)

+ return true

+// pageMove sends one page if its rule has not been paged: once per move,

+// or, for the release deadline, once per hour since the deadline. The rule

+// is recorded whether or not delivery worked (pages are best effort).

+func (s *Server) pageMove(ctx context.Context, kind movePageKind, o hostops.ConsoleMoveObservation, rec provision.Record, now time.Time) {

+ rule := string(kind) + " " + o.Host + " " + o.OpID

+ level := alerting.LevelWarning

+ var subject, body string

+ switch kind {

+ case movePageReleaseDeadline:

+ rb := rec.Move.Rollback

+ rule += fmt.Sprintf(" %d", int(now.Sub(*rb.ReleaseDeadline)/time.Hour))

+ level = alerting.LevelIncident

+ subject, body = moveWaitingPage(o.Host, o.OpID, rec.Move.To)

+ case movePagePreparing:

+ subject, body = movePreparingPageText(o.Host, o.OpID, rec.Move.To)

+ case movePageOrphan:

+ level = alerting.LevelIncident

+ subject, body = moveOrphanPage(o.Host, o.OpID)

+ case movePageCloseOverdue:

+ subject, body = moveCloseOverduePage(o.Host, o.OpID)

+ default:

+ return

+ }

+ if s.store == nil {

+ return

+ }

+ seen, known, err := s.store.AlertSeen(movePageBox, rule, noMailboxAddress)

+ if err != nil {

+ log.Printf("console: moves: %s: the page record for %s: %v", o.Host, rule, err)

+ return

+ }

+ if known && seen {

+ return

+ }

+ if err := alerting.Page(ctx, level, subject, body); err != nil {

+ log.Printf("console: moves: %s: page %s: %v", o.Host, kind, err)

+ }

+ if err := s.store.SetAlertSeen(movePageBox, rule, noMailboxAddress, true); err != nil {

+ log.Printf("console: moves: %s: record the page %s: %v", o.Host, rule, err)

+ }

+// moveWaitingPage is a rollback waiting past its release deadline (§7.8

+// waiting): the subject (MV-N325) and the body (MV-N326).

+func moveWaitingPage(host, id, to string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move waiting for %s to release: %s", to, host)

+ body = fmt.Sprintf("The move %s of %s is rolling back, and %s has not released it by its deadline. %s stays offline until a box in %s confirms it has stopped, or that box is declared lost.",

+ id, host, to, host, to)

+ return subject, body

+// movePreparingPageText is a move preparing for over an hour (§13): the

+// subject (MV-N327) and the body (MV-N328).

+func movePreparingPageText(host, id, to string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move preparing for over an hour: %s", host)

+ body = fmt.Sprintf("The move %s of %s to %s has been preparing for over an hour. It aborts itself six hours after the request if %s is still not ready.",

+ id, host, to, to)

+ return subject, body

+// moveOrphanPage is a claim held by a move the record does not carry (K3):

+// the subject (MV-N329) and the body (MV-N330).

+func moveOrphanPage(host, id string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move without a record: %s", host)

+ body = fmt.Sprintf("The host claim of %s is held by the move %s, and the instance record carries no move. Nothing releases it on its own. An operator must check the record.",

+ host, id)

+ return subject, body

+// moveCloseOverduePage is a close the console wrote because no box did: the

+// subject (MV-N331) and the body (MV-N332).

+func moveCloseOverduePage(host, id string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move close overdue: %s", host)

+ body = fmt.Sprintf("The move %s of %s was ready to close for more than 10 minutes and no box closed it. The console has closed it.", id, host)

+ return subject, body

diff --git a/internal/console/movesrecovery_test.go b/internal/console/movesrecovery_test.go

new file mode 100644

index 000000000..82d7ff161

--- /dev/null

+++ b/internal/console/movesrecovery_test.go

@@ -0,0 +1,671 @@

+package console

+import (

+ "context"

+ "crypto/rand"

+ "encoding/json"

+ "strings"

+ "sync"

+ "testing"

+ "time"

+ "github.com/carlosframework/platform/internal/alerting"

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/hostops"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+// The console's move pass (moves Plan 5 Task 7; claim Plan 10's

+// Recovery.Moves hook under the move-recovery grant). The decision table is

+// tested on values; every action is then driven end to end through the real

+// recovery pass (ControlWriter.RecoveryPass over hostops.Recovery.Pass) on a

+// host-claim console whose move the door started.

+// ---- the decision, on values ----

+var mvT0 = time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)

+func mvClaim(kind, phase string, mut ...func(*hostclaim.Claim)) *hostclaim.Claim {

+ c := &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc"},

+ Op: hostclaim.Op{Kind: kind, Phase: phase, ID: "mv", Exec: "ie-box", Participants: []string{"ie-box", "uk-box"}, StartedAt: mvT0,

+ Intent: map[string]string{"source": "ie-box", "target": "uk-box"}}}

+ for _, m := range mut {

+ m(c)

+ }

+ return c

+func mvRecord(state string, mut ...func(*provision.Record)) provision.Record {

+ at := mvT0

+ r := provision.Record{Region: "ie", Move: &provision.MoveState{ID: "mv", Incarnation: "inc", From: "ie", To: "uk", State: state, RequestedAt: mvT0}}

+ switch state {

+ case provision.MoveLanding, provision.MoveCommitted:

+ r.Region = "uk"

+ r.Move.FlippedAt = &at

+ }

+ if state == provision.MoveCommitted {

+ r.Move.CommittedAt = &at

+ }

+ if state == provision.MoveRollingBack {

+ r.Move.FlippedAt = &at

+ r.Move.Rollback = &provision.MoveRollback{ID: "rb", TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceWaiting}

+ }

+ for _, m := range mut {

+ m(&r)

+ }

+ return r

+func mvAt(d time.Duration) *time.Time { t := mvT0.Add(d); return &t }

+func TestRecoveryPassWritesOnlyDecidedHalves(t *testing.T) {

+ now := mvT0.Add(time.Minute)

+ for _, c := range []struct {

+ name string

+ claim *hostclaim.Claim

+ rec provision.Record

+ action moveRecoveryAction

+ half hostops.MoveRecoveryClaimAction

+ }{

+ {"record landing, claim moving/sealed: the flip half", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding), moveActClaim, hostops.RecoveryMoveFlip},

+ {"record committed, claim landing/probed: the commit half", mvClaim(hostclaim.KindLanding, hostclaim.PhaseLandingProbed), mvRecord(provision.MoveCommitted), moveActClaim, hostops.RecoveryMoveCommit},

+ {"record rolling_back, claim landing: the rollback half", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack), moveActClaim, hostops.RecoveryMoveRollback},

+ {"record rolling_back, claim moving/sealed: RollBackFlipped (K22)", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveRollingBack), moveActClaim, hostops.RecoveryMoveRollback},

+ {"record landing, claim landing: in step", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveLanding), moveActNone, ""},

+ {"record cutting, claim moving/sealed: no decision", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveCutting), moveActNone, ""},

+ {"record committed, claim landing/probing: the claim is behind the probe", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveCommitted), moveActNone, ""},

+ {"record committed, claim landing/committed, stamps missing", mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), mvRecord(provision.MoveCommitted), moveActNone, ""},

+ {"record landing without its flip stamp", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.FlippedAt = nil }), moveActNone, ""},

+ {"record rolling_back in the target's region", mvClaim(hostclaim.KindLanding, "probing"), mvRecord(provision.MoveRollingBack, func(r *provision.Record) { r.Region = "uk" }), moveActNone, ""},

+ {"record preparing, claim moving: nothing yet", mvClaim(hostclaim.KindMoving, hostclaim.PhaseMoving), mvRecord(provision.MovePreparing), moveActNone, ""},

+ {"another move's record", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.ID = "other" }), moveActNone, ""},

+ {"another incarnation's record", mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed), mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.Incarnation = "old" }), moveActNone, ""},

+ {"a relocating claim is region HA's", mvClaim(hostclaim.KindRelocating, "pinning"), mvRecord(provision.MoveRelocating), moveActNone, ""},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ got := decideMoveRecovery(moveRecoveryInput{Now: now, Claim: c.claim, Rec: c.rec})

+ if got.Action != c.action || got.Claim != c.half {

+ t.Fatalf("decided %+v, want %s %s", got, c.action, c.half)

+ }

+ })

+ }

+// One eligible action per observation, in the brief's order: a pending

+// claim half comes before the deadline, the deadline before a close, the

+// lost target's stamp before the close it enables.

+func TestRecoveryPassOneActionPerObservation(t *testing.T) {

+ past := mvT0.Add(-time.Hour)

+ // Flip half pending and the landing deadline passed: the half first.

+ d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindMoving, hostclaim.PhaseSealed),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })})

+ if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveFlip {

+ t.Fatalf("flip and deadline: %+v", d)

+ }

+ // With the claim in step, the deadline.

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.LandingDeadline = &past })})

+ if d.Action != moveActRecord || d.Record != hostops.RecoveryMoveDeadlineRollback {

+ t.Fatalf("the deadline: %+v", d)

+ }

+ // The earlier of the cut and landing deadlines (J11).

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, "probing"),

+ Rec: mvRecord(provision.MoveLanding, func(r *provision.Record) { r.Move.CutDeadline = &past; r.Move.LandingDeadline = mvAt(time.Hour) })})

+ if d.Action != moveActRecord {

+ t.Fatalf("the cut deadline: %+v", d)

+ }

+ // A lost, released target: the stamp, never the close it enables.

+ lost := func(c *hostclaim.Claim) { c.Lost = []string{"uk-box"} }

+ rb := func(r *provision.Record) {

+ r.Move.Rollback.TargetPhase, r.Move.Rollback.SourcePhase = provision.RbTargetReleased, provision.RbSourceOpened

+ r.Move.Rollback.RecoveredAt, r.Move.Rollback.OpenedAt = mvAt(0), mvAt(0)

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened, lost), Rec: mvRecord(provision.MoveRollingBack, rb)})

+ if d.Action != moveActStampLost {

+ t.Fatalf("the lost target: %+v", d)

+ }

+ // A chore of the target's region still owed: neither.

+ owed := func(c *hostclaim.Claim) {

+ c.Obligations = []hostclaim.Obligation{{ID: "o", Op: "mv", Kind: "park-replica", Region: "uk", Owner: "region:uk"}}

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened, lost, owed), Rec: mvRecord(provision.MoveRollingBack, rb)})

+ if d.Action != moveActNone {

+ t.Fatalf("the lost target with its region's chore owed: %+v", d)

+ }

+ // Every close waits for the op's chores (the grant's rule, M4).

+ stamped := func(r *provision.Record) { r.Move.OpenedAt, r.Move.SourceParkedAt = mvAt(0), mvAt(0) }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted, owed), Rec: mvRecord(provision.MoveCommitted, stamped)})

+ if d.Action != moveActNone {

+ t.Fatalf("a close with a chore owed: %+v", d)

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), Rec: mvRecord(provision.MoveCommitted, stamped)})

+ if d.Action != moveActClose || d.Record != hostops.RecoveryMoveClose {

+ t.Fatalf("the committed close: %+v", d)

+ }

+ for _, c := range []struct {

+ claim *hostclaim.Claim

+ rec provision.Record

+ want hostops.MoveRecoveryRecordAction

+ }{

+ {mvClaim(hostclaim.KindRestoring, hostclaim.PhaseOpened), mvRecord(provision.MoveRollingBack, rb, func(r *provision.Record) { r.Move.Rollback.TargetParkedAt = mvAt(0) }), hostops.RecoveryMoveRolledBackClose},

+ {mvClaim(hostclaim.KindReverting, hostclaim.PhaseOpened), mvRecord(provision.MoveReverting, func(r *provision.Record) { r.Move.ReopenedAt = mvAt(0) }), hostops.RecoveryMoveRevertClose},

+ {mvClaim(hostclaim.KindMoving, hostclaim.PhaseAborting), mvRecord(provision.MoveAborting), hostops.RecoveryMoveAbortComplete},

+ } {

+ if d := decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: c.claim, Rec: c.rec}); d.Action != moveActClose || d.Record != c.want {

+ t.Fatalf("%s/%s: %+v, want close %s", c.claim.Op.Kind, c.claim.Op.Phase, d, c.want)

+ }

+ // The record closed, the claim not released: the release.

+ closed := func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: "mv", Outcome: "done", At: mvT0}

+ }

+ d = decideMoveRecovery(moveRecoveryInput{Now: mvT0, Claim: mvClaim(hostclaim.KindLanding, hostclaim.PhaseCommitted), Rec: mvRecord(provision.MoveCommitted, closed)})

+ if d.Action != moveActClaim || d.Claim != hostops.RecoveryMoveRelease {

+ t.Fatalf("the release from LastMove: %+v", d)

+ }

+// ---- the pass, end to end ----

+// movePassWorld is a host-claim console whose move the request door

+// started (j17StartedMove), with the console's own move pass bound as the

+// Recovery.Moves hook, pages captured from the armed alert sinks, and the

+// pass's clock under the test's control.

+type movePassWorld struct {

+ fx *consoleFixture

+ id string

+ auth hostclaim.FenceAuthority

+ clock time.Time

+ mu sync.Mutex

+ pages []pagePayload

+type pagePayload struct{ Kind, Level, Subject, Body string }

+func newMovePassWorld(t *testing.T, fx *consoleFixture, id string) *movePassWorld {

+ t.Helper()

+ w := &movePassWorld{fx: fx, id: id, clock: time.Now().UTC()}

+ fx.srv.movesClock = func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.clock }

+ // As ConfigureClaimRuntime binds it: the lost target's stamp over the

+ // conditional record store under the fixture's data boundary.

+ records, err := hostops.NewConditionalRecordStore(fx.rel, fx.pub)

+ if err != nil {

+ t.Fatal(err)

+ }

+ fx.srv.stampLostMoveTarget = func(ctx context.Context, host string, now time.Time) error {

+ return hostops.StampLostMoveTargetParked(ctx, records, host, now)

+ }

+ alerting.Arm(&alerting.Sinks{WebhookURL: "http://pages.test/hook", Post: func(_ string, body []byte) error {

+ var p pagePayload

+ if err := json.Unmarshal(body, &p); err != nil {

+ return err

+ }

+ w.mu.Lock()

+ w.pages = append(w.pages, p)

+ w.mu.Unlock()

+ return nil

+ }})

+ t.Cleanup(func() { alerting.Arm(nil) })

+ return w

+func (w *movePassWorld) advance(d time.Duration) {

+ w.mu.Lock()

+ w.clock = w.clock.Add(d)

+ w.mu.Unlock()

+// pass runs one console recovery pass with only the moves hook live.

+func (w *movePassWorld) pass(t *testing.T) {

+ t.Helper()

+ a := w.fx.srv.cfg.ClaimAuthority

+ nop := func(context.Context) error { return nil }

+ nopObs := func(context.Context, hostclaim.Observed) error { return nil }

+ r := hostops.Recovery{Moves: w.fx.srv.movesRecovery, MoveFenceAuthority: w.auth,

+ Controller: hostops.Controller{Store: w.fx.srv.cfg.Releases, Control: a.Control, Authority: a, Now: time.Now, NewID: claimOperationID},

+ Purge: nopObs, Placement: nopObs, SweepRecords: nop, SweepPurges: nop, SweepAdmissions: nop}

+ if err := a.Control.RecoveryPass(context.Background(), r); err != nil {

+ t.Fatalf("the recovery pass: %v", err)

+ }

+func (w *movePassWorld) claim(t *testing.T) *hostclaim.Claim {

+ t.Helper()

+ o, err := hostclaim.Read(context.Background(), w.fx.rel, "acme.go.tito.io")

+ if err != nil || o.Absent {

+ t.Fatalf("read the claim: %v", err)

+ }

+ return o.Claim

+func (w *movePassWorld) record(t *testing.T) provision.Record {

+ t.Helper()

+ rec, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "titogo", "acme.go.tito.io")

+ if err != nil || rec == nil {

+ t.Fatalf("read the record: %v", err)

+ }

+ return *rec

+// setClaim rewrites the claim under the raw bucket, as a box would.

+func (w *movePassWorld) setClaim(t *testing.T, mut func(*hostclaim.Claim)) {

+ t.Helper()

+ ctx := context.Background()

+ obs, err := hostclaim.Read(ctx, w.fx.rel, "acme.go.tito.io")

+ if err != nil {

+ t.Fatal(err)

+ }

+ if _, err := hostclaim.Update(ctx, w.fx.rel, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil {

+ t.Fatalf("rewrite the claim: %v", err)

+ }

+func (w *movePassWorld) startEffect(t *testing.T) hostops.Effect {

+ t.Helper()

+ a, err := hostops.ReadAdmission(context.Background(), w.fx.rel, "bab", "titogo", claimMoveAppID)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return a.Value.Effects[w.id]

+// setStartEffect rewrites the move-start effect of the admission object.

+func (w *movePassWorld) setStartEffect(t *testing.T, mut func(*hostops.Effect)) {

+ t.Helper()

+ ctx := context.Background()

+ body, etag, err := w.fx.rel.Get(ctx, "apps/bab/titogo/admission.json")

+ if err != nil {

+ t.Fatal(err)

+ }

+ var a hostops.AppAdmission

+ if err := json.Unmarshal(body, &a); err != nil {

+ t.Fatal(err)

+ }

+ e := a.Effects[w.id]

+ mut(&e)

+ a.Effects[w.id] = e

+ if body, err = json.Marshal(a); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := w.fx.rel.Put(ctx, "apps/bab/titogo/admission.json", body, etag); err != nil {

+ t.Fatal(err)

+ }

+func (w *movePassWorld) paged(t *testing.T) []pagePayload {

+ t.Helper()

+ w.mu.Lock()

+ defer w.mu.Unlock()

+ return append([]pagePayload(nil), w.pages...)

+func startedMovePass(t *testing.T, code string) *movePassWorld {

+ t.Helper()

+ fx, _, _, id := j17StartedMove(t, code)

+ return newMovePassWorld(t, fx, id)

+func landingMovePass(t *testing.T, code string) *movePassWorld {

+ t.Helper()

+ fx, _, _ := j4LandingMove(t, code)

+ return newMovePassWorld(t, fx, j17MoveState(t, fx).ID)

+func newRollbackID(t *testing.T) string {

+ t.Helper()

+ id, err := moves.NewRollbackID(rand.Reader)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return id

+// TestRecoveryPassRollsBackAtTheLandingDeadline: a landing past the earlier

+// of its deadlines is rolled back by the record (one pass), and the claim

+// follows on the next (the rollback half). Each pass writes one object.

+func TestRecoveryPassRollsBackAtTheLandingDeadline(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-deadline")

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveLanding {

+ t.Fatalf("rolled back before the deadline: %+v", m)

+ }

+ past := time.Now().UTC().Add(-time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move.LandingDeadline = &past })

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move.State != provision.MoveRollingBack || rec.Move.Rollback == nil || rec.Move.Rollback.Reason != provision.RollbackLandingDeadline || rec.Region != "ie" {

+ t.Fatalf("the deadline's rollback: region %s %+v", rec.Region, rec.Move)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the pass that wrote the record wrote the claim too")

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindRestoring || c.Op.ID != w.id {

+ t.Fatalf("the claim did not follow the rollback: %+v", c.Op)

+ }

+ if after := w.record(t); after.Move.Rollback.ID != rec.Move.Rollback.ID || after.RecordSeq != rec.RecordSeq {

+ t.Fatalf("the claim's pass wrote the record: %+v", after.Move)

+ }

+// TestRecoveryPassSettlesAnOrphanStartThenReleases (K3): the request door's

+// record write was lost, so the claim is moving/"" and the record carries no

+// move, with the start effect uncertain. Under ten minutes the pass leaves

+// it. After, it settles the start not-applied from record_seq (no handle),

+// and on the next pass releases the claim to serving under the source. A

+// landing orphan is paged once and never released.

+func TestRecoveryPassSettlesAnOrphanStartThenReleases(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-orphan")

+ j17SetRecord(t, w.fx, func(r *provision.Record) { r.Move = nil })

+ w.setStartEffect(t, func(e *hostops.Effect) { e.State, e.ResultReference = "uncertain", "" })

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving || w.startEffect(t).State != "uncertain" {

+ t.Fatalf("acted inside ten minutes: claim %+v effect %+v", c.Op, w.startEffect(t))

+ }

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.StartedAt = time.Now().UTC().Add(-11 * time.Minute) })

+ w.pass(t)

+ if e := w.startEffect(t); e.State != "complete" || e.ResultReference != "not-applied" {

+ t.Fatalf("the start was not settled: %+v", e)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving {

+ t.Fatalf("the settling pass released too: %+v", c.Op)

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(c) != nil {

+ t.Fatalf("the orphan was not released: %+v", c.Op)

+ }

+ if p := w.paged(t); len(p) != 0 {

+ t.Fatalf("a moving orphan paged: %+v", p)

+ }

+ // A landing orphan pages once, and is left.

+ l := landingMovePass(t, "code-mvpass-orphan-landing")

+ j17SetRecord(t, l.fx, func(r *provision.Record) { r.Move = nil })

+ before := claimBytes(t, l.fx, "acme.go.tito.io")

+ l.pass(t)

+ if claimBytes(t, l.fx, "acme.go.tito.io") != before {

+ t.Fatal("a landing orphan's claim was written")

+ }

+ p := l.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelIncident || !strings.Contains(p[0].Subject, "acme.go.tito.io") || !strings.Contains(p[0].Body, l.id) {

+ t.Fatalf("the landing orphan's pages: %+v", p)

+ }

+// oneBoxFenceAuthority is c7MoveAuthority with the one-box-region check the

+// grant's Convert needs (the console's hostfence.ClaimFenceAuthority has

+// both).

+type oneBoxFenceAuthority struct{ c7MoveAuthority }

+func (oneBoxFenceAuthority) CheckOneBoxRegion(context.Context, string, string) error { return nil }

+// TestRecoveryPassConvertsAFencedSourceInTwoPasses (K13): a source

+// terminated before the flip. The first pass writes the record's relocation

+// decision, the second the claim's conversion to relocating.

+func TestRecoveryPassConvertsAFencedSourceInTwoPasses(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-convert")

+ w.auth = oneBoxFenceAuthority{c7MoveAuthority{region: "ie"}}

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MovePreparing {

+ t.Fatalf("converted with no fence: %+v", m)

+ }

+ terminatedSourceFence(t, w.fx, "ie-box", "uk-box")

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveRelocating {

+ t.Fatalf("the decision pass: %+v", m)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindMoving {

+ t.Fatalf("the decision pass converted the claim: %+v", c.Op)

+ }

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != "pinning" || c.Op.ID != w.id {

+ t.Fatalf("the conversion pass: %+v", c.Op)

+ }

+ w.pass(t) // a relocating claim is region HA's: nothing

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRelocating {

+ t.Fatalf("after the conversion: %+v", c.Op)

+ }

+// lostTargetRollback puts the landing world into a rollback whose target

+// box was lost and released: the source recovered and opened, the claim

+// restoring/opened under the source.

+func lostTargetRollback(t *testing.T, w *movePassWorld) {

+ t.Helper()

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box"

+ c.Lost = []string{"uk-box"}

+ c.Obligations = nil

+ })

+ rb, at := newRollbackID(t), time.Now().UTC().Add(-time.Hour)

+ deadline := at.Add(15 * time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Region = "ie"

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: rb, RequestedAt: &at, Reason: provision.RollbackDeclaredLost, TargetPhase: provision.RbTargetReleased,

+ SourcePhase: provision.RbSourceOpened, RecoveredAt: &at, OpenedAt: &at, ReleaseDeadline: &deadline}

+ })

+// TestRecoveryPassStampsALostTargetThenCloses (p10 Task G): with the target

+// lost and released and nothing of its box or region owed, the pass stamps

+// target_parked_at outside the grant; the close follows once overdue, then

+// the release.

+func TestRecoveryPassStampsALostTargetThenCloses(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-lost")

+ lostTargetRollback(t, w)

+ w.pass(t)

+ rb := w.record(t).Move.Rollback

+ if rb.TargetParkedAt == nil || rb.TargetPhase != provision.RbTargetCleaned {

+ t.Fatalf("the lost target was not stamped: %+v", rb)

+ }

+ w.pass(t)

+ if w.record(t).Move == nil {

+ t.Fatal("closed inside the boxes' grace")

+ }

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != "rolled_back" {

+ t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindRestoring {

+ t.Fatalf("the close's pass released too: %+v", c.Op)

+ }

+ w.pass(t)

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

+ t.Fatalf("not released: %+v", c.Op)

+ }

+// committedAndOpened puts the landing world at a committed, opened and

+// source-parked move with nothing owed: the close is due.

+func committedAndOpened(t *testing.T, w *movePassWorld) {

+ t.Helper()

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseCommitted, nil })

+ at := time.Now().UTC().Add(-time.Hour)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move.State, r.Move.LandingPhase = provision.MoveCommitted, provision.LandProbed

+ r.Move.CommittedAt, r.Move.OpenedAt, r.Move.SourceParkedAt = &at, &at, &at

+ })

+// TestRecoveryPassFinishesTheClose: a close the boxes did not write within

+// moveCloseGrace is overdue. The pass pages it once, closes the record

+// (LastMove done) and, on the next pass, releases under the target.

+func TestRecoveryPassFinishesTheClose(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-close")

+ committedAndOpened(t, w)

+ w.pass(t)

+ if w.record(t).Move == nil || len(w.paged(t)) != 0 {

+ t.Fatalf("closed or paged inside the grace: %+v", w.paged(t))

+ }

+ w.advance(moveCloseGrace + time.Minute)

+ w.pass(t)

+ rec := w.record(t)

+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != w.id || rec.LastMove.Outcome != "done" {

+ t.Fatalf("not closed: %+v %+v", rec.Move, rec.LastMove)

+ }

+ if c := w.claim(t); c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted {

+ t.Fatalf("released in the close's pass: %+v", c.Op)

+ }

+ if p := w.paged(t); len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Body, w.id) {

+ t.Fatalf("the overdue close's page: %+v", p)

+ }

+ w.pass(t)

+ c := w.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "uk-box" || hostclaim.RequireReleasedMove(w.id)(c) != nil {

+ t.Fatalf("not released under the target: %+v", c.Op)

+ }

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again: %+v", w.paged(t))

+ }

+// TestRecordFirstCloseCrashIsFinishedByThePass (ruling M4): a box closed the

+// record and crashed before the claim's release. The record carries

+// LastMove and no move; the claim is still in the move's closing op. The

+// console's pass releases from LastMove, under the executor the outcome

+// names, with no grace (the release is the record's decision).

+func TestRecordFirstCloseCrashIsFinishedByThePass(t *testing.T) {

+ for _, c := range []struct {

+ name, outcome, exec string

+ setup func(*testing.T, *movePassWorld)

+ }{

+ {"committed", "done", "uk-box", func(t *testing.T, w *movePassWorld) {

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseCommitted, nil })

+ }},

+ {"rolled back", "rolled_back", "ie-box", func(t *testing.T, w *movePassWorld) {

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Op.Exec, c.Obligations = hostclaim.KindRestoring, hostclaim.PhaseOpened, "ie-box", nil

+ })

+ }},

+ } {

+ t.Run(c.name, func(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-crash-"+strings.ReplaceAll(c.name, " ", "-"))

+ c.setup(t, w)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ if c.outcome == "rolled_back" {

+ r.Region = "ie"

+ }

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: c.outcome, At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != c.exec || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+ }

+ t.Run("reverted", func(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-crash-reverted")

+ w.setClaim(t, func(c *hostclaim.Claim) {

+ c.Op.Kind, c.Op.Phase, c.Obligations = hostclaim.KindReverting, hostclaim.PhaseOpened, nil

+ if c.Op.Journal == nil {

+ c.Op.Journal = map[string]string{}

+ }

+ c.Op.Journal[hostclaim.JournalRevPhase] = hostclaim.PhaseOpened

+ })

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: "rolled_back", At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+ t.Run("aborted", func(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-crash-aborted")

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Phase, c.Obligations = hostclaim.PhaseAborting, nil })

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Move, r.LastMove = nil, &provision.MoveOutcome{ID: w.id, Outcome: "aborted", At: time.Now().UTC()}

+ })

+ w.pass(t)

+ got := w.claim(t)

+ if got.Op.Kind != hostclaim.KindServing || got.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(w.id)(got) != nil {

+ t.Fatalf("not released: %+v", got.Op)

+ }

+ })

+// TestRecoveryPassAbortsAfterSixHours (K18): a move still preparing

+// moves.AbortAfter after its request is aborted through ConsoleMove; before

+// that the pass pages once at an hour and writes nothing.

+func TestRecoveryPassAbortsAfterSixHours(t *testing.T) {

+ w := startedMovePass(t, "code-mvpass-abort")

+ claim := claimBytes(t, w.fx, "acme.go.tito.io")

+ requested := w.record(t).Move.RequestedAt

+ w.advance(30 * time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 0 {

+ t.Fatalf("paged at 30 minutes: %+v", w.paged(t))

+ }

+ w.advance(time.Hour)

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MovePreparing {

+ t.Fatalf("aborted early: %+v", m)

+ }

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Level != alerting.LevelWarning || !strings.Contains(p[0].Subject, "acme.go.tito.io") {

+ t.Fatalf("the hour's page: %+v", p)

+ }

+ w.mu.Lock()

+ w.clock = requested.Add(moves.AbortAfter + time.Minute)

+ w.mu.Unlock()

+ w.pass(t)

+ if m := w.record(t).Move; m.State != provision.MoveAborting {

+ t.Fatalf("not aborted at six hours: %+v", m)

+ }

+ if claimBytes(t, w.fx, "acme.go.tito.io") != claim {

+ t.Fatal("the abort wrote the claim (the source's abort takes it to aborting)")

+ }

+// TestConsolePagesThroughAlerting (§13, M14): the pass pages through

+// alerting.Page on the armed sinks. A rollback waiting past its release

+// deadline is an incident, paged once an hour and no more often, and the

+// rate limit is durable: it is kept in the console's store (alert_seen), so

+// a restarted console does not page again inside the hour since the

+// deadline.

+func TestConsolePagesThroughAlerting(t *testing.T) {

+ w := landingMovePass(t, "code-mvpass-pages")

+ rb, at := newRollbackID(t), time.Now().UTC().Add(-time.Hour)

+ deadline := at.Add(15 * time.Minute)

+ j17SetRecord(t, w.fx, func(r *provision.Record) {

+ r.Region = "ie"

+ r.Move.State = provision.MoveRollingBack

+ r.Move.Rollback = &provision.MoveRollback{ID: rb, RequestedAt: &at, Reason: provision.RollbackLandingDeadline, TargetPhase: provision.RbTargetReleasing,

+ SourcePhase: provision.RbSourceWaiting, ReleaseDeadline: &deadline}

+ })

+ w.setClaim(t, func(c *hostclaim.Claim) { c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseClaimed })

+ w.pass(t)

+ p := w.paged(t)

+ if len(p) != 1 || p[0].Kind != "page" || p[0].Level != alerting.LevelIncident || !strings.Contains(p[0].Subject, "uk") || !strings.Contains(p[0].Body, w.id) {

+ t.Fatalf("the release deadline's page: %+v", p)

+ }

+ // A restarted console: the in-memory state is gone, the store is not.

+ // The deadline passed 45 minutes ago; ten minutes on is the same hour

+ // since the deadline.

+ w.fx.srv.moveCloseSeen = nil

+ w.advance(10 * time.Minute)

+ w.pass(t)

+ if len(w.paged(t)) != 1 {

+ t.Fatalf("paged again inside the hour: %+v", w.paged(t))

+ }

+ if seen, known, err := w.fx.st.AlertSeen(movePageBox, "release-deadline acme.go.tito.io "+w.id+" 0", noMailboxAddress); err != nil || !known || !seen {

+ t.Fatalf("the page's durable record: seen %v known %v err %v", seen, known, err)

+ }

+ w.advance(time.Hour)

+ w.pass(t)

+ if len(w.paged(t)) != 2 {

+ t.Fatalf("not paged again the next hour: %+v", w.paged(t))

+ }

diff --git a/internal/console/server.go b/internal/console/server.go

index 89eb5c44e..1e5c08d1d 100644

--- a/internal/console/server.go

+++ b/internal/console/server.go

@@ -828,9 +828,18 @@ type Server struct {

claimBackendObserver func(backendEvent) // fixture recorder only

claimEntries claimInventory

claimWorkers map[string]func(context.Context)

- dbHandleKey []byte

- cfg Config

- store *Store

+ // The console's move pass (movesrecovery.go): its clock (nil is

+ // time.Now; a TEST SEAM) and when each close was first seen due.

+ movesClock func() time.Time

+ moveCloseMu sync.Mutex

+ moveCloseSeen map[string]time.Time

+ // stampLostMoveTarget is hostops.StampLostMoveTargetParked over the

+ // conditional record store, bound by ConfigureClaimRuntime with the

+ // moves switch on; nil otherwise.

+ stampLostMoveTarget func(ctx context.Context, host string, now time.Time) error

+ dbHandleKey []byte

+ cfg Config

+ store *Store

// appRegionOf and beforeClaimInsert are TEST SEAMS (nil in

// production): a store failure on the app-region read (C20), and a

// competing insert landing after the region was resolved.

diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt

index 96aaeeb23..8e7d07046 100644

--- a/internal/console/testdata/claim-authority-sites.txt

+++ b/internal/console/testdata/claim-authority-sites.txt

@@ -92,6 +92,7 @@ claim_runtime.go|ConfigureClaimRuntime|hostops.NewControlWriter(raw, s.cfg.Relea

claim_runtime.go|ConfigureClaimRuntime|hostops.NewObservedMutationBoundary(true, records, check, s.claimMutationObserver)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewProductionMutationCheck(raw, control)

claim_runtime.go|ConfigureClaimRuntime|hostops.NewVersionedBoundary(true, s.cfg.DBVersions, raw, control, s.cfg.Bucket, dataBuckets, s.claimMutationObserver)

+claim_runtime.go|ConfigureClaimRuntime|hostops.StampLostMoveTargetParked(ctx, records, host, now)

claim_runtime.go|sweepOneClaimAdmission|c.Control.RecoverSQLTransaction(ctx, app.Account, app.Name, appID, id, s.store)

claim_sql_auto.go|withAutoAppSQL|s.WithAppMutation(pctx, []sqlGrant{permitGrant(pctx, s.claimBucket, acct.Sqid, scope)}, call)

claim_sql_grants.go|WithAppMutation|p.BindTransaction(database)

@@ -105,6 +106,13 @@ geoippublish.go|Commit|os.Open(st.path)

hostclaimbackfill.go|RunClaimRecoveryPass|a.Control.RunRecovery(ctx, s.runClaimRecoveryPassBody)

hostwakereset.go|registerWakeRetryReset|a.Control.RegisterStart(ctx, h.Account, h.App, reg.AppID, id, reg, registeredAt)

hostwakereset.go|registerWakeRetryReset|start.FinishWakeRetryReset(cleanup)

+movesrecovery.go|movesRecovery|a.Control.ConsoleMove(ctx, co, hostops.ConsoleMoveAbort)

+movesrecovery.go|movesRecovery|g.Claim(ctx, d.Claim)

+movesrecovery.go|movesRecovery|g.Convert(ctx, *in.Fence)

+movesrecovery.go|movesRecovery|g.Record(ctx, d.Record)

+movesrecovery.go|movesRecovery|g.ResolveStart(ctx)

+movesrecovery.go|movesRecovery|s.stampLostMoveTarget(ctx, o.Host, now)

scanpass.go|storeLister|objstore.S3ClientFor(ctx, p.Region, p.Endpoint)

store.go|Open|sql.Open("sqlite", path)

store_gate.go|Connect|k.g.d.Open(k.g.dsn)