da76647be · carlos/platform

diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go

index 4803a1430..11de97d30 100644

--- a/approved_moves_copy_test.go

+++ b/approved_moves_copy_test.go

@@ -73,6 +73,7 @@ var movesCopyIDs = []string{

"MV-N182", "MV-N183", "MV-N184", "MV-N185", "MV-N186",

"MV-N300", "MV-N301", "MV-N302", "MV-N303", "MV-N304", "MV-N305", "MV-N306",

"MV-N307", "MV-N308", "MV-N309", "MV-N310", "MV-N311", "MV-N312",

+ "MV-N313", "MV-N314",

}

// copyRow is one row of the record. The File cell is the file the sentence is

@@ -570,6 +571,7 @@ var movesScanFiles = []string{

"internal/host/movesfloor.go",

"internal/host/movesfloor_linux.go",

"internal/host/movesfloor_other.go",

+ "internal/host/movesabort.go",

"internal/host/movesarchive.go",

"internal/host/movescut.go",

"internal/host/movesfinal.go",

@@ -610,6 +612,7 @@ var movesScanFiles = []string{

"internal/lease/mirror.go",

"internal/lease/staging.go",

"internal/lease/versioned.go",

+ "internal/moves/abort.go",

"internal/moves/archive/archive.go",

"internal/moves/archive/extract.go",

"internal/moves/archive/limits.go",

@@ -629,6 +632,7 @@ var movesScanFiles = []string{

"internal/moves/journal.go",

"internal/moves/lease.go",

"internal/moves/ltxbuild.go",

+ "internal/moves/parkscope.go",

"internal/moves/probe.go",

"internal/moves/readiness.go",

"internal/moves/returnlane.go",

@@ -725,6 +729,7 @@ var movesReturnedSentences = map[string]map[string]bool{

"s1OverCap": true, "s1LongPath": true, "s1NoAnswer": true, "s1ManyBoxes": true, "oneBoxRefusal": true},

"internal/host/movesreport.go": {"t1Below": true, "t3Differs": true, "t3Shadowed": true, "t4Failed": true,

"t6NoRoom": true, "t8Count": true, "t12NotQuiet": true, "replicaReason": true},

+ "internal/host/movesrollback.go": {"stuckPage": true},

}

// movesLiteralFuncs are functions whose every string literal is a sentence

diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

index b56d80ec7..c908fef05 100644

--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md

@@ -263,7 +263,12 @@ Columns: id, file, text, status.

## Plan 5

-(appended by Plan 5 Task 8)

+(each task's rows in the task that writes them; the rest appended by Plan 5 Task 8. Task 0's stuck page continues Plan 4's MV-N3xx block, ruling K11)

+

+|---|---|---|---|

## Plan 6

diff --git a/docs/superpowers/specs/2026-10-02-move-any-app-design.md b/docs/superpowers/specs/2026-10-02-move-any-app-design.md

index 26471345f..c8a59c34d 100644

--- a/docs/superpowers/specs/2026-10-02-move-any-app-design.md

+++ b/docs/superpowers/specs/2026-10-02-move-any-app-design.md

@@ -3063,7 +3063,11 @@ an unreachable target never needs a human.

Delete and prefix-scoped List on `moves/*` only), so it cannot park target

staging. Abort therefore adds a `park-staging` obligation **owned by the

target region's box**, not before the staging horizon of §7.3. The source

-parks its own snapshots. The claim stays `moving` until every obligation

+parks its own snapshots. (Plan 5 note, ruling K19: "parks" here means the

+source retires them from its own bucket, every version of

+`move-snapshots/<host>/<op>/` through `DeleteVersion`; that prefix is outside

+the park scope grammar, and F3's retention of snapshots covers completed

+moves only.) The claim stays `moving` until every obligation

has acknowledged, then releases to `serving`. The model gives the source

the target's park and records an IAM violation on every abort.

@@ -6328,7 +6332,7 @@ reads "cold" until #234 lands.

|---|---|---|---|

diff --git a/internal/alerting/alerting.go b/internal/alerting/alerting.go

new file mode 100644

index 000000000..782cc052b

--- /dev/null

+++ b/internal/alerting/alerting.go

@@ -0,0 +1,245 @@

+// Package alerting is the operator alert delivery the agent and the console

+// can share: a webhook, an SNS topic, either, both or neither (Sinks), and

+// Page, the paging primitive the moves code calls (moves spec §13).

+//

+// Delivery is best effort. A sink that fails is counted through its failure

+// hook and its error returned; the other sink still runs. Nothing here

+// retries beyond what the AWS SDK does inside one publish.

+package alerting

+

+import (

+ "bytes"

+ "context"

+ "encoding/json"

+ "errors"

+ "fmt"

+ "log"

+ "net/http"

+ "regexp"

+ "strings"

+ "sync"

+ "sync/atomic"

+ "time"

+

+ "github.com/aws/aws-sdk-go-v2/aws"

+ "github.com/aws/aws-sdk-go-v2/aws/arn"

+ awsconfig "github.com/aws/aws-sdk-go-v2/config"

+ "github.com/aws/aws-sdk-go-v2/service/sns"

+ snstypes "github.com/aws/aws-sdk-go-v2/service/sns/types"

+)

+

+// Page levels (moves spec §13): an incident wants someone now, a warning

+// wants someone today.

+const (

+ LevelIncident = "incident"

+ LevelWarning = "warning"

+)

+

+// Publisher is the one SNS call this package makes.

+type Publisher interface {

+ Publish(context.Context, *sns.PublishInput, ...func(*sns.Options)) (*sns.PublishOutput, error)

+}

+

+// PublishTimeout bounds one SNS publish, so a black-holed endpoint cannot

+// hold the caller for the SDK's own default retries.

+const PublishTimeout = 10 * time.Second

+

+// subjectLimit is SNS's: a subject is under 100 characters.

+const subjectLimit = 100

+

+// Sinks is one process's delivery. An empty WebhookURL or a nil Post

+// switches the webhook off; an empty TopicARN or a nil Pub switches SNS off.

+// The failure hooks count a failed send; nil counts nothing.

+type Sinks struct {

+ WebhookURL string

+ Post func(url string, body []byte) error

+ TopicARN string

+ Pub Publisher

+

+ WebhookFailed func()

+ SNSFailed func()

+}

+

+// HasWebhook and HasSNS say whether that sink is on. Safe on nil.

+func (s *Sinks) HasWebhook() bool { return s != nil && s.WebhookURL != "" && s.Post != nil }

+func (s *Sinks) HasSNS() bool { return s != nil && s.TopicARN != "" && s.Pub != nil }

+

+// PostWebhook posts body to the webhook. A failure is counted and returned.

+// The caller checks HasWebhook first.

+func (s *Sinks) PostWebhook(body []byte) error {

+ if err := s.Post(s.WebhookURL, body); err != nil {

+ if s.WebhookFailed != nil {

+ s.WebhookFailed()

+ }

+ return err

+ }

+ return nil

+}

+

+// PublishSNS publishes a subject and message to the topic, bounded by

+// PublishTimeout. A failure is counted and returned. The caller checks

+// HasSNS first.

+func (s *Sinks) PublishSNS(ctx context.Context, subject, message string) error {

+ return s.publish(ctx, &sns.PublishInput{TopicArn: &s.TopicARN, Subject: &subject, Message: &message})

+}

+

+func (s *Sinks) publish(ctx context.Context, in *sns.PublishInput) error {

+ pctx, cancel := context.WithTimeout(ctx, PublishTimeout)

+ defer cancel()

+ if _, err := s.Pub.Publish(pctx, in); err != nil {

+ if s.SNSFailed != nil {

+ s.SNSFailed()

+ }

+ return err

+ }

+ return nil

+}

+

+// pagePayload is a page on the webhook.

+type pagePayload struct {

+ Kind string `json:"kind"`

+ Level string `json:"level"`

+ Subject string `json:"subject"`

+ Body string `json:"body"`

+}

+

+// Page sends one page to every configured sink: the webhook gets

+// {"kind":"page","level","subject","body"}, SNS the subject (made to fit

+// SNS's rules) and the body, with the level as a message attribute so a

+// subscription can filter on it. The caller's subject and body are the

+// whole message: no URL, ARN or credential is added here, and the caller

+// must not put a secret or a tenant path in either. With no sink it logs

+// once (until the next Arm) and returns nil. A failed sink is counted, and

+// every failure is returned joined.

+func (s *Sinks) Page(ctx context.Context, level, subject, body string) error {

+ if !s.HasWebhook() && !s.HasSNS() {

+ if !noSinkLogged.Swap(true) {

+ log.Printf("alerting: page not delivered, no sink configured (set CARLOS_ALERT_SNS_TOPIC_ARN or CARLOS_ALERT_WEBHOOK): %s", fitSubject(subject))

+ }

+ return nil

+ }

+ var errs []error

+ if s.HasWebhook() {

+ b, err := json.Marshal(pagePayload{Kind: "page", Level: level, Subject: subject, Body: body})

+ if err == nil {

+ err = s.PostWebhook(b)

+ }

+ if err != nil {

+ errs = append(errs, fmt.Errorf("webhook: %w", err))

+ }

+ if s.HasSNS() {

+ subj := fitSubject(subject)

+ in := &sns.PublishInput{TopicArn: &s.TopicARN, Subject: &subj, Message: &body,

+ MessageAttributes: map[string]snstypes.MessageAttributeValue{"level": {DataType: aws.String("String"), StringValue: aws.String(level)}}}

+ if err := s.publish(ctx, in); err != nil {

+ errs = append(errs, fmt.Errorf("sns: %w", err))

+ }

+ return errors.Join(errs...)

+}

+

+// fitSubject makes a subject SNS accepts: printable ASCII, no line break,

+// under 100 characters. A line break becomes a space and anything else

+// outside printable ASCII a question mark.

+func fitSubject(s string) string {

+ out := make([]byte, 0, len(s))

+ for _, r := range s {

+ switch {

+ case r == '\n' || r == '\r' || r == '\t':

+ if n := len(out); n == 0 || out[n-1] != ' ' {

+ out = append(out, ' ')

+ }

+ case r < 0x20 || r > 0x7e:

+ out = append(out, '?')

+ default:

+ out = append(out, byte(r))

+ }

+ if len(out) >= subjectLimit-1 {

+ break

+ }

+ return string(out)

+}

+

+// The armed sinks: the process's own, set once its sinks are built (Arm).

+var (

+ armedMu sync.RWMutex

+ armed *Sinks

+ noSinkLogged atomic.Bool

+)

+

+// Arm makes s the sinks Page delivers on. nil means none. Arming starts a

+// fresh "no sink" notice.

+func Arm(s *Sinks) {

+ armedMu.Lock()

+ armed = s

+ armedMu.Unlock()

+ noSinkLogged.Store(false)

+}

+

+// Page pages an operator on the armed sinks (Sinks.Page).

+func Page(ctx context.Context, level, subject, body string) error {

+ armedMu.RLock()

+ s := armed

+ armedMu.RUnlock()

+ return s.Page(ctx, level, subject, body)

+}

+

+// PostJSON is the production webhook post: a 5 second POST with a JSON

+// content type. Any non-2xx status is an error.

+func PostJSON(url string, body []byte) error {

+ client := &http.Client{Timeout: 5 * time.Second}

+ resp, err := client.Post(url, "application/json", bytes.NewReader(body))

+ if err != nil {

+ return err

+ }

+ defer resp.Body.Close()

+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {

+ return fmt.Errorf("host: webhook: non-2xx status %s", resp.Status)

+ }

+ return nil

+}

+

+// topicARNPattern is the whole shape of a standard SNS topic ARN: partition,

+// region, a 12 digit account, and a topic name that is only letters, digits,

+// hyphen and underscore. The name has no colon, so a subscription ARN

+// (topic ARN plus ":<subscription-id>") does not match, and no dot, so FIFO

+// topics (which need a message group id this sink does not send) do not

+// either.

+var topicARNPattern = regexp.MustCompile(`^arn:aws(-cn|-us-gov)?:sns:([a-z0-9-]+):[0-9]{12}:[A-Za-z0-9_-]{1,256}$`)

+

+// ParseTopicARN checks a CARLOS_ALERT_SNS_TOPIC_ARN value and returns the

+// region the topic lives in. SNS is regional and a topic is reached only

+// through its own region's endpoint, so the region is read from the ARN and

+// not from AWS_REGION (host.env pins that to the deployment bucket's region

+// on every flagship box, which would aim three of the four at the wrong

+// place).

+func ParseTopicARN(s string) (region string, err error) {

+ s = strings.TrimSpace(s)

+ if s == "" {

+ return "", errors.New("empty")

+ }

+ a, err := arn.Parse(s)

+ if err != nil {

+ return "", err

+ }

+ if a.Service != "sns" {

+ return "", fmt.Errorf("service %q, want sns", a.Service)

+ }

+ if !topicARNPattern.MatchString(s) {

+ return "", errors.New("not a topic arn (want arn:aws:sns:<region>:<12 digit account>:<topic name>)")

+ }

+ return a.Region, nil

+}

+

+// NewSNSClient is the production publisher: the default credential chain

+// (a box's instance role, the console's task role), aimed at the topic's

+// own region.

+func NewSNSClient(ctx context.Context, region string) (Publisher, error) {

+ cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))

+ if err != nil {

+ return nil, err

+ }

+ return sns.NewFromConfig(cfg), nil

+}

diff --git a/internal/alerting/alerting_test.go b/internal/alerting/alerting_test.go

new file mode 100644

index 000000000..90df95149

--- /dev/null

+++ b/internal/alerting/alerting_test.go

@@ -0,0 +1,208 @@

+package alerting

+

+import (

+ "bytes"

+ "context"

+ "encoding/json"

+ "errors"

+ "log"

+ "strings"

+ "testing"

+

+ "github.com/aws/aws-sdk-go-v2/service/sns"

+)

+

+// fakeSNS records every Publish and can be told to fail.

+type fakeSNS struct {

+ in []*sns.PublishInput

+ err error

+}

+

+func (f *fakeSNS) Publish(_ context.Context, in *sns.PublishInput, _ ...func(*sns.Options)) (*sns.PublishOutput, error) {

+ f.in = append(f.in, in)

+ if f.err != nil {

+ return nil, f.err

+ }

+ return &sns.PublishOutput{}, nil

+}

+

+const testTopicARN = "arn:aws:sns:eu-west-1:271376211898:carlos-flagship-ie-alerts"

+

+type posted struct {

+ url string

+ body []byte

+}

+

+func poster(into *[]posted, err error) func(string, []byte) error {

+ return func(url string, body []byte) error {

+ *into = append(*into, posted{url, body})

+ return err

+ }

+}

+

+// captureLog sends the standard logger to a buffer for the test.

+func captureLog(t *testing.T) *bytes.Buffer {

+ t.Helper()

+ var buf bytes.Buffer

+ prev, flags := log.Writer(), log.Flags()

+ log.SetOutput(&buf)

+ log.SetFlags(0)

+ t.Cleanup(func() { log.SetOutput(prev); log.SetFlags(flags) })

+ return &buf

+}

+

+func TestPageDeliversOnBothSinks(t *testing.T) {

+ var posts []posted

+ pub := &fakeSNS{}

+ s := &Sinks{WebhookURL: "http://example.test/hook", Post: poster(&posts, nil), TopicARN: testTopicARN, Pub: pub}

+ if err := s.Page(context.Background(), LevelIncident, "carlos move stuck: shop.example", "the body.\nsecond line."); err != nil {

+ t.Fatal(err)

+ }

+ if len(posts) != 1 || posts[0].url != "http://example.test/hook" {

+ t.Fatalf("webhook posts: %+v", posts)

+ }

+ var got map[string]string

+ if err := json.Unmarshal(posts[0].body, &got); err != nil {

+ t.Fatal(err)

+ }

+ want := map[string]string{"kind": "page", "level": "incident", "subject": "carlos move stuck: shop.example", "body": "the body.\nsecond line."}

+ if len(got) != len(want) {

+ t.Fatalf("webhook payload %s", posts[0].body)

+ }

+ for k, v := range want {

+ if got[k] != v {

+ t.Fatalf("webhook payload %s: %s = %q, want %q", posts[0].body, k, got[k], v)

+ }

+ if len(pub.in) != 1 {

+ t.Fatalf("publishes = %d, want 1", len(pub.in))

+ }

+ in := pub.in[0]

+ if *in.TopicArn != testTopicARN || *in.Subject != "carlos move stuck: shop.example" || *in.Message != "the body.\nsecond line." {

+ t.Fatalf("publish: %q %q %q", *in.TopicArn, *in.Subject, *in.Message)

+ }

+ if a, ok := in.MessageAttributes["level"]; !ok || a.StringValue == nil || *a.StringValue != "incident" || *a.DataType != "String" {

+ t.Fatalf("the level attribute: %+v", in.MessageAttributes)

+ }

+}

+

+// SNS takes a subject of under 100 printable ASCII characters with no line

+// break; a page's subject is made to fit rather than refused.

+func TestPageSubjectFitsSNS(t *testing.T) {

+ pub := &fakeSNS{}

+ s := &Sinks{TopicARN: testTopicARN, Pub: pub}

+ long := "carlos move stuck:\n" + strings.Repeat("x", 120) + "é"

+ if err := s.Page(context.Background(), LevelWarning, long, "body"); err != nil {

+ t.Fatal(err)

+ }

+ got := *pub.in[0].Subject

+ if len(got) >= 100 || strings.ContainsAny(got, "\r\n") || !strings.HasPrefix(got, "carlos move stuck: x") {

+ t.Fatalf("subject %q (%d)", got, len(got))

+ }

+ for _, r := range got {

+ if r < 0x20 || r > 0x7e {

+ t.Fatalf("subject %q has %q", got, r)

+ }

+}

+

+// A failing sink is counted and returned, and never stops the other one.

+func TestPageFailureIsCountedAndReturned(t *testing.T) {

+ var posts []posted

+ pub := &fakeSNS{err: errors.New("AccessDenied")}

+ var webhookFailed, snsFailed int

+ s := &Sinks{WebhookURL: "http://example.test/hook", Post: poster(&posts, errors.New("boom")), TopicARN: testTopicARN, Pub: pub,

+ WebhookFailed: func() { webhookFailed++ }, SNSFailed: func() { snsFailed++ }}

+ err := s.Page(context.Background(), LevelIncident, "s", "b")

+ if err == nil || !strings.Contains(err.Error(), "boom") || !strings.Contains(err.Error(), "AccessDenied") {

+ t.Fatalf("err = %v, want both failures", err)

+ }

+ if len(posts) != 1 || len(pub.in) != 1 || webhookFailed != 1 || snsFailed != 1 {

+ t.Fatalf("posts %d publishes %d failures %d/%d", len(posts), len(pub.in), webhookFailed, snsFailed)

+ }

+}

+

+// With no sink configured a page is not an error: it says so once in the

+// journal and returns nil, as the alert delivery does.

+func TestPageWithNoSinksLogsOnce(t *testing.T) {

+ for _, c := range []struct {

+ name string

+ sinks *Sinks

+ }{{"nil", nil}, {"empty", &Sinks{}}, {"a webhook with no poster", &Sinks{WebhookURL: "http://example.test/hook"}}} {

+ t.Run(c.name, func(t *testing.T) {

+ buf := captureLog(t)

+ Arm(c.sinks)

+ t.Cleanup(func() { Arm(nil) })

+ for range 3 {

+ if err := Page(context.Background(), LevelIncident, "s", "b"); err != nil {

+ t.Fatal(err)

+ }

+ if n := strings.Count(buf.String(), "no sink configured"); n != 1 {

+ t.Fatalf("logged %d times, want once:\n%s", n, buf)

+ }

+ // Arming again starts a fresh notice.

+ Arm(c.sinks)

+ if err := Page(context.Background(), LevelIncident, "s", "b"); err != nil {

+ t.Fatal(err)

+ }

+ if n := strings.Count(buf.String(), "no sink configured"); n != 2 {

+ t.Fatalf("after re-arming: logged %d times, want twice:\n%s", n, buf)

+ }

+ })

+ }

+}

+

+func TestPageUsesTheArmedSinks(t *testing.T) {

+ var posts []posted

+ Arm(&Sinks{WebhookURL: "http://example.test/hook", Post: poster(&posts, nil)})

+ t.Cleanup(func() { Arm(nil) })

+ if err := Page(context.Background(), LevelWarning, "subject", "body"); err != nil {

+ t.Fatal(err)

+ }

+ if len(posts) != 1 || !strings.Contains(string(posts[0].body), `"subject":"subject"`) {

+ t.Fatalf("posts: %+v", posts)

+ }

+}

+

+// The delivery the alert sinks share: a webhook post and an SNS publish,

+// each counted on failure.

+func TestSinksPostAndPublish(t *testing.T) {

+ var posts []posted

+ pub := &fakeSNS{}

+ var webhookFailed int

+ s := &Sinks{WebhookURL: "http://example.test/hook", Post: poster(&posts, errors.New("boom")), TopicARN: testTopicARN, Pub: pub,

+ WebhookFailed: func() { webhookFailed++ }}

+ if !s.HasWebhook() || !s.HasSNS() {

+ t.Fatal("both sinks are configured")

+ }

+ if err := s.PostWebhook([]byte(`{}`)); err == nil || webhookFailed != 1 {

+ t.Fatalf("post: %v, %d failures", err, webhookFailed)

+ }

+ if err := s.PublishSNS(context.Background(), "subj", "msg"); err != nil || len(pub.in) != 1 || *pub.in[0].Subject != "subj" || pub.in[0].MessageAttributes != nil {

+ t.Fatalf("publish: %v %+v", err, pub.in)

+ }

+ var none *Sinks

+ if none.HasWebhook() || none.HasSNS() {

+ t.Fatal("a nil sink set has no sinks")

+ }

+}

+

+func TestParseTopicARN(t *testing.T) {

+ for _, c := range []struct {

+ in, region string

+ ok bool

+ }{

+ {testTopicARN, "eu-west-1", true},

+ {" " + testTopicARN + " ", "eu-west-1", true},

+ {"", "", false},

+ {testTopicARN + ":sub-id", "", false},

+ {"arn:aws:sqs:eu-west-1:271376211898:q", "", false},

+ {"arn:aws:sns:eu-west-1:271376211898:topic.fifo", "", false},

+ } {

+ region, err := ParseTopicARN(c.in)

+ if (err == nil) != c.ok || region != c.region {

+ t.Errorf("ParseTopicARN(%q) = %q, %v", c.in, region, err)

+ }

+}

diff --git a/internal/console/apidbclaim.go b/internal/console/apidbclaim.go

index fbd685ead..d9fb9e8f0 100644

--- a/internal/console/apidbclaim.go

+++ b/internal/console/apidbclaim.go

@@ -139,6 +139,11 @@ func (s *Server) dbClaim(ctx context.Context, g dbRequest, rec provision.Record)

switch obs.Claim.Op.Kind {

case hostclaim.KindDeleting, hostclaim.KindParking, hostclaim.KindPurging:

return obs, errDBClaimClosed

+ case hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring, hostclaim.KindRelocating:

+ // Every move kind, in every phase, landing/committed and the

+ // opened phases included (moves spec §19.1): the door reopens at

+ // the close, when the claim is serving again.

+ return obs, errDBClaimClosed

}

if h.Location == "" {

return obs, errDBClaimClosed

diff --git a/internal/console/apidbclaim_test.go b/internal/console/apidbclaim_test.go

index 560258018..898057e82 100644

--- a/internal/console/apidbclaim_test.go

+++ b/internal/console/apidbclaim_test.go

@@ -384,3 +384,82 @@ func TestDoorPlanRangeReaderCloses(t *testing.T) {

})

}

+

+// TestDBDoorRefusesDuringMoves (moves spec §19.1, Review Focus 4, K14): the

+// db door refuses on every move kind, in every phase, including

+// landing/committed and the restoring and reverting opened phases; it

+// reopens at the close, when the claim is serving again. The refusal is a

+// 409 with Retry-After: 1, and the data bucket is never read.

+func TestDBDoorRefusesDuringMoves(t *testing.T) {

+ const move = "mv_abcdefghijklmnopqrstuvwxyz"

+ reloc := func(phase, exec string) hostclaim.Op {

+ return hostclaim.Op{Kind: hostclaim.KindRelocating, ID: move, Phase: phase, Exec: exec,

+ Intent: map[string]string{"region": "uk", "source_box": "ie-box", "target_box": "uk-box"}}

+ }

+ cases := []hostclaim.Op{

+ {Kind: hostclaim.KindMoving, ID: move, Phase: hostclaim.PhaseMoving, Exec: "ie-box"},

+ {Kind: hostclaim.KindMoving, ID: move, Phase: hostclaim.PhaseSealed},

+ {Kind: hostclaim.KindMoving, ID: move, Phase: hostclaim.PhaseAborting, Exec: "ie-box"},

+ {Kind: hostclaim.KindLanding, ID: move, Phase: "checking", Exec: "uk-box"},

+ {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseLandingProbing, Exec: "uk-box"},

+ {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseLandingProbed, Exec: "uk-box"},

+ {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseLandingCommitted, Exec: "uk-box"},

+ {Kind: hostclaim.KindReverting, ID: move, Exec: "ie-box", Journal: map[string]string{hostclaim.JournalRevPhase: hostclaim.RevPhaseEvidence}},

+ {Kind: hostclaim.KindReverting, ID: move, Exec: "ie-box", Journal: map[string]string{hostclaim.JournalRevPhase: hostclaim.RevPhaseVerify}},

+ {Kind: hostclaim.KindReverting, ID: move, Phase: hostclaim.PhaseProbing, Exec: "ie-box"},

+ {Kind: hostclaim.KindReverting, ID: move, Phase: hostclaim.PhaseOpened, Exec: "ie-box"},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseReleasing, Exec: "uk-box"},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseReleased},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseClaimed, Exec: "ie-box"},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseProbing, Exec: "ie-box"},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseProbed, Exec: "ie-box"},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseOpened, Exec: "ie-box"},

+ reloc("starting", "ie-box"),

+ reloc("releasing", "none"),

+ reloc("pinning", "none"),

+ reloc("installing", "uk-box"),

+ reloc("installed", "uk-box"),

+ reloc("probing", "uk-box"),

+ reloc("probe-failed", "uk-box"),

+ reloc("serving", "uk-box"),

+ }

+ kinds := map[string]bool{}

+ for _, op := range cases {

+ kinds[op.Kind] = true

+ name := op.Kind + "/" + op.Phase

+ if rp := op.Journal[hostclaim.JournalRevPhase]; rp != "" {

+ name += "rev_phase=" + rp

+ }

+ t.Run(name, func(t *testing.T) {

+ s, g, rec, m := doorPlanFixture(t, false)

+ m.FailOn = func(op, key string) error { t.Fatalf("a refusal reached the data bucket: %s %s", op, key); return nil }

+ c := cloneDoorClaim(g.claim.Claim)

+ c.Op = op

+ writeDoorClaim(t, s, dbRequest{host: rec.Host}, c)

+ _, err := s.dbClaim(t.Context(), g, rec)

+ if !errors.Is(err, errDBClaimClosed) {

+ t.Fatalf("%v, want the door closed", err)

+ }

+ w := httptest.NewRecorder()

+ s.dbClaimFailure(w, httptest.NewRequest(http.MethodGet, "/", nil), g, err)

+ if w.Code != http.StatusConflict || w.Header().Get("Retry-After") != "1" {

+ t.Fatalf("%d Retry-After %q, want 409 and 1", w.Code, w.Header().Get("Retry-After"))

+ }

+ })

+ }

+ for _, k := range []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring, hostclaim.KindRelocating} {

+ if !kinds[k] {

+ t.Errorf("the table has no %s row", k)

+ }

+ // The close: serving again, the door reopens.

+ t.Run("serving", func(t *testing.T) {

+ s, g, rec, _ := doorPlanFixture(t, false)

+ c := cloneDoorClaim(g.claim.Claim)

+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing}

+ writeDoorClaim(t, s, dbRequest{host: rec.Host}, c)

+ if _, err := s.dbClaim(t.Context(), g, rec); err != nil {

+ t.Fatalf("serving: %v", err)

+ }

+ })

+}

diff --git a/internal/host/agent.go b/internal/host/agent.go

index aa9ddafa5..39258dedb 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -246,6 +246,13 @@ func runAgent(args []string, buildClaims ClaimFactory) {

cgm = cgroup.Setup(log.Printf)

}

+ // Alert sinks and the operator paging hook (moves spec §13), built from

+ // the sink env alone and armed here, before armMovesPass and before any

+ // loop that runs a moves pass starts, so a stuck mark never finds paging

+ // unarmed. They do not wait on the metrics store: the metrics loop is

+ // handed this same instance, and a box whose store fails still pages.

+ alerts := armAgentAlerts(agentCtx, os.Getenv, &processAlertCounters, newSNSClient)

+

// Log shipping (`carlos logs`): one sink for every producer in this

// process, armed only when there is somewhere to ship to — a nil sink

// leaves every producer exactly as before (no tees, no pipes, no

@@ -596,7 +603,7 @@ func runAgent(args []string, buildClaims ClaimFactory) {

go steeringLoop()

// The sweep reads the registry only through this closure — the stamped

// bespoke units to sample beside the carlos- ones, re-read each tick.

- go metricsLoop(cgm.Root(), func() []string { return stampedUnitNames(reg) })

+ go metricsLoop(cgm.Root(), func() []string { return stampedUnitNames(reg) }, alerts)

go configSyncLoop(reg, svc, dd, claims.Writes)

go configGenLoop(reg, svc, dd, claims.Writes)

go socketAdvisoryLoop(reg, svc)

diff --git a/internal/host/agentalerts_test.go b/internal/host/agentalerts_test.go

new file mode 100644

index 000000000..149f3a49b

--- /dev/null

+++ b/internal/host/agentalerts_test.go

@@ -0,0 +1,161 @@

+package host

+

+import (

+ "context"

+ "encoding/json"

+ "go/ast"

+ "go/parser"

+ "go/token"

+ "io"

+ "net/http"

+ "net/http/httptest"

+ "os"

+ "path/filepath"

+ "sync"

+ "testing"

+

+ "github.com/carlosframework/platform/internal/provision"

+)

+

+// TestStuckMarkBeforeMetricsReachesTheSink (fix round 1, finding 1): the

+// agent arms paging at startup from the sink env alone (armAgentAlerts), so

+// a stuck mark reaches a configured webhook whether or not the metrics loop

+// has started, and when its store fails to initialise. The revert's stuck

+// mark runs with the production hook, PageOperator.

+func TestStuckMarkBeforeMetricsReachesTheSink(t *testing.T) {

+ for _, metrics := range []string{"never started", "store init failed"} {

+ t.Run(metrics, func(t *testing.T) {

+ var mu sync.Mutex

+ var bodies [][]byte

+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

+ b, _ := io.ReadAll(r.Body)

+ mu.Lock()

+ bodies = append(bodies, b)

+ mu.Unlock()

+ }))

+ defer srv.Close()

+ env := map[string]string{"CARLOS_ALERT_WEBHOOK": srv.URL}

+ sinks := armAgentAlerts(context.Background(), func(k string) string { return env[k] }, &alertCounters{}, (&fakeClientFactory{}).build)

+ t.Cleanup(func() { armPaging(nil) })

+

+ if metrics == "store init failed" {

+ // No bucket, and a fleet name that does not validate:

+ // openStatusStore fails and the loop returns at once.

+ t.Setenv("CARLOS_STATUS_BUCKET", "")

+ t.Setenv("CARLOS_DEPLOYMENT_BUCKET", "")

+ t.Setenv("CARLOS_FLEET", "not a fleet!")

+ if _, _, err := openStatusStore(context.Background(), os.Getenv); err == nil {

+ t.Fatal("fixture: the metrics store opened")

+ }

+ metricsLoop(t.TempDir(), nil, sinks)

+ }

+

+ h := newRevertHarness(t)

+ h.fromStopped(t)

+ h.arm(t)

+ h.cfg.Page = PageOperator

+ if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host, "upload.txt"), []byte("changed while stopped"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ if m := h.move(t); m.State != provision.MoveStuck {

+ t.Fatalf("not stuck: %+v", m)

+ }

+ mu.Lock()

+ defer mu.Unlock()

+ if len(bodies) != 1 {

+ t.Fatalf("webhook received %d posts, want the one page", len(bodies))

+ }

+ var p struct{ Kind, Level, Subject, Body string }

+ if err := json.Unmarshal(bodies[0], &p); err != nil {

+ t.Fatal(err)

+ }

+ if p.Kind != "page" || p.Level != "incident" || p.Subject != "carlos move stuck: "+p1Host || p.Body != stuckPageBody("ie-box", "source", provision.StuckBaselineMismatch) {

+ t.Fatalf("the page: %+v", p)

+ }

+ })

+ }

+}

+

+// TestAgentArmsPagingBeforeMovesConsumers (fix round 1, finding 1): in

+// runAgent the alert sinks are built and paging armed once, before

+// armMovesPass and before any loop that runs a moves pass starts; the

+// metrics loop is handed that same instance and neither builds sinks nor

+// arms paging itself.

+func TestAgentArmsPagingBeforeMovesConsumers(t *testing.T) {

+ fset := token.NewFileSet()

+ parse := func(name string) *ast.File {

+ f, err := parser.ParseFile(fset, name, nil, 0)

+ if err != nil {

+ t.Fatal(err)

+ }

+ return f

+ }

+ body := func(f *ast.File, name string) *ast.BlockStmt {

+ for _, d := range f.Decls {

+ if fd, ok := d.(*ast.FuncDecl); ok && fd.Recv == nil && fd.Name.Name == name {

+ return fd.Body

+ }

+ t.Fatalf("no func %s", name)

+ return nil

+ }

+ callee := func(c *ast.CallExpr) string {

+ if id, ok := c.Fun.(*ast.Ident); ok {

+ return id.Name

+ }

+ return ""

+ }

+ // first call position of each name in runAgent, and how many times.

+ at := map[string]token.Pos{}

+ count := map[string]int{}

+ var armed string // the variable armAgentAlerts' result is assigned to

+ var metricsArg ast.Expr

+ ast.Inspect(body(parse("agent.go"), "runAgent"), func(n ast.Node) bool {

+ switch n := n.(type) {

+ case *ast.AssignStmt:

+ if len(n.Rhs) == 1 && len(n.Lhs) == 1 {

+ if c, ok := n.Rhs[0].(*ast.CallExpr); ok && callee(c) == "armAgentAlerts" {

+ if id, ok := n.Lhs[0].(*ast.Ident); ok {

+ armed = id.Name

+ }

+ case *ast.CallExpr:

+ name := callee(n)

+ if name == "" {

+ return true

+ }

+ count[name]++

+ if _, ok := at[name]; !ok {

+ at[name] = n.Pos()

+ }

+ if name == "metricsLoop" && len(n.Args) == 3 {

+ metricsArg = n.Args[2]

+ }

+ return true

+ })

+ if count["armAgentAlerts"] != 1 || armed == "" {

+ t.Fatalf("runAgent calls armAgentAlerts %d times, assigned to %q; want once, kept", count["armAgentAlerts"], armed)

+ }

+ for _, later := range []string{"armMovesPass", "adoptLoop", "fastConvergeLoop", "metricsLoop"} {

+ if at[later] == token.NoPos {

+ t.Fatalf("runAgent no longer calls %s; update this test", later)

+ }

+ if at["armAgentAlerts"] > at[later] {

+ t.Errorf("paging is armed after %s starts (%s before %s)", later, fset.Position(at[later]), fset.Position(at["armAgentAlerts"]))

+ }

+ if id, ok := metricsArg.(*ast.Ident); !ok || id.Name != armed {

+ t.Errorf("metricsLoop is not handed the armed sinks %q", armed)

+ }

+ ast.Inspect(body(parse("metrics_loop.go"), "metricsLoop"), func(n ast.Node) bool {

+ if c, ok := n.(*ast.CallExpr); ok {

+ if name := callee(c); name == "buildAlertSinks" || name == "armPaging" || name == "armAgentAlerts" {

+ t.Errorf("metricsLoop calls %s; it must reuse the armed sinks", name)

+ }

+ return true

+ })

+}

diff --git a/internal/host/alertsink.go b/internal/host/alertsink.go

index bd5c081b9..2a1509634 100644

--- a/internal/host/alertsink.go

+++ b/internal/host/alertsink.go

@@ -3,19 +3,14 @@ package host

import (

"context"

"encoding/json"

- "errors"

"fmt"

"log"

"math"

- "regexp"

"strings"

"sync/atomic"

"time"

- "github.com/aws/aws-sdk-go-v2/aws/arn"

- awsconfig "github.com/aws/aws-sdk-go-v2/config"

- "github.com/aws/aws-sdk-go-v2/service/sns"

-

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/metrics"

"github.com/carlosframework/platform/internal/release"

"github.com/carlosframework/platform/internal/status"

@@ -44,15 +39,14 @@ import (

// write in alertTick repeats the whole transition next tick, delivery

// included.) The count is what makes a lost send visible, through

// status.AlertDelivery in the box snapshot.

+//

+// The delivery itself (the webhook post, the SNS publish and its timeout)

+// is internal/alerting's, shared with the console; this file formats the

+// box's alerts and counts their failures. The same sinks carry the

+// operator pages of the moves code (PageOperator, moves spec §13).

-// snsPublisher is the one SNS call this file makes.

-type snsPublisher interface {

- Publish(context.Context, *sns.PublishInput, ...func(*sns.Options)) (*sns.PublishOutput, error)

-}

-

-// alertPublishTimeout bounds one SNS publish, so a black-holed endpoint

-// cannot hold the 60 second sampler tick for the SDK's own default retries.

-const alertPublishTimeout = 10 * time.Second

+// snsPublisher is the one SNS call the sinks make.

+type snsPublisher = alerting.Publisher

// alertCounters counts failed sends since the process started, and holds the

// one fact the status loop cannot derive from the env: whether the SNS client

@@ -76,13 +70,11 @@ const (

// shared state, so this is the seam between them.

var processAlertCounters alertCounters

-// alertSinks is the configured delivery for one agent.

+// alertSinks is the configured delivery for one agent: alerting's sinks,

+// counting their failures in counts.

type alertSinks struct {

- webhookURL string

- post func(url string, body []byte) error

- topicARN string

- pub snsPublisher

- counts *alertCounters

+ sinks alerting.Sinks

+ counts *alertCounters

}

// newAlertSinks builds a sink set. An empty webhookURL or topicARN switches

@@ -92,7 +84,8 @@ func newAlertSinks(webhookURL string, post func(url string, body []byte) error,

if counts == nil {

counts = &processAlertCounters

}

- return &alertSinks{webhookURL: webhookURL, post: post, topicARN: topicARN, pub: pub, counts: counts}

+ return &alertSinks{counts: counts, sinks: alerting.Sinks{WebhookURL: webhookURL, Post: post, TopicARN: topicARN, Pub: pub,

+ WebhookFailed: func() { counts.webhook.Add(1) }, SNSFailed: func() { counts.sns.Add(1) }}}

}

// deliver sends one transition to every configured sink. Safe on a nil

@@ -101,38 +94,66 @@ func (a *alertSinks) deliver(ctx context.Context, rules metrics.Rules, at time.T

if a == nil {

return

}

- if a.webhookURL != "" && a.post != nil {

+ if a.sinks.HasWebhook() {

evBody, err := json.Marshal(ev)

if err != nil {

log.Printf("host: alert event %s: marshal: %v", ev.Rule, err)

- } else if err := a.post(a.webhookURL, evBody); err != nil {

- a.counts.webhook.Add(1)

+ } else if err := a.sinks.PostWebhook(evBody); err != nil {

log.Printf("host: alert webhook post (rule %s): %v", ev.Rule, err)

}

- if a.topicARN != "" && a.pub != nil {

+ if a.sinks.HasSNS() {

subject, message := formatAlert(rules, at, ev)

- pctx, cancel := context.WithTimeout(ctx, alertPublishTimeout)

- defer cancel()

- _, err := a.pub.Publish(pctx, &sns.PublishInput{

- TopicArn: &a.topicARN,

- Subject: &subject,

- Message: &message,

- })

- if err != nil {

- a.counts.sns.Add(1)

+ if err := a.sinks.PublishSNS(ctx, subject, message); err != nil {

log.Printf("host: alert sns publish (rule %s): %v", ev.Rule, err)

}

+// PageOperator pages an operator through this agent's sinks: the same

+// webhook and SNS topic deliver uses, its failures counted with theirs

+// (alerting.Sinks.Page). Safe on a nil receiver, which means "no sinks":

+// nil, and one line in the journal.

+func (a *alertSinks) PageOperator(ctx context.Context, level, subject, body string) error {

+ if a == nil {

+ return (*alerting.Sinks)(nil).Page(ctx, level, subject, body)

+ }

+ return a.sinks.Page(ctx, level, subject, body)

+}

+

+// PageOperator pages an operator on the sinks the metrics loop armed

+// (armPaging): the moves code's paging primitive (moves spec §13). Before

+// they are armed, or with none configured, it returns nil and says so once.

+func PageOperator(ctx context.Context, level, subject, body string) error {

+ return alerting.Page(ctx, level, subject, body)

+}

+

+// armAgentAlerts builds the agent's alert sinks from the env and arms

+// paging on them (armPaging). runAgent calls it once at startup, before

+// anything that can run a moves pass, and hands the same instance to the

+// metrics loop, so arming never waits on the metrics store.

+func armAgentAlerts(ctx context.Context, getenv func(string) string, counts *alertCounters, newClient snsClientFactory) *alertSinks {

+ sinks := buildAlertSinks(ctx, getenv, counts, newClient)

+ armPaging(sinks)

+ return sinks

+}

+

+// armPaging makes a the sinks PageOperator pages on. nil means none.

+func armPaging(a *alertSinks) {

+ if a == nil {

+ alerting.Arm(nil)

+ return

+ }

+ alerting.Arm(&a.sinks)

+}

+

// deliverRelease sends a D1 refusal to every configured sink. The webhook

// body is the record with kind "release_refused"; SNS gets a sentence.

func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) {

if a == nil {

return

}

- if a.webhookURL != "" && a.post != nil {

+ if a.sinks.HasWebhook() {

body, err := json.Marshal(struct {

Kind string `json:"kind"`

// The record's own Kind is shadowed by the envelope's "kind"

@@ -141,18 +162,14 @@ func (a *alertSinks) deliverRelease(ctx context.Context, r release.Refusal) {

release.Refusal

}{"release_refused", r.Kind, r})

if err == nil {

- if err := a.post(a.webhookURL, body); err != nil {

- a.counts.webhook.Add(1)

+ if err := a.sinks.PostWebhook(body); err != nil {

log.Printf("host: alert webhook post (release refusal %s): %v", r.Host, err)

}

- if a.topicARN != "" && a.pub != nil {

+ if a.sinks.HasSNS() {

subject, msg := formatReleaseAlert(r)

- pctx, cancel := context.WithTimeout(ctx, alertPublishTimeout)

- defer cancel()

- if _, err := a.pub.Publish(pctx, &sns.PublishInput{TopicArn: &a.topicARN, Subject: &subject, Message: &msg}); err != nil {

- a.counts.sns.Add(1)

+ if err := a.sinks.PublishSNS(ctx, subject, msg); err != nil {

log.Printf("host: alert sns publish (release refusal %s): %v", r.Host, err)

}

@@ -215,37 +232,9 @@ func recoveryError(r release.Refusal) string {

return "outcome not recorded"

}

-// topicARNPattern is the whole shape of a standard SNS topic ARN: partition,

-// region, a 12 digit account, and a topic name that is only letters, digits,

-// hyphen and underscore. The name has no colon, so a subscription ARN

-// (topic ARN plus ":<subscription-id>") does not match, and no dot, so FIFO

-// topics (which need a message group id this sink does not send) do not

-// either.

-var topicARNPattern = regexp.MustCompile(`^arn:aws(-cn|-us-gov)?:sns:([a-z0-9-]+):[0-9]{12}:[A-Za-z0-9_-]{1,256}$`)

-

// parseAlertTopicARN checks CARLOS_ALERT_SNS_TOPIC_ARN and returns the

-// region the topic lives in. SNS is regional and a topic is reached only

-// through its own region's endpoint, so the region is read from the ARN and

-// not from AWS_REGION (host.env pins that to the deployment bucket's region

-// on every flagship box, which would aim three of the four at the wrong

-// place).

-func parseAlertTopicARN(s string) (region string, err error) {

- s = strings.TrimSpace(s)

- if s == "" {

- return "", errors.New("empty")

- }

- a, err := arn.Parse(s)

- if err != nil {

- return "", err

- }

- if a.Service != "sns" {

- return "", fmt.Errorf("service %q, want sns", a.Service)

- }

- if !topicARNPattern.MatchString(s) {

- return "", errors.New("not a topic arn (want arn:aws:sns:<region>:<12 digit account>:<topic name>)")

- }

- return a.Region, nil

-}

+// region the topic lives in (alerting.ParseTopicARN).

+func parseAlertTopicARN(s string) (region string, err error) { return alerting.ParseTopicARN(s) }

// alertDeliveryProbe is the box's answer to "do my alert rules have anywhere

// to go". The webhook and the ARN's shape come from the same env

@@ -283,11 +272,7 @@ type snsClientFactory func(ctx context.Context, region string) (snsPublisher, er

// newSNSClient is the production factory: the default credential chain (the

// box's instance role), aimed at the topic's own region.

func newSNSClient(ctx context.Context, region string) (snsPublisher, error) {

- cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))

- if err != nil {

- return nil, err

- }

- return sns.NewFromConfig(cfg), nil

+ return alerting.NewSNSClient(ctx, region)

}

// buildAlertSinks reads the sink env and builds the sinks. A bad topic ARN

diff --git a/internal/host/alertsink_page_test.go b/internal/host/alertsink_page_test.go

new file mode 100644

index 000000000..97f4fe123

--- /dev/null

+++ b/internal/host/alertsink_page_test.go

@@ -0,0 +1,91 @@

+package host

+

+import (

+ "bytes"

+ "context"

+ "encoding/json"

+ "errors"

+ "log"

+ "strings"

+ "testing"

+

+ "github.com/carlosframework/platform/internal/alerting"

+)

+

+// TestPageOperatorUsesTheArmedSinks: the paging primitive goes out through

+// the same webhook (and SNS topic) the agent's alerts use, its failures are

+// counted where the alert failures are, and the package function pages on

+// whatever the metrics loop armed. With no sinks it returns nil and says so

+// once.

+func TestPageOperatorUsesTheArmedSinks(t *testing.T) {

+ type page struct{ Kind, Level, Subject, Body string }

+ var pages []page

+ var fail error

+ post := func(url string, body []byte) error {

+ var p page

+ if err := json.Unmarshal(body, &p); err != nil {

+ t.Fatalf("webhook body %s: %v", body, err)

+ }

+ if url != "http://example.test/hook" {

+ t.Fatalf("posted to %s", url)

+ }

+ pages = append(pages, p)

+ return fail

+ }

+ counts := &alertCounters{}

+ pub := &fakeSNS{}

+ sinks := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, counts)

+ ctx := context.Background()

+

+ if err := sinks.PageOperator(ctx, alerting.LevelIncident, "carlos move stuck: shop.example", "the body"); err != nil {

+ t.Fatal(err)

+ }

+ want := page{"page", "incident", "carlos move stuck: shop.example", "the body"}

+ if len(pages) != 1 || pages[0] != want {

+ t.Fatalf("pages %+v, want %+v", pages, want)

+ }

+ if len(pub.in) != 1 || deref(pub.in[0].Subject) != want.Subject || deref(pub.in[0].Message) != want.Body {

+ t.Fatalf("sns: %+v", pub.in)

+ }

+

+ // The package function pages on the armed sinks.

+ armPaging(sinks)

+ t.Cleanup(func() { armPaging(nil) })

+ if err := PageOperator(ctx, alerting.LevelWarning, "second", "body two"); err != nil {

+ t.Fatal(err)

+ }

+ if len(pages) != 2 || pages[1].Subject != "second" || pages[1].Level != "warning" {

+ t.Fatalf("pages %+v", pages)

+ }

+

+ // A failed send is returned and counted with the alert failures.

+ fail = errors.New("boom")

+ if err := PageOperator(ctx, alerting.LevelIncident, "third", "body"); err == nil {

+ t.Fatal("a failed webhook post returned nil")

+ }

+ if counts.webhook.Load() != 1 || counts.sns.Load() != 0 {

+ t.Fatalf("failures webhook=%d sns=%d", counts.webhook.Load(), counts.sns.Load())

+ }

+

+ // No sinks: nil, and one line in the journal however often it is asked.

+ var buf bytes.Buffer

+ prev := log.Writer()

+ log.SetOutput(&buf)

+ t.Cleanup(func() { log.SetOutput(prev) })

+ armPaging(nil)

+ var none *alertSinks

+ for range 2 {

+ if err := PageOperator(ctx, alerting.LevelIncident, "s", "b"); err != nil {

+ t.Fatal(err)

+ }

+ if err := none.PageOperator(ctx, alerting.LevelIncident, "s", "b"); err != nil {

+ t.Fatal(err)

+ }

+ if n := strings.Count(buf.String(), "no sink configured"); n != 1 {

+ t.Fatalf("no-sink notice logged %d times, want once:\n%s", n, buf.String())

+ }

+ if len(pages) != 3 {

+ t.Fatalf("an unarmed page reached the old sinks: %+v", pages)

+ }

+}

diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go

index 9e92c102d..2e0dad037 100644

--- a/internal/host/metrics_loop.go

+++ b/internal/host/metrics_loop.go

@@ -9,17 +9,15 @@

package host

import (

- "bytes"

"context"

"encoding/json"

"errors"

- "fmt"

"log"

- "net/http"

"os"

"path/filepath"

"time"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/metrics"

"github.com/carlosframework/platform/internal/release"

"github.com/carlosframework/platform/internal/status"

@@ -592,18 +590,7 @@ func currentAlertEtag(ctx context.Context, s release.Store, key string) (string,

// httpPostWebhook is the production webhook post func: a 5s-timeout POST

// with a JSON content type. Any non-2xx status is an error — the caller

// (alertTick) only logs it, but it still needs to know delivery failed.

-func httpPostWebhook(url string, body []byte) error {

- client := &http.Client{Timeout: 5 * time.Second}

- resp, err := client.Post(url, "application/json", bytes.NewReader(body))

- if err != nil {

- return err

- }

- defer resp.Body.Close()

- if resp.StatusCode < 200 || resp.StatusCode >= 300 {

- return fmt.Errorf("host: webhook: non-2xx status %s", resp.Status)

- }

- return nil

-}

+func httpPostWebhook(url string, body []byte) error { return alerting.PostJSON(url, body) }

// realSampleProbes wires sampleProbes to the actual OS/cgroup readers in

// metrics_probes.go and probes.go — the production seam metricsLoop uses,

@@ -758,7 +745,11 @@ func BoxLabel() string { return metricsBoxLabel(os.Getenv, readMachineID()) }

// re-read once per tick so a freshly stamped route is picked up within a

// minute. It is a func, not the registry handle, so this loop keeps its "no

// state but a bucket and /proc" shape; nil = none.

-func metricsLoop(execRoot string, stampedUnits func() []string) {

+//

+// sinks are the agent's alert sinks, built and armed for paging by runAgent

+// at startup (armAgentAlerts); the loop delivers its alerts on them and

+// never builds or arms its own. nil delivers nowhere.

+func metricsLoop(execRoot string, stampedUnits func() []string, sinks *alertSinks) {

// This loop is always on and shares a process with the agent that serves

// this box's traffic, so a panic in it — an unreadable cgroup shape, a

// nil no one expected — must not take the box down with it. Monitoring

@@ -802,7 +793,6 @@ func metricsLoop(execRoot string, stampedUnits func() []string) {

st := loadAlertState(statePath, box)

rules := metrics.RulesFromEnv(os.Getenv)

logRejectedAlertOverrides(os.Getenv)

- sinks := buildAlertSinks(ctx, os.Getenv, &processAlertCounters, newSNSClient)

log.Printf("agent: metrics loop armed (60s ticks, 5min flush), publishing to %s", target)

diff --git a/internal/host/movesabort.go b/internal/host/movesabort.go

new file mode 100644

index 000000000..7f1ecd3e1

--- /dev/null

+++ b/internal/host/movesabort.go

@@ -0,0 +1,288 @@

+package host

+

+import (

+ "context"

+ "errors"

+ "fmt"

+ "log"

+ "os"

+ "sort"

+ "time"

+

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/provision"

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// The abort (moves spec §7.11, joint N13; Plan 5 Task 1 with rulings K5, K6,

+// K7, K19 and K21), behind CARLOS_MOVES_PHASE1. It replaces Plan 1's interim

+// abort, which released the claim at once and left the staging lane in the

+// target bucket with no owner.

+//

+// Who retires what. The source cannot read the target's moves/* nor write

+// its parked/ (M17), so it never touches the lane: the lane is parked by the

+// target region's box. The source retires only its own state.

+//

+// The source, for a record aborting in its region (phase1MoveRound):

+//

+// 1. with the claim at this move's moving/"" and this box executing, it

+// takes the staging lease, so no staging writer is mid-round and none

+// can sign another request (K5);

+// 2. it journals abort_bound once: the later of now + SignatureLife +

+// DefaultSkew and any seal's bound. Every staging request is signed at a

+// check that leaves its whole signature life before the lease's

+// SafeUntil, and none is signed after this moment while the lease is

+// held, so no request can land after the bound. A replay reads the

+// journalled bound and never computes a later one (revertClaim's

+// pattern);

+// 3. it CASes hostclaim.Abort with moves.AbortScope(host, op, inc, bound):

+// moving/aborting, the lane's park-replica queued for the target region,

+// owned by it. Execution stays with the source, which serves throughout;

+// 4. only then it retires its own state: the move's snapshots in its own

+// bucket, every version through DeleteVersion (K19: they are outside the

+// park grammar, so "the source parks its own snapshots" means it

+// retires them from its own bucket), then the receipts and the ledger,

+// and last the op's journal directory (the bound is in the claim by

+// then). It runs on every pass while the claim is aborting, before the

+// release, and is a no-op once everything is gone: nothing infers from

+// a missing journal that it already ran, and any error (a listing, a

+// delete, the journal directory) ends the pass unreleased;

+// 5. it waits until no obligation under the op remains, then releases with

+// hostclaim.ReleaseAbort (K7: from moving/aborting only, so never before

+// the lane park was queued), and last clears the record with

+// provision.MoveGuardedAbortComplete (§7.5: the claim releases first,

+// the record clears second). A round that finds the claim already

+// serving and recording this move's release completes the record; any

+// other serving claim is left alone.

+//

+// The target region's box, for a record aborting whose To is its region:

+// movesParticipantRecord answers it as the target on the state alone (the

+// record never flipped, so the target has neither the row nor a journal)

+// and laneParkTarget runs the lane's park-replica through the park engine

+// over its own bucket until the engine acknowledges it after the horizon,

+// abort_bound + SignatureAge + RequestLimit. A late request found by the

+// engine's verification listing is parked as a further retirement event.

+//

+// A dead target region (K21). If no box of the target region runs the lane

+// park, the obligation stays owed, the claim stays moving/aborting and

+// nothing releases. That blocks a delete (and any other operation) of the

+// host until the region returns. Nothing needs to be done by hand; the

+// overdue page is Task 8's.

+//

+// Arming. The source's half needs only what a preparing round already has.

+// The target's lane park is unarmed in production, as the revert's is,

+// until Task 9 wires the park engine's leases (revertLaneArmed); the tests

+// inject it. Plan 1's DeletePrefix path stays for legacy records only

+// (legacyMoveRound); this path never deletes from the target bucket.

+

+// abortStepBound is the abort's journalled writers' horizon (K5).

+const abortStepBound = "abort_bound"

+

+// abFault is the abort's test seam after a side effect.

+func (w *movesWorker) abFault(point string) error {

+ if f := w.cfg.abFault; f != nil {

+ return f(point)

+ }

+ return nil

+}

+

+// abortSource is phase1MoveRound's MoveAborting arm.

+func (w *movesWorker) abortSource(ctx context.Context, s release.Store, rec provision.Record) error {

+ cfg := w.cfg

+ id := rec.Move.ID

+ if cfg.Box == "" {

+ return errors.New("the abort needs this box's label")

+ }

+ obs, err := hostclaim.Read(ctx, s, rec.Host)

+ if err != nil {

+ return fmt.Errorf("read the host claim: %w", err)

+ }

+ if obs.Absent {

+ w.notice(rec.Host+"#abort", "moves: %s: aborting move %s has no host claim; left for the console pass", rec.Host, id)

+ return nil

+ }

+ op := obs.Claim.Op

+ mine := op.Kind == hostclaim.KindMoving && op.ID == id

+ switch {

+ case mine && op.Exec != cfg.Box:

+ w.notice(rec.Host+"#abort", "moves: %s: aborting move %s: the claim's executor is %q, not this box; left for the console pass", rec.Host, id, op.Exec)

+ return nil

+ case mine && op.Phase == hostclaim.PhaseMoving:

+ ok, err := w.abortClaim(ctx, s, rec)

+ if err != nil || !ok {

+ return err

+ }

+ if obs, err = hostclaim.Read(ctx, s, rec.Host); err != nil {

+ return fmt.Errorf("read the host claim: %w", err)

+ }

+ if obs.Absent || obs.Claim.Op.ID != id || obs.Claim.Op.Phase != hostclaim.PhaseAborting {

+ return nil // another writer moved it; the next pass reads it again

+ }

+ fallthrough

+ case mine && op.Phase == hostclaim.PhaseAborting:

+ // Every pass, before the release: the retirement is idempotent, and

+ // a release never follows from inferring that it already ran (fix

+ // round 1). Any error, the journal directory's included, ends the

+ // pass with the claim still aborting.

+ if err := w.retireOwnMoveState(ctx, rec); err != nil {

+ return err

+ }

+ if released, err := w.releaseAbort(ctx, s, rec, obs); err != nil || !released {

+ return err

+ }

+ case hostclaim.RequireReleasedMove(id)(obs.Claim) == nil:

+ // This abort released the claim before a crash: finish. The own

+ // state went before the release; retiring it again is a no-op.

+ if err := w.retireOwnMoveState(ctx, rec); err != nil {

+ return err

+ }

+ default:

+ w.notice(rec.Host+"#abort", "moves: %s: aborting move %s: the claim is %s/%q (%s) and records no release of this move; left for the console pass", rec.Host, id, op.Kind, op.Phase, op.ID)

+ return nil

+ }

+ g := provision.MoveGuard{ID: id, Incarnation: rec.Move.Incarnation}

+ if _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, provision.MoveGuardedAbortComplete(g, cfg.Now())); err != nil {

+ if errors.Is(err, provision.ErrMoveConflict) {

+ log.Printf("moves: %s: abort of %s: the record moved on (%v)", rec.Host, id, err)

+ return nil

+ }

+ return err

+ }

+ nudgeSerial(ctx, s)

+ log.Printf("moves: %s: move %s aborted; the instance stays in %q", rec.Host, id, rec.Region)

+ return nil

+}

+

+// abortClaim is steps 1 to 3: the staging lease, the journalled bound and

+// the Abort CAS. ok is false when the pass should wait (another worker holds

+// the lease). Without a deployment bucket no round ever took the lease and

+// none staged, so there is no lease to take and the bound alone holds.

+func (w *movesWorker) abortClaim(ctx context.Context, s release.Store, rec provision.Record) (bool, error) {

+ cfg := w.cfg

+ host, id := rec.Host, rec.Move.ID

+ if cfg.StagingLease != nil {

+ _, rel, err := cfg.StagingLease(ctx, rec.Move.To, host)

+ if err != nil {

+ var he *lease.HeldError

+ if errors.As(err, &he) {

+ w.notice(host+"#abort", "moves: %s: aborting move %s waits: another worker holds the staging lease", host, id)

+ return false, nil

+ }

+ return false, fmt.Errorf("staging lease: %w", err)

+ }

+ defer func() {

+ rctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

+ defer cancel()

+ if err := rel(rctx); err != nil {

+ log.Printf("moves: %s: release the staging lease: %v", host, err)

+ }

+ }()

+ }

+ // Under the lease, the claim again: still this move's, before the seal.

+ obs, err := hostclaim.Read(ctx, s, host)

+ if err != nil {

+ return false, fmt.Errorf("read the host claim: %w", err)

+ }

+ if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindMoving || obs.Claim.Op.ID != id || obs.Claim.Op.Phase != hostclaim.PhaseMoving || obs.Claim.Op.Exec != cfg.Box {

+ return true, nil

+ }

+ if obs.Claim.Holder == nil {

+ return false, errors.New("the abort needs the claim's holder")

+ }

+ j, err := moves.ReadJournal(cfg.DataDir, host, id)

+ if err != nil {

+ return false, err

+ }

+ var bound time.Time

+ if raw, ok := j.Steps[abortStepBound]; ok {

+ if bound, err = time.Parse(time.RFC3339Nano, raw); err != nil {

+ bound = time.Time{}

+ }

+ if bound.IsZero() {

+ bound = cfg.Now().Add(moves.SignatureLife + moves.DefaultSkew).UTC()

+ if seal, err := journalledSeal(obs.Claim); err == nil && seal.Bound().After(bound) {

+ bound = seal.Bound().UTC()

+ }

+ j.Mark(abortStepBound, bound.Format(time.RFC3339Nano))

+ if err := j.Write(cfg.DataDir); err != nil {

+ return false, err

+ }

+ if err := w.abFault("after-abort-bound"); err != nil {

+ return false, err

+ }

+ scope := moves.AbortScope(host, id, obs.Claim.Holder.Incarnation, bound)

+ if _, err := hostclaim.Update(ctx, s, obs, hostclaim.Abort(id, rec.Move.To, scope, obligationID)); err != nil {

+ return false, fmt.Errorf("abort the claim: %w", err)

+ }

+ log.Printf("moves: %s: the claim aborts move %s; the staging lane in %s is queued for its park there", host, id, rec.Move.To)

+ if err := w.abFault("after-abort-claim"); err != nil {

+ return false, err

+ }

+ return true, nil

+}

+

+// releaseAbort is step 5: the release once no obligation of the op remains.

+// released is false while the lane park is still owed.

+func (w *movesWorker) releaseAbort(ctx context.Context, s release.Store, rec provision.Record, obs hostclaim.Observed) (bool, error) {

+ id := rec.Move.ID

+ for _, o := range obs.Claim.Obligations {

+ if o.Op == id {

+ w.notice(rec.Host+"#abort-wait", "moves: %s: aborting move %s waits for %s to park the staging lane (%s %s)", rec.Host, id, rec.Move.To, o.Kind, o.Region+o.Box)

+ return false, nil

+ }

+ if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseAbort(id, w.cfg.Box)); err != nil {

+ return false, fmt.Errorf("release the host claim: %w", err)

+ }

+ if err := w.abFault("after-release-abort"); err != nil {

+ return false, err

+ }

+ return true, nil

+}

+

+// retireOwnMoveState is step 4: the move's snapshots in this box's own

+// bucket, every version through DeleteVersion (K19), then its receipts and

+// ledger, and last its journal directory, which holds the bound. Idempotent.

+func (w *movesWorker) retireOwnMoveState(ctx context.Context, rec provision.Record) error {

+ cfg := w.cfg

+ if cfg.Versioned == nil {

+ return errors.New("the abort needs this box's versioned store to retire its snapshots")

+ }

+ vs, err := cfg.Versioned(ctx, cfg.OwnBucket, cfg.OwnBucketRegion)

+ if err != nil {

+ return err

+ }

+ prefix := moves.SnapshotsPrefix(rec.Host, rec.Move.ID)

+ vers, err := vs.ListVersions(ctx, cfg.OwnBucket, prefix)

+ if err != nil {

+ return fmt.Errorf("list the move's snapshots: %w", err)

+ }

+ for _, v := range vers {

+ if err := vs.DeleteVersion(ctx, cfg.OwnBucket, v.Key, v.VersionID); err != nil {

+ return fmt.Errorf("delete %s version %s: %w", v.Key, v.VersionID, err)

+ }

+ own, err := w.mirror(ctx, cfg.OwnBucketRegion)

+ if err != nil {

+ return err

+ }

+ keys := []string{moves.ReceiptsKey(rec.Host, rec.Move.ID), moves.LedgerKey(rec.Host, rec.Move.ID)}

+ sort.Strings(keys)

+ for _, k := range keys {

+ if err := own.Delete(ctx, cfg.OwnBucket, k); err != nil && !errors.Is(err, lease.ErrMirrorNotFound) {

+ return fmt.Errorf("delete %s: %w", k, err)

+ }

+ if err := w.abFault("after-abort-own-state"); err != nil {

+ return err

+ }

+ // The move's local directory: its journal and the archive kept for it.

+ if err := os.RemoveAll(moves.JournalDir(cfg.DataDir, rec.Host, rec.Move.ID)); err != nil {

+ return fmt.Errorf("remove the move's local directory: %w", err)

+ }

+ return w.abFault("after-abort-journal")

+}

diff --git a/internal/host/movesabort_test.go b/internal/host/movesabort_test.go

new file mode 100644

index 000000000..44b728d79

--- /dev/null

+++ b/internal/host/movesabort_test.go

@@ -0,0 +1,570 @@

+package host

+

+import (

+ "context"

+ "errors"

+ "os"

+ "path/filepath"

+ "strings"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/lease"

+ "github.com/carlosframework/platform/internal/moves"

+ "github.com/carlosframework/platform/internal/park"

+ "github.com/carlosframework/platform/internal/provision"

+)

+

+// The abort (moves spec §7.11; Plan 5 Task 1). The fixture is the Phase 1

+// harness after one preparing round, so the lane holds staged objects in the

+// target bucket (data-uk) and the source's own bucket (data-ie) holds its

+// receipts, its ledger and a snapshot. The target region's box (uk-box) runs

+// the lane park through the real park engine over the same buckets with a

+// fake lease table. Everything else is real: the source's abort, hostclaim's

+// Abort and ReleaseAbort, and provision's guarded transitions.

+

+type abortHarness struct {

+ *phase1Harness

+ tdir string

+ tleases *fakeParkLeases

+ tw *movesWorker

+ abAt string

+ abDone bool

+ abHits []string

+}

+

+func newAbortHarness(t *testing.T) *abortHarness {

+ t.Helper()

+ h := &abortHarness{phase1Harness: newPhase1Harness(t), tdir: t.TempDir(), tleases: &fakeParkLeases{}}

+ h.start(t, "manual")

+ h.pass(t)

+ if len(h.laneKeys()) == 0 {

+ t.Fatal("the fixture staged nothing")

+ }

+ h.f.put("data-ie", moves.SnapshotsPrefix(p1Host, p1ID)+"files-x.sealed", "s")

+ h.cfg.abFault = func(point string) error {

+ h.abHits = append(h.abHits, point)

+ if point == h.abAt && !h.abDone {

+ h.abDone = true

+ return errCrash

+ }

+ return nil

+ }

+ return h

+}

+

+// abort is the console's abort door: the guarded record CAS.

+func (h *abortHarness) abort(t *testing.T) {

+ t.Helper()

+ g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}

+ if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

+ t.Fatal(err)

+ }

+}

+

+// laneCfg is the target region's box with its lane park armed.

+func (h *abortHarness) laneCfg() moveCfg {

+ c := h.targetCfg()

+ c.DataDir = h.tdir

+ c.ParkLeases = h.tleases

+ return c

+}

+

+// tPass runs one pass as the target region's box.

+func (h *abortHarness) tPass(t *testing.T) {

+ t.Helper()

+ if h.tw == nil {

+ h.tw = &movesWorker{mirrors: map[string]lease.MirrorStore{}}

+ }

+ h.tw.cfg = h.laneCfg()

+ if err := movesPass(context.Background(), h.s, h.tw); err != nil {

+ t.Fatal(err)

+ }

+}

+

+// laneChore is the lane's park the abort queued in the target region.

+func (h *abortHarness) laneChore(t *testing.T) *hostclaim.Obligation {

+ t.Helper()

+ for _, o := range h.claim(t).Obligations {

+ if o.Op == p1ID && o.Kind == "park-replica" && o.Region == "uk" && o.Park != nil && o.Park.Location == moves.ScopeLocation(p1Host, p1ID) {

+ return &o

+ }

+ return nil

+}

+

+// laneKeys are the objects still in the move's lane in the target bucket.

+func (h *abortHarness) laneKeys() []string {

+ var out []string

+ for k := range h.f.buckets["data-uk"] {

+ if strings.HasPrefix(k, moves.ScopeLocation(p1Host, p1ID)+"/") {

+ out = append(out, k)

+ }

+ return out

+}

+

+// archived counts the objects under one retirement's archive of the lane.

+func (h *abortHarness) archived(inc, ret string) int {

+ prefix := "parked/" + inc + "/" + p1Host + "/" + ret + "/moves/"

+ n := 0

+ for k := range h.f.buckets["data-uk"] {

+ if strings.HasPrefix(k, prefix) {

+ n++

+ }

+ return n

+}

+

+// targetDeletes are every delete request anyone sent to the target bucket:

+// a plain delete, a guarded staging delete, or a delete by version.

+func (h *abortHarness) targetDeletes() []string {

+ var out []string

+ for _, op := range h.f.ops {

+ if strings.HasPrefix(op, "delete data-uk/") {

+ out = append(out, op)

+ }

+ for _, c := range h.f.stagingCalls() {

+ if c.Op == "delete" && c.Bucket == "data-uk" {

+ out = append(out, "staging delete "+c.Key)

+ }

+ out = append(out, h.vs.callsOn("delete-version data-uk/")...)

+ return out

+}

+

+func (h *abortHarness) pastHorizon(t *testing.T, o *hostclaim.Obligation) {

+ t.Helper()

+ until := o.Park.EvidenceAt.Add(park.SignatureAge + o.Park.RequestLimit)

+ if d := until.Sub(h.clock.now); d > 0 {

+ h.clock.Advance(d + time.Second)

+ }

+}

+

+// waiting asserts the abort is still waiting for the lane park: the record

+// aborting, the claim moving/aborting under the source, the chore owed.

+func (h *abortHarness) waiting(t *testing.T) {

+ t.Helper()

+ if m := h.read(t, p1Host).Move; m == nil || m.State != provision.MoveAborting {

+ t.Fatalf("the record is not aborting: %+v", m)

+ }

+ c := h.claim(t)

+ if c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseAborting || c.Op.ID != p1ID || c.Op.Exec != "ie-box" {

+ t.Fatalf("the claim while the abort waits: %+v", c.Op)

+ }

+}

+

+// closed asserts the end: the record at rest in ie with LastMove aborted,

+// the claim serving under the source with this move recorded as released.

+func (h *abortHarness) closed(t *testing.T) {

+ t.Helper()

+ rec := h.read(t, p1Host)

+ if rec.Move != nil || rec.Region != "ie" || rec.LastMove == nil || rec.LastMove.ID != p1ID || rec.LastMove.Outcome != "aborted" {

+ t.Fatalf("the record after the abort: region %s move %+v last %+v", rec.Region, rec.Move, rec.LastMove)

+ }

+ c := h.claim(t)

+ if c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" || hostclaim.RequireReleasedMove(p1ID)(c) != nil || len(c.Obligations) != 0 {

+ t.Fatalf("the claim after the abort: %+v obligations %+v", c.Op, c.Obligations)

+ }

+}

+

+// ownStateGone asserts the source retired its own state for the move: the

+// snapshots, receipts and ledger in its bucket, and its journal directory.

+func (h *abortHarness) ownStateGone(t *testing.T) {

+ t.Helper()

+ for k := range h.f.buckets["data-ie"] {

+ if strings.HasPrefix(k, moves.SnapshotsPrefix(p1Host, p1ID)) || k == moves.ReceiptsKey(p1Host, p1ID) || k == moves.LedgerKey(p1Host, p1ID) {

+ t.Fatalf("%s survived the abort", k)

+ }

+ if _, err := os.Stat(moves.JournalDir(h.cfg.DataDir, p1Host, p1ID)); !os.IsNotExist(err) {

+ t.Fatalf("the source's journal directory is still there: %v", err)

+ }

+}

+

+// TestAbortParksFromTheTargetAndWaits (Review Focus 2, §7.11, joint N13): the

+// source's passes never send a delete to the target bucket; the source takes

+// the claim to moving/aborting with the lane's park queued for the target

+// region, and retires its own snapshots (by version), receipts, ledger and

+// journal. The target region's box archives moves/<host>/<id>/... under

+// parked/<inc>/<host>/<ret>/moves/... and acknowledges only after the

+// horizon. Only then does the source release the claim to serving under

+// itself, and the record clear.

+func TestAbortParksFromTheTargetAndWaits(t *testing.T) {

+ h := newAbortHarness(t)

+ lane := h.laneKeys()

+ h.abort(t)

+ deletes := len(h.targetDeletes())

+ acquired := h.lease.acquired

+ decided := h.clock.now

+ h.pass(t)

+ h.waiting(t)

+ o := h.laneChore(t)

+ if o == nil {

+ t.Fatal("no lane park queued")

+ }

+ if o.Owner != "region:uk" || o.Park.Incarnation != h.inc || o.Park.RequestLimit != moves.ParkRequestLimit {

+ t.Fatalf("the lane park: %+v %+v", o, o.Park)

+ }

+ if min := decided.Add(moves.SignatureLife + moves.DefaultSkew); o.Park.EvidenceAt.Before(min) || o.Park.EvidenceAt.After(min.Add(time.Second)) {

+ t.Fatalf("the park's horizon starts at %v, want the decision's time plus a signature's life and the skew, %v", o.Park.EvidenceAt, min)

+ }

+ if h.lease.acquired != acquired+1 || h.lease.released != h.lease.acquired {

+ t.Fatalf("the staging lease: acquired %d (was %d), released %d", h.lease.acquired, acquired, h.lease.released)

+ }

+ h.ownStateGone(t)

+ if got := h.vs.callsOn("delete-version data-ie/" + moves.SnapshotsPrefix(p1Host, p1ID)); len(got) == 0 {

+ t.Fatal("the snapshots were not retired by version")

+ }

+ if got := h.laneKeys(); len(got) != len(lane) {

+ t.Fatalf("the source touched the lane: %d keys, had %d", len(got), len(lane))

+ }

+ // The source waits: another pass changes nothing and deletes nothing there.

+ gen := h.claim(t).Gen

+ h.pass(t)

+ h.waiting(t)

+ if h.claim(t).Gen != gen {

+ t.Fatal("a waiting source wrote the claim")

+ }

+ if got := h.targetDeletes(); len(got) != deletes {

+ t.Fatalf("the source sent deletes to the target bucket: %v", got[deletes:])

+ }

+ // The target region's box archives the lane; the horizon has not passed.

+ h.tPass(t)

+ if left := h.laneKeys(); len(left) != 0 {

+ t.Fatalf("the lane still holds %v", left)

+ }

+ if n := h.archived(o.Park.Incarnation, o.Retirement); n < len(lane) {

+ t.Fatalf("%d archived, the lane had %d", n, len(lane))

+ }

+ if h.laneChore(t) == nil {

+ t.Fatal("the lane park was acknowledged before its horizon")

+ }

+ h.pass(t)

+ h.waiting(t)

+ // Past the horizon the target acknowledges; the source then releases.

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ if h.laneChore(t) != nil {

+ t.Fatal("the lane park was not acknowledged after its horizon")

+ }

+ h.waiting(t)

+ h.pass(t)

+ h.closed(t)

+ before := h.claim(t)

+ h.pass(t)

+ h.tPass(t)

+ if after := h.claim(t); after.Gen != before.Gen {

+ t.Fatalf("a pass after the abort wrote the claim (%d to %d)", before.Gen, after.Gen)

+ }

+}

+

+// TestAbortScopeIsTheWritersHorizon (witness wStagingQuiet; ruling K5): the

+// abort's horizon covers every staging request the source signed before it,

+// and a staging PUT signed before the abort that lands after the park's

+// first pass is found by the verification listing and parked as a further

+// retirement event. The park acknowledges only after abort_bound + H.

+func TestAbortScopeIsTheWritersHorizon(t *testing.T) {

+ h := newAbortHarness(t)

+ signed := h.f.stagingCalls()

+ if len(signed) == 0 {

+ t.Fatal("the fixture signed no staging request")

+ }

+ h.abort(t)

+ h.pass(t)

+ o := h.laneChore(t)

+ if o == nil {

+ t.Fatal("no lane park queued")

+ }

+ bound := o.Park.EvidenceAt

+ for _, c := range signed {

+ if c.SignAt.Add(moves.SignatureLife + moves.DefaultSkew).After(bound) {

+ t.Fatalf("a staging request signed at %v can land after the abort's horizon %v", c.SignAt, bound)

+ }

+ // The park's first pass archives the lane.

+ h.tPass(t)

+ first := o.Retirement

+ if len(h.laneKeys()) != 0 || h.laneChore(t) == nil {

+ t.Fatalf("after the first pass: lane %v", h.laneKeys())

+ }

+ // A request signed before the abort lands now, in the lane.

+ late := moves.ScopeLocation(p1Host, p1ID) + "/replica/0000/0000000000000009-0000000000000009.ltx"

+ h.f.store("data-uk", late, fakeMoveObject{lease.MirrorObject{ETag: "late", Size: 4}, "late"})

+ h.f.setVersion("data-uk", late, h.f.nextVersion())

+ // One second before abort_bound + H nothing is acknowledged, whatever

+ // the listing says.

+ deadline := bound.Add(park.SignatureAge + o.Park.RequestLimit)

+ if d := deadline.Add(-time.Second).Sub(h.clock.now); d > 0 {

+ h.clock.Advance(d)

+ }

+ h.tPass(t)

+ if h.laneChore(t) == nil {

+ t.Fatal("the lane park was acknowledged before abort_bound + H")

+ }

+ h.pass(t)

+ h.waiting(t)

+ // Past the horizon the listing finds the late write: a further retirement.

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ o2 := h.laneChore(t)

+ if o2 == nil {

+ t.Fatal("the park acknowledged over the late write")

+ }

+ if o2.Retirement == first || !o2.Park.EvidenceAt.Equal(bound) {

+ t.Fatalf("the further event: retirement %q (first %q), horizon %v", o2.Retirement, first, o2.Park.EvidenceAt)

+ }

+ h.tPass(t)

+ if h.laneChore(t) != nil {

+ t.Fatal("the further retirement was not acknowledged")

+ }

+ if _, ok := h.f.buckets["data-uk"][late]; ok {

+ t.Fatal("the late write is still in the lane")

+ }

+ if n := h.archived(o.Park.Incarnation, o2.Retirement); n != 1 {

+ t.Fatalf("the further retirement archived %d objects, want the late write", n)

+ }

+ if q := h.claim(t).QuietAfter["uk"]; !q.Equal(deadline) {

+ t.Fatalf("quiet_after %v, want %v", q, deadline)

+ }

+ h.pass(t)

+ h.closed(t)

+}

+

+// TestAbortWaitsForAnUnreachableTargetRegion (ruling K21): with no box of

+// the target region running the lane park, the claim stays moving/aborting

+// and nothing releases, however long the source waits. A delete cannot

+// begin over it, because the claim is not serving.

+func TestAbortWaitsForAnUnreachableTargetRegion(t *testing.T) {

+ h := newAbortHarness(t)

+ h.abort(t)

+ h.pass(t)

+ o := h.laneChore(t)

+ if o == nil {

+ t.Fatal("no lane park queued")

+ }

+ for i := 0; i < 4; i++ {

+ h.clock.Advance(12 * time.Hour)

+ h.pass(t)

+ h.waiting(t)

+ if h.laneChore(t) == nil {

+ t.Fatal("the lane park vanished with no target pass")

+ }

+ c := h.claim(t)

+ if err := hostclaim.StartGuard(c, []string{"ie"}, []string{"ie-box"}); err == nil {

+ t.Fatal("a new operation could start over the waiting abort")

+ }

+ // An unarmed target box (production until Task 9) parks nothing either.

+ unarmed := h.laneCfg()

+ unarmed.ParkLeases = nil

+ w := &movesWorker{cfg: unarmed, mirrors: map[string]lease.MirrorStore{}}

+ if err := movesPass(context.Background(), h.s, w); err != nil {

+ t.Fatal(err)

+ }

+ if h.laneChore(t) == nil || len(h.laneKeys()) == 0 {

+ t.Fatal("an unarmed target box parked the lane")

+ }

+ // Once the region returns, the abort completes.

+ h.tPass(t)

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ h.pass(t)

+ h.closed(t)

+}

+

+// TestAbortReplays: a crash after any of the abort's side effects, then a

+// restarted agent, still reaches the close, with the journalled bound named

+// once (the lane park's scope never changes), one lane park queued, and the

+// source's own state retired.

+func TestAbortReplays(t *testing.T) {

+ for _, point := range []string{"after-abort-bound", "after-abort-claim", "after-abort-own-state", "after-abort-journal", "after-release-abort"} {

+ t.Run(point, func(t *testing.T) {

+ h := newAbortHarness(t)

+ h.abort(t)

+ h.abAt = point

+ if err := movesPass(context.Background(), h.s, h.w()); err != nil {

+ t.Fatal(err)

+ }

+ for i := 0; i < 6 && !h.abDone; i++ {

+ h.tPass(t)

+ if o := h.laneChore(t); o != nil {

+ h.pastHorizon(t, o)

+ }

+ h.pass(t)

+ }

+ if !h.abDone {

+ t.Fatalf("never reached %s (%v)", point, h.abHits)

+ }

+ var bound time.Time

+ if j, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID); err == nil {

+ if raw := j.Steps[abortStepBound]; raw != "" {

+ bound, _ = time.Parse(time.RFC3339Nano, raw)

+ }

+ if o := h.laneChore(t); o != nil {

+ if !bound.IsZero() && !o.Park.EvidenceAt.Equal(bound) {

+ t.Fatalf("the park's horizon %v is not the journalled bound %v", o.Park.EvidenceAt, bound)

+ }

+ bound = o.Park.EvidenceAt

+ }

+ h.clock.Advance(time.Minute)

+ h.worker1 = nil

+ for i := 0; i < 8 && h.read(t, p1Host).Move != nil; i++ {

+ h.pass(t)

+ if o := h.laneChore(t); o != nil {

+ if !bound.IsZero() && !o.Park.EvidenceAt.Equal(bound) {

+ t.Fatalf("the replay named another horizon: %v, journalled %v", o.Park.EvidenceAt, bound)

+ }

+ h.tPass(t)

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ }

+ h.closed(t)

+ h.ownStateGone(t)

+ })

+ }

+}

+

+// An aborting record never flipped: its region is the source's, so the

+// target region's box has neither the row nor a journal of the move. The

+// participant rule answers it as the target on the state alone, as for a

+// revert: its one piece of work is the lane park (ruling K6). The source's

+// region answers nothing without the row (its abort is phase1MoveRound's),

+// and a third region nothing.

+func TestAbortingTargetIsAParticipant(t *testing.T) {

+ cfg := moveCfg{Region: "uk", DataDir: t.TempDir(), OwnsRoute: func(string) bool { return false }}

+ rec := provision.Record{Host: p1Host, Region: "ie", Move: &provision.MoveState{ID: p1ID, State: provision.MoveAborting, From: "ie", To: "uk"}}

+ if got := movesParticipantRecord(cfg, rec); got != roleTarget {

+ t.Fatalf("an aborting record's target: %q", got)

+ }

+ cfg.Region = "ie"

+ if got := movesParticipantRecord(cfg, rec); got != roleNone {

+ t.Fatalf("the source's region without the row: %q", got)

+ }

+ cfg.Region = "fr"

+ if got := movesParticipantRecord(cfg, rec); got != roleNone {

+ t.Fatalf("a third region: %q", got)

+ }

+}

+

+// The abort's crash boundary between the claim's release and the record's

+// clear: a round that finds the record aborting and the claim serving

+// completes the record only when the serving op records this move as the

+// one it released (RequireReleasedMove). A serving claim without that

+// evidence was released by someone else, and the record is left alone.

+func TestAbortCompletesOnlyWithItsReleaseEvidence(t *testing.T) {

+ ctx := context.Background()

+ h := newAbortHarness(t)

+ h.abort(t)

+ obs, err := hostclaim.Read(ctx, h.s, p1Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ scope := moves.AbortScope(p1Host, p1ID, h.inc, h.clock.now.Add(time.Hour))

+ if obs, err = hostclaim.Update(ctx, h.s, obs, hostclaim.Abort(p1ID, "uk", scope, obligationID)); err != nil {

+ t.Fatal(err)

+ }

+ if obs, err = hostclaim.Update(ctx, h.s, obs, hostclaim.CompleteObligation(p1ID, obs.Claim.Obligations[0].ID)); err != nil {

+ t.Fatal(err)

+ }

+ if _, err = hostclaim.Update(ctx, h.s, obs, hostclaim.ReleaseAbort(p1ID, "ie-box")); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ h.closed(t)

+ h.ownStateGone(t)

+

+ // A serving claim with no evidence of this release: left alone.

+ h2 := newAbortHarness(t)

+ h2.abort(t)

+ rewriteClaimOp(t, h2.s, func(op *hostclaim.Op) { *op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} })

+ h2.pass(t)

+ if got := h2.read(t, p1Host); got.Move == nil || got.Move.State != provision.MoveAborting {

+ t.Fatalf("a serving claim without evidence completed the record: %+v", got.Move)

+ }

+}

+

+// Fix round 1 (task-1-review.md, Important): release never follows from an

+// inference that the source's cleanup finished. After the claim's CAS, the

+// source retires its own state on every pass before ReleaseAbort, and any

+// failure to do so blocks the release until it succeeds.

+func TestAbortReleaseWaitsForTheSourcesCleanup(t *testing.T) {

+ // crashAfterClaim stops the source right after the Abort CAS, then lets

+ // the target region park and acknowledge the lane, so only the source's

+ // own cleanup stands between the claim and its release.

+ crashAfterClaim := func(t *testing.T) *abortHarness {

+ t.Helper()

+ h := newAbortHarness(t)

+ h.abort(t)

+ h.abAt = "after-abort-claim"

+ if err := movesPass(context.Background(), h.s, h.w()); err != nil {

+ t.Fatal(err)

+ }

+ if !h.abDone {

+ t.Fatalf("never reached after-abort-claim (%v)", h.abHits)

+ }

+ o := h.laneChore(t)

+ if o == nil {

+ t.Fatal("no lane park queued")

+ }

+ h.tPass(t)

+ h.pastHorizon(t, o)

+ h.tPass(t)

+ if h.laneChore(t) != nil {

+ t.Fatal("the lane park was not acknowledged")

+ }

+ h.worker1 = nil

+ return h

+ }

+ t.Run("the journal is gone but the state is not", func(t *testing.T) {

+ h := crashAfterClaim(t)

+ if err := os.RemoveAll(moves.JournalDir(h.cfg.DataDir, p1Host, p1ID)); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ h.closed(t)

+ h.ownStateGone(t)

+ })

+ t.Run("the journal cannot be inspected", func(t *testing.T) {

+ if os.Geteuid() == 0 {

+ t.Skip("root reads and removes past any mode")

+ }

+ h := crashAfterClaim(t)

+ hostDir := filepath.Dir(moves.JournalDir(h.cfg.DataDir, p1Host, p1ID))

+ if err := os.Chmod(hostDir, 0); err != nil {

+ t.Fatal(err)

+ }

+ t.Cleanup(func() { os.Chmod(hostDir, 0o755) })

+ for i := 0; i < 3; i++ {

+ movesPass(context.Background(), h.s, h.w())

+ h.waiting(t)

+ }

+ if err := os.Chmod(hostDir, 0o755); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ h.closed(t)

+ h.ownStateGone(t)

+ })

+ t.Run("the snapshots cannot be listed", func(t *testing.T) {

+ h := crashAfterClaim(t)

+ h.vs.deny = func(op, bucket, key string) error {

+ if op == "list-versions" && bucket == "data-ie" {

+ return errors.New("injected 503")

+ }

+ return nil

+ }

+ for i := 0; i < 3; i++ {

+ movesPass(context.Background(), h.s, h.w())

+ h.waiting(t)

+ }

+ h.vs.deny = nil

+ h.pass(t)

+ h.closed(t)

+ h.ownStateGone(t)

+ })

+}

diff --git a/internal/host/movescut.go b/internal/host/movescut.go

index fbf4d0cb8..26306c03f 100644

--- a/internal/host/movescut.go

+++ b/internal/host/movescut.go

@@ -52,7 +52,7 @@ const cutLandingWindow = 15 * time.Minute

// cutRequestLimit is the staging park's request limit, the longest a

// staging request runs (lease's stagingRequestTimeout).

-const cutRequestLimit = 120 * time.Second

+const cutRequestLimit = moves.ParkRequestLimit

// cutMaxDeferred is how many final rounds may defer before the cut reverts

// (§7.4 staged: three deferred rounds).

@@ -1224,7 +1224,7 @@ func (w *movesWorker) revertClaim(ctx context.Context, s release.Store, rec prov

return err

}

- scope := hostclaim.ParkScope{Location: moves.ScopeLocation(rec.Host, id), Incarnation: obs.Claim.Holder.Incarnation, EvidenceAt: bound, RequestLimit: cutRequestLimit}

+ scope := moves.StagingScope(rec.Host, id, obs.Claim.Holder.Incarnation, bound)

if _, err := hostclaim.Update(ctx, s, obs, hostclaim.Revert(id, cfg.Box, rec.Move.To, &scope, obligationID)); err != nil {

return fmt.Errorf("revert the claim: %w", err)

}

diff --git a/internal/host/movesfinal.go b/internal/host/movesfinal.go

index a339f7694..b1cd2a78d 100644

--- a/internal/host/movesfinal.go

+++ b/internal/host/movesfinal.go

@@ -84,12 +84,13 @@ const (

const finalConfigAckName = "config-ack-source.json"

// movesParticipantStates are the record states after the flip (and a

-// revert) in which a participant may have work for a record whose region

-// no longer names it, or whose row it has already deleted.

+// revert or an abort) in which a participant may have work for a record

+// whose region no longer names it, or whose row it has already deleted.

var movesParticipantStates = map[string]bool{

provision.MoveCommitted: true,

provision.MoveRollingBack: true,

provision.MoveReverting: true,

+ provision.MoveAborting: true,

}

// movesParticipantRecord is the routing rule for the states after the flip

@@ -117,10 +118,10 @@ func movesParticipantRecord(cfg moveCfg, rec provision.Record) string {

if cfg.OwnsRoute == nil || cfg.OwnsRoute(rec.Host) || moveJournalPresent(cfg.DataDir, rec.Host, m.ID) {

return role

}

- // A revert never flipped: its target has neither the row nor a journal,

- // and its one piece of work is the lane park the claim owes its region

- // (movesrevert.go, Plan 4 Task 6).

- if role == roleTarget && m.State == provision.MoveReverting {

+ // A revert or an abort never flipped: its target has neither the row

+ // nor a journal, and its one piece of work is the lane park the claim

+ // owes its region (laneParkTarget; Plan 4 Task 6, Plan 5 Task 1).

+ if role == roleTarget && (m.State == provision.MoveReverting || m.State == provision.MoveAborting) {

return role

}

return roleNone

@@ -139,8 +140,9 @@ func moveJournalPresent(dataDir, host, op string) bool {

// participantRound is a post-flip participant's step for a record the

// region rule did not admit (movesPass): the source's finalization of a

-// committed move, and the target's half of a rollback (movesrollback.go,

-// J10). The rest is Plan 4's later tasks'.

+// committed move, the target's half of a rollback (movesrollback.go, J10),

+// and the target region's lane park of a revert or an abort

+// (laneParkTarget). The rest is Plan 4's later tasks'.

func (w *movesWorker) participantRound(ctx context.Context, s release.Store, rec provision.Record, role string) error {

switch {

case role == roleSource && rec.Move.State == provision.MoveCommitted:

@@ -148,7 +150,9 @@ func (w *movesWorker) participantRound(ctx context.Context, s release.Store, rec

case role == roleTarget && rec.Move.State == provision.MoveRollingBack:

return w.rollbackTarget(ctx, s, rec)

case role == roleTarget && rec.Move.State == provision.MoveReverting:

- return w.revertLanePark(ctx, s, rec)

+ return w.laneParkTarget(ctx, s, rec, hostclaim.KindReverting)

+ case role == roleTarget && rec.Move.State == provision.MoveAborting:

+ return w.laneParkTarget(ctx, s, rec, hostclaim.KindMoving, hostclaim.PhaseAborting)

}

return nil

}

@@ -174,8 +178,7 @@ func sourceLocationParkScope(rec provision.Record, holder hostclaim.Holder) (hos

if rec.Move == nil || rec.Move.FlippedAt == nil {

return hostclaim.ParkScope{}, errors.New("the source's location park needs the flip's time")

}

- return hostclaim.ParkScope{Location: strings.TrimSuffix(holder.Location, "/"), Incarnation: holder.Incarnation,

- EvidenceAt: rec.Move.FlippedAt.UTC(), RequestLimit: cutRequestLimit}, nil

+ return moves.SourceLocationScope(holder, *rec.Move.FlippedAt), nil

}

// errFinalChanged is the record or claim no longer being this move's commit

diff --git a/internal/host/movesinstall.go b/internal/host/movesinstall.go

index 5d1f4e8f8..f90509e38 100644

--- a/internal/host/movesinstall.go

+++ b/internal/host/movesinstall.go

@@ -424,7 +424,7 @@ func (ir *installRun) parking(ctx context.Context) (bool, error) {

if ir.rec.Move.FlippedAt == nil {

return false, fmt.Errorf("move %s: the record has no flip time", id)

}

- scope := hostclaim.ParkScope{Location: location, Incarnation: holder.Incarnation, EvidenceAt: ir.rec.Move.FlippedAt.UTC(), RequestLimit: landingRequestLimit}

+ scope := moves.ResidueScope(holder, *ir.rec.Move.FlippedAt)

if obs, err = hostclaim.Update(ctx, ir.s, obs, hostclaim.QueueResidueParks(id, cfg.Region, scope, gens, obligationID)); err != nil {

return false, fmt.Errorf("queue the residue parks: %w", err)

}

diff --git a/internal/host/movespass.go b/internal/host/movespass.go

index 5285cc85c..2bf3d96d5 100644

--- a/internal/host/movespass.go

+++ b/internal/host/movespass.go

@@ -232,6 +232,7 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,

ReadyDelta: readyDelta,

Enabled: moves.Enabled(os.Getenv),

claimRecords: writes.Enabled,

+ Page: PageOperator,

}

w := &movesWorker{mirrors: map[string]lease.MirrorStore{}}

if cfg.Enabled {

@@ -584,6 +585,13 @@ type moveCfg struct {

LogEvent func(ctx context.Context, rec provision.Record, event string)

rbFault func(point string) error

+ // Page pages an operator (moves spec §13, K11): PageOperator in

+ // production (armMovesPass), nil in tests, where nothing pages. The box

+ // arms call it where a MoveMarkStuck lands (pageStuck): the rollback's

+ // return on the target, and the recovery, its dirty return's install

+ // and the revert on the source.

+ Page func(ctx context.Context, level, subject, body string) error

+

// The source's recovery after a rollback (§7.8, movesrecover.go).

// ReleaseHeld opens the row: registry.ReleaseHold under the op that

// holds it, touching the stage-requested marker when it landed

@@ -610,6 +618,12 @@ type moveCfg struct {

// production.

RestatusHold func(host, status, op string, validate func() error) (bool, error)

rvFault func(point string) error

+

+ // The abort (§7.11, movesabort.go): the source uses the staging lease,

+ // Versioned and its own bucket above; the target region's lane park is

+ // laneParkTarget's, armed as the revert's (revertLaneArmed). abFault is

+ // its test seam, nil in production.

+ abFault func(point string) error

}

// defaultMoveReadyDelta is the readiness bar when

diff --git a/internal/host/movesphase1.go b/internal/host/movesphase1.go

index d997b5f3f..485677065 100644

--- a/internal/host/movesphase1.go

+++ b/internal/host/movesphase1.go

@@ -10,7 +10,6 @@ import (

"log"

"os"

"path/filepath"

- "sort"

"strings"

"time"

@@ -185,7 +184,9 @@ func phase1MoveRound(ctx context.Context, s release.Store, w *movesWorker, rec p

case provision.MovePreparing, provision.MoveReady:

return w.preparingRound(ctx, s, rec, fleet)

case provision.MoveAborting:

- return w.interimAbort(ctx, s, rec)

+ // The abort's source half (movesabort.go, Plan 5 Task 1): the lane

+ // is the target region's to park.

+ return w.abortSource(ctx, s, rec)

case provision.MoveCutting:

return w.runCut(ctx, s, rec, fleet)

case provision.MoveReverting:

@@ -955,85 +956,3 @@ func readReport(ctx context.Context, s release.Store, key string, into any) (boo

}

return true, nil

}

-

-// interimAbort is the abort's box half until Plan 5 replaces it (§7.11):

-// the source deletes its own snapshots, receipts and ledger for the move,

-// releases the claim back to serving (K8), and then completes the record.

-// The staging lane in the target bucket is left for the target's park.

-//

-// The crash boundary between the claim release and the record write: a

-// round that finds the record aborting and the claim already serving

-// completes the record only when the serving op records this move as the

-// one it released (RequireReleasedMove). A serving claim without that

-// evidence was released by someone else (the console pass, Plan 5), and

-// the record is left to it.

-func (w *movesWorker) interimAbort(ctx context.Context, s release.Store, rec provision.Record) error {

- cfg := w.cfg

- id := rec.Move.ID

- g := provision.MoveGuard{ID: id, Incarnation: rec.Move.Incarnation}

- obs, err := hostclaim.Read(ctx, s, rec.Host)

- if err != nil {

- return fmt.Errorf("read the host claim: %w", err)

- }

- if obs.Absent {

- w.notice(rec.Host+"#abort", "moves: %s: aborting move %s has no host claim; left for the console pass", rec.Host, id)

- return nil

- }

- c := obs.Claim

- switch {

- case c.Op.Kind == hostclaim.KindMoving && c.Op.ID == id:

- if err := w.deleteOwnMoveState(ctx, rec); err != nil {

- return err

- }

- box := cfg.Box

- if box == "" {

- box = c.Op.Exec

- }

- if _, err := hostclaim.Update(ctx, s, obs, hostclaim.ReleaseMove(id, box)); err != nil {

- return fmt.Errorf("release the host claim: %w", err)

- }

- case hostclaim.RequireReleasedMove(id)(c) == nil:

- // The claim was released by this abort before a crash: finish.

- if err := w.deleteOwnMoveState(ctx, rec); err != nil {

- return err

- }

- default:

- w.notice(rec.Host+"#abort", "moves: %s: aborting move %s: the claim is %s/%s and records no release of this move; left for the console pass", rec.Host, id, c.Op.Kind, c.Op.ID)

- return nil

- }

- if _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, provision.MoveGuardedAbortComplete(g, cfg.Now())); err != nil {

- if errors.Is(err, provision.ErrMoveConflict) {

- log.Printf("moves: %s: abort of %s: the record moved on (%v)", rec.Host, id, err)

- return nil

- }

- return err

- }

- nudgeSerial(ctx, s)

- log.Printf("moves: %s: move %s aborted; the instance stays in %q", rec.Host, id, rec.Region)

- return nil

-}

-

-// deleteOwnMoveState removes the move's snapshots, receipts and ledger from

-// this box's own bucket. Idempotent.

-func (w *movesWorker) deleteOwnMoveState(ctx context.Context, rec provision.Record) error {

- cfg := w.cfg

- own, err := w.mirror(ctx, cfg.OwnBucketRegion)

- if err != nil {

- return err

- }

- if err := lease.DeletePrefix(ctx, own, cfg.OwnBucket, moves.SnapshotsPrefix(rec.Host, rec.Move.ID)); err != nil {

- return fmt.Errorf("delete the move's snapshots: %w", err)

- }

- keys := []string{moves.ReceiptsKey(rec.Host, rec.Move.ID), moves.LedgerKey(rec.Host, rec.Move.ID)}

- sort.Strings(keys)

- for _, k := range keys {

- if err := own.Delete(ctx, cfg.OwnBucket, k); err != nil && !errors.Is(err, lease.ErrMirrorNotFound) {

- return fmt.Errorf("delete %s: %w", k, err)

- }

- // The move's local directory: its journal and the archive kept for it.

- if err := os.RemoveAll(moves.JournalDir(cfg.DataDir, rec.Host, rec.Move.ID)); err != nil {

- return fmt.Errorf("remove the move's local directory: %w", err)

- }

- return nil

-}

diff --git a/internal/host/movesphase1_test.go b/internal/host/movesphase1_test.go

index 4c1fb6e5a..a6efc9a4c 100644

--- a/internal/host/movesphase1_test.go

+++ b/internal/host/movesphase1_test.go

@@ -199,67 +199,6 @@ func TestStagingMoveJSONReplayAfterLostLedger(t *testing.T) {

}

-// The interim abort (§7.11): the source deletes its own snapshots, receipts

-// and ledger, releases the claim to serving, then completes the record;

-// the staging lane is left for the target's park. A crash between the two

-// writes completes on the next round only with the release's evidence, and

-// a serving claim without it is left to the console pass.

-func TestInterimAbortReleasesTheClaim(t *testing.T) {

- ctx := context.Background()

- h := newPhase1Harness(t)

- h.start(t, "manual")

- h.pass(t)

- h.f.put("data-ie", moves.SnapshotsPrefix(p1Host, p1ID)+"files-x.sealed", "s")

- g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}

- if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

- t.Fatal(err)

- }

- lane := len(stagedKeys(h, moves.StagingPrefix(p1Host, p1ID)))

- h.pass(t)

- got := h.read(t, p1Host)

- if got.Move != nil || got.LastMove == nil || got.LastMove.Outcome != "aborted" {

- t.Fatalf("abort did not complete: %+v %+v", got.Move, got.LastMove)

- }

- if c := h.claim(t); c.Op.Kind != hostclaim.KindServing || c.Op.Exec != "ie-box" {

- t.Fatalf("claim after the abort: %+v", c.Op)

- }

- for _, k := range []string{moves.ReceiptsKey(p1Host, p1ID), moves.LedgerKey(p1Host, p1ID), moves.SnapshotsPrefix(p1Host, p1ID) + "files-x.sealed"} {

- if _, ok := h.f.buckets["data-ie"][k]; ok {

- t.Fatalf("%s survived the abort", k)

- }

- if n := len(stagedKeys(h, moves.StagingPrefix(p1Host, p1ID))); n != lane || n == 0 {

- t.Fatalf("the source touched the lane: %d keys, had %d", n, lane)

- }

-

- // The crash case: the claim released, the record still aborting.

- h2 := newPhase1Harness(t)

- h2.start(t, "manual")

- if _, err := provision.SetRecordMove(ctx, h2.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

- t.Fatal(err)

- }

- obs, _ := hostclaim.Read(ctx, h2.s, p1Host)

- if _, err := hostclaim.Update(ctx, h2.s, obs, hostclaim.ReleaseMove(p1ID, "ie-box")); err != nil {

- t.Fatal(err)

- }

- h2.pass(t)

- if got := h2.read(t, p1Host); got.Move != nil {

- t.Fatalf("the replay did not complete the record: %+v", got.Move)

- }

-

- // A serving claim with no evidence of this release: left alone.

- h3 := newPhase1Harness(t)

- h3.start(t, "manual")

- if _, err := provision.SetRecordMove(ctx, h3.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

- t.Fatal(err)

- }

- rewriteClaimOp(t, h3.s, func(op *hostclaim.Op) { *op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} })

- h3.pass(t)

- if got := h3.read(t, p1Host); got.Move == nil || got.Move.State != provision.MoveAborting {

- t.Fatalf("a serving claim without evidence completed the record: %+v", got.Move)

- }

-}

-

// rewriteClaimOp edits p1Host's claim op in place, as another writer would.

func rewriteClaimOp(t *testing.T, s release.Store, edit func(*hostclaim.Op)) {

t.Helper()

diff --git a/internal/host/movesrecover.go b/internal/host/movesrecover.go

index d51924ce3..a2764a6fa 100644

--- a/internal/host/movesrecover.go

+++ b/internal/host/movesrecover.go

@@ -303,6 +303,7 @@ func (r *recoverRun) stuck(ctx context.Context, reason string, why error) (bool,

return false, fmt.Errorf("mark the recovery stuck: %w", err)

}

r.w.logEvent(ctx, r.rec, "source-stuck")

+ r.w.pageStuck(ctx, r.rec, "source", reason)

nudgeSerial(ctx, r.s)

return true, nil

}

@@ -579,6 +580,7 @@ func (r *recoverRun) install(ctx context.Context) (bool, error) {

if refused != "" {

log.Printf("moves: %s: the return of %s stopped for an operator: %s", host, r.rb, refused)

r.w.logEvent(ctx, r.rec, "source-stuck")

+ r.w.pageStuck(ctx, r.rec, "source", refused)

}

return advanced, err

}

@@ -652,8 +654,7 @@ func (r *recoverRun) probed(ctx context.Context) (bool, error) {

if err != nil {

return false, err

}

- scope := hostclaim.ParkScope{Location: moves.ScopeLocation(host, r.rb), Incarnation: obs.Claim.Holder.Incarnation,

- EvidenceAt: seal.Bound().UTC(), RequestLimit: cutRequestLimit}

+ scope := moves.ReturnLaneScope(host, r.rb, obs.Claim.Holder.Incarnation, seal.Bound())

lane = hostclaim.QueueReturnLanePark(id, cfg.Region, scope, obligationID)

}

switch obs.Claim.Op.Phase {

diff --git a/internal/host/movesrecover_test.go b/internal/host/movesrecover_test.go

index bd270da0b..928368cb6 100644

--- a/internal/host/movesrecover_test.go

+++ b/internal/host/movesrecover_test.go

@@ -151,6 +151,7 @@ func (h *recoverHarness) sourceCfg(t *testing.T) moveCfg {

return true, nil

}

c.LogEvent = func(_ context.Context, _ provision.Record, event string) { h.sEvents = append(h.sEvents, event) }

+ c.Page = h.pages.hook()

c.recoveryProbe = func(ctx context.Context, _ release.Store, rec provision.Record, req recoveryProbeRequest) (recoveryProbeOutcome, error) {

h.probes = append(h.probes, req)

if h.probe != nil {

diff --git a/internal/host/movesreport.go b/internal/host/movesreport.go

index 147dbe336..b1ffb51a3 100644

--- a/internal/host/movesreport.go

+++ b/internal/host/movesreport.go

@@ -133,7 +133,13 @@ func (w *movesWorker) targetReportPass(ctx context.Context, s release.Store, rec

// A move into this box is live from preparing through the cut and

// the landing (ruling G26): the held refusal a landing's configured

// phase writes, and T5's verdict, stay until the move is over. The

- // report itself is written only while preparing or ready.

+ // report itself is written only while preparing or ready. An

+ // aborting move is over for this box: it never flipped, so no

+ // landing wrote a refusal, and its record stays aborting only until

+ // the lane park is acknowledged (Plan 5 Task 1).

+ if m.State == provision.MoveAborting {

+ continue

+ }

live[m.ID] = true

if m.State != provision.MovePreparing && m.State != provision.MoveReady {

continue

diff --git a/internal/host/movesreport_test.go b/internal/host/movesreport_test.go

index 49f0109c5..ed4541d3e 100644

--- a/internal/host/movesreport_test.go

+++ b/internal/host/movesreport_test.go

@@ -582,7 +582,13 @@ func TestArchiveVerdictsAreEvictedWhenTheMoveEnds(t *testing.T) {

if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveGuardedAbort(g)); err != nil {

t.Fatal(err)

}

- h.pass(t) // the abort completes; the record is at rest

+ // The source's pass begins the abort; the record stays aborting while

+ // the target region parks the lane (Plan 5 Task 1). The move is over for

+ // the target's caches already.

+ h.pass(t)

+ if m := h.read(t, p1Host).Move; m == nil || m.State != provision.MoveAborting {

+ t.Fatalf("the record during the park wait: %+v", m)

+ }

h.targetPass(t, h.targetCfg())

if len(h.worker2.archives) != 0 {

t.Fatalf("a verdict outlived its move: %v", h.worker2.archives)

diff --git a/internal/host/movesretire_byop_test.go b/internal/host/movesretire_byop_test.go

new file mode 100644

index 000000000..f054c87b7

--- /dev/null

+++ b/internal/host/movesretire_byop_test.go

@@ -0,0 +1,105 @@

+package host

+

+import (

+ "encoding/json"

+ "os"

+ "path/filepath"

+ "sort"

+ "strings"

+ "testing"

+

+ "github.com/carlosframework/platform/internal/hostclaim"

+)

+

+// retirementsByOp is how F3 finds a host's local retirements by op, one rule

+// for both receipts a retirement can carry (K20): park.Local's .journal.json

+// (Op) and the moves-owned rename's moves-retirement.json (op), each in the

+// event directory .parked/<incarnation>/<host>/<retirement>/. It returns the

+// retirement ids whose receipt names op, and those with no readable receipt.

+// This is the one copy of the rule; internal/park's test checks only that

+// park.Local's journal carries Op.

+func retirementsByOp(t *testing.T, dataDir, incarnation, host, op string) (found, unreadable []string) {

+ t.Helper()

+ dir := filepath.Join(dataDir, ".parked", incarnation, host)

+ es, err := os.ReadDir(dir)

+ if err != nil {

+ t.Fatal(err)

+ }

+ for _, e := range es {

+ var engine struct{ Op string }

+ var owned struct {

+ Op string `json:"op"`

+ }

+ named := ""

+ if b, err := os.ReadFile(filepath.Join(dir, e.Name(), ".journal.json")); err == nil && json.Unmarshal(b, &engine) == nil {

+ named = engine.Op

+ } else if b, err := os.ReadFile(filepath.Join(dir, e.Name(), localParkReceiptName)); err == nil && json.Unmarshal(b, &owned) == nil {

+ named = owned.Op

+ } else {

+ unreadable = append(unreadable, e.Name())

+ continue

+ }

+ if named == op {

+ found = append(found, e.Name())

+ }

+ sort.Strings(found)

+ return found, unreadable

+}

+

+// TestMoveLocalRetirementsAreFindableByOp (moves spec §6.4, D6, K20): a

+// move leaves two local receipts, not one. A registered source's

+// finalization retires its installation through park.Local (.journal.json)

+// and parks what the inventory does not list by the moves-owned rename

+// (moves-retirement.json), each in its own retirement under the engine's

+// layout. Both name the move's op, and nothing under the host's

+// incarnation lacks a receipt, so F3 finds every local retirement of the

+// move by op with one rule.

+func TestMoveLocalRetirementsAreFindableByOp(t *testing.T) {

+ h := newFinalHarness(t)

+ dd := h.cfg.DataDir

+ for _, d := range []string{"hostguard", ".parked"} {

+ if err := os.MkdirAll(filepath.Join(dd, d), 0o770); err != nil {

+ t.Fatal(err)

+ }

+ rec := `{"v":1,"host":"` + p1Host + `","incarnation":"` + h.inc + `","active_stamp":1,"retired_stamps":[]}`

+ if err := os.WriteFile(filepath.Join(dd, "hostguard", p1Host+".json"), []byte(rec), 0o660); err != nil {

+ t.Fatal(err)

+ }

+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {

+ if c.Inventory == nil {

+ c.Inventory = map[string]hostclaim.InventoryEntry{}

+ }

+ c.Inventory["ie-box"] = hostclaim.InventoryEntry{Layout: "flat", Paths: map[string]uint64{p1Host + ".db": 1, p1Host + ".db.gen": 1, p1Host: 1, p1Host + ".env": 1}}

+ })

+ h.commit(t)

+ var chore hostclaim.Obligation

+ for _, o := range h.claim(t).Obligations {

+ if o.Kind == "park-local" {

+ chore = o

+ }

+ if chore.Op != p1ID {

+ t.Fatalf("the commit queued no park-local of the move: %+v", h.claim(t).Obligations)

+ }

+ h.sourcePass(t)

+ owned := h.parkedJournal(t).Steps[finalStepLocal]

+ if owned == "" || owned == chore.Retirement {

+ t.Fatalf("the moves-owned retirement %q, park.Local's %q", owned, chore.Retirement)

+ }

+ for _, receipt := range []string{filepath.Join(chore.Retirement, ".journal.json"), filepath.Join(owned, localParkReceiptName)} {

+ if _, err := os.Stat(filepath.Join(dd, ".parked", h.inc, p1Host, receipt)); err != nil {

+ t.Fatalf("the receipt %s: %v", receipt, err)

+ }

+ found, unreadable := retirementsByOp(t, dd, h.inc, p1Host, p1ID)

+ want := []string{chore.Retirement, owned}

+ sort.Strings(want)

+ if strings.Join(found, " ") != strings.Join(want, " ") || len(unreadable) != 0 {

+ t.Fatalf("by op %s: found %v (want %v), no receipt %v", p1ID, found, want, unreadable)

+ }

+ if other, _ := retirementsByOp(t, dd, h.inc, p1Host, "mv_zzzzzzzzzzzzzzzzzzzzzzzzzz"); len(other) != 0 {

+ t.Fatalf("another op found the move's retirements: %v", other)

+ }

+}

diff --git a/internal/host/movesrevert.go b/internal/host/movesrevert.go

index d57869f32..57537e4dd 100644

--- a/internal/host/movesrevert.go

+++ b/internal/host/movesrevert.go

@@ -68,7 +68,7 @@ import (

// the claim steps to opened.

// - opened: the claim waits for the lane's park the revert queued in the

// target region (§7.11), which the target region's box runs

-// (revertLanePark) and the park engine acknowledges after its horizon.

+// (laneParkTarget) and the park engine acknowledges after its horizon.

// Then ReleaseMove to serving under this box, and MoveRevertClose:

// LastMove rolled_back, offline from cut_at to the reopen. The source's

// journal of the op is parked in the same pass; a crash before that

@@ -77,13 +77,13 @@ import (

// A claim this move already released (RequireReleasedMove: a crash before the

// record's close) only closes.

//

-// The target region's half (revertLanePark). A revert never flipped, so its

+// The target region's half (laneParkTarget). A revert never flipped, so its

// target box has neither the row nor a journal of the move; the participant

// rule answers it as the target on the state alone, and its one piece of work

// is the lane's park-replica the claim owes its region: the park engine over

// its own bucket archives moves/<host>/<op>/ under

// parked/<inc>/<host>/<ret>/moves/ and acknowledges once the horizon has

-// passed (exactly as Plan 5 Task 1 dispatches an abort's).

+// passed (laneParkTarget dispatches an abort's the same way, Plan 5 Task 1).

//

// Every record write goes through a provision transition (the record_seq

// rule); every claim write through a hostclaim transition.

@@ -738,6 +738,7 @@ func (r *revertRun) stuck(ctx context.Context, reason string, why error) (bool,

return false, err

}

r.w.logEvent(ctx, r.rec, "source-stuck")

+ r.w.pageStuck(ctx, r.rec, "source", reason)

nudgeSerial(ctx, r.s)

return true, nil

}

@@ -834,28 +835,40 @@ func (w *movesWorker) sweepRevertJournal(dataDir, host, op string, j *moves.Jour

return parkRevertJournal(dataDir, host, op, true)

}

-// revertLanePark is the target region's half of a revert (movesPass through

-// participantRound): the lane's park-replica the revert queued in this

-// region, run by the park engine over this box's own bucket until it is

-// acknowledged after its horizon. The view and the authorizer are Plan 5's

-// when wired, else the stand-ins: the source's region authoritative (a revert

-// never flipped), and only this move's reverting lane park.

-func (w *movesWorker) revertLanePark(ctx context.Context, s release.Store, rec provision.Record) error {

+// laneParkTarget is the target region's half of a move outcome that leaves

+// the move's own staging lane behind in its bucket (movesPass through

+// participantRound): the lane's park-replica the claim owes this region, run

+// by the park engine over this box's own bucket until it is acknowledged

+// after its horizon. It serves a revert (the claim reverting, any phase; the

+// park hostclaim.Revert queued) and an abort (moving/aborting; the park

+// hostclaim.Abort queued, Plan 5 Task 1); Task 3 adds the committed landing.

+// No phases means any phase of kind. The view and the authorizer are Plan

+// 5's when wired, else the stand-ins: the source's region authoritative

+// (neither outcome flipped), and only this move's lane park under kind and

+// its phases.

+func (w *movesWorker) laneParkTarget(ctx context.Context, s release.Store, rec provision.Record, kind string, phases ...string) error {

cfg := w.cfg

m := rec.Move

- if m == nil || m.Legacy() || m.State != provision.MoveReverting || m.To != cfg.Region || m.From == cfg.Region {

+ if m == nil || m.Legacy() || m.To != cfg.Region || m.From == cfg.Region {

return nil

}

+ notice := rec.Host + "#lane-" + kind

+ if kind == hostclaim.KindReverting {

+ notice = rec.Host + "#rv-lane"

+ }

if !revertLaneArmed(cfg) {

- w.notice(rec.Host+"#rv-lane", "moves: %s: the park of move %s's lane waits: this box's park engine is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.ID)

+ w.notice(notice, "moves: %s: the park of move %s's lane waits: this box's park engine is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.ID)

return nil

}

host, id := rec.Host, m.ID

+ inOp := func(op hostclaim.Op) bool {

+ return op.ID == id && op.Kind == kind && (len(phases) == 0 || containsString(phases, op.Phase))

+ }

obs, err := hostclaim.Read(ctx, s, host)

if err != nil {

return fmt.Errorf("read the host claim: %w", err)

}

- if obs.Absent || obs.Claim.Op.ID != id || obs.Claim.Op.Kind != hostclaim.KindReverting {

+ if obs.Absent || !inOp(obs.Claim.Op) {

return nil // the claim half has not followed yet, or it is over

}

if h := obs.Claim.Holder; h == nil || h.Incarnation != m.Incarnation {

@@ -882,9 +895,9 @@ func (w *movesWorker) revertLanePark(ctx context.Context, s release.Store, rec p

if auth == nil {

auth = func(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {

c := obs.Claim

- if obs.Absent || c == nil || c.Op.ID != id || c.Op.Kind != hostclaim.KindReverting || c.Op.Exec == cfg.Box ||

+ if obs.Absent || c == nil || !inOp(c.Op) || c.Op.Exec == cfg.Box ||

o.Op != id || o.Kind != "park-replica" || o.Region != cfg.Region || o.Park == nil || o.Park.Location != lane {

- return fmt.Errorf("%w: not this revert's lane park", park.ErrUnsafe)

+ return fmt.Errorf("%w: not this move's lane park", park.ErrUnsafe)

}

return nil

}

@@ -895,7 +908,7 @@ func (w *movesWorker) revertLanePark(ctx context.Context, s release.Store, rec p

err := e.Run(ctx, park.Work{Host: host, ID: o.ID, Op: id})

switch {

case errors.Is(err, park.ErrPending):

- w.notice(host+"#rv-lane", "moves: %s: move %s's lane is archived; its park is acknowledged after its horizon", host, id)

+ w.notice(notice, "moves: %s: move %s's lane is archived; its park is acknowledged after its horizon", host, id)

case errors.Is(err, park.ErrChanged):

case err != nil:

return fmt.Errorf("park move %s's lane: %w", id, err)

diff --git a/internal/host/movesrollback.go b/internal/host/movesrollback.go

index c9d5d85d0..d52afdedc 100644

--- a/internal/host/movesrollback.go

+++ b/internal/host/movesrollback.go

@@ -14,6 +14,7 @@ import (

"strings"

"time"

+ "github.com/carlosframework/platform/internal/alerting"

"github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/hostclaim"

"github.com/carlosframework/platform/internal/hostguard"

@@ -92,8 +93,9 @@ import (

// (ParkLeases), which production does not wire, so rollbackTargetArmed

// answers false there and the pass logs a standing notice. The decision is

// not gated (J10). The move's event log (moves.LogEvent, Plan 6) is the

-// LogEvent hook: nil in production, where the event is a log line. Nothing

-// here pages: the release deadline is the console's.

+// LogEvent hook: nil in production, where the event is a log line. The

+// stuck mark pages an operator (pageStuck, the Page hook); the release

+// deadline is the console's.

// rollbackStepsPerPass bounds the phases one pass runs.

const rollbackStepsPerPass = 8

@@ -135,6 +137,38 @@ func (w *movesWorker) logEvent(ctx context.Context, rec provision.Record, event

log.Printf("moves: %s: %s %s (the move log is Plan 6's)", rec.Host, id, event)

}

+// pageStuck pages an operator once a MoveMarkStuck has landed (moves spec

+// §13: stuck is an incident; K11). role is this box's part in the move,

+// "source" or "target", and reason the recorded stuck sentence. The page

+// names the host, the move, the box and the reason, and nothing else: no

+// path, no error text, no secret. Delivery is best effort: a replay that

+// lands the same mark again pages again, and a crash between the mark and

+// the send, or a send that fails (logged, never retried), pages nothing.

+// Unwired (nil Page), nothing pages; a failed page never stops the arm.

+func (w *movesWorker) pageStuck(ctx context.Context, rec provision.Record, role, reason string) {

+ f := w.cfg.Page

+ if f == nil {

+ return

+ }

+ id := ""

+ if rec.Move != nil {

+ id = rec.Move.ID

+ }

+ subject, body := stuckPage(rec.Host, id, w.cfg.Box, role, reason)

+ if err := f(ctx, alerting.LevelIncident, subject, body); err != nil {

+ log.Printf("moves: %s: the page for the stuck mark of %s: %v", rec.Host, id, err)

+ }

+}

+

+// stuckPage is the stuck mark's page: the subject (MV-N313) and the body

+// (MV-N314).

+func stuckPage(host, id, box, role, reason string) (subject, body string) {

+ subject = fmt.Sprintf("carlos move stuck: %s", host)

+ body = fmt.Sprintf("The move %s of %s stopped for an operator on %s, its %s: %s. Run carlos move resolve --host %s --retry once the cause is fixed.",

+ id, host, box, role, reason, host)

+ return subject, body

+}

+

// rollBackLanding is the rollback decision on the target (§7.8; J10, J11):

// a landing marked for a rollback, or past landingDeadlineOf at any phase,

// is rolled back by MoveRollBack, and the claim follows. done is true when

@@ -548,8 +582,7 @@ func (r *rollbackRun) claimReleased(ctx context.Context, evidence string) error

return err

}

holder := obs.Claim.Holder

- scope := hostclaim.ParkScope{Location: strings.TrimSuffix(holder.Location, "/"), Incarnation: holder.Incarnation,

- EvidenceAt: report.CheckedAt.UTC(), RequestLimit: cutRequestLimit}

+ scope := moves.TargetLocationScope(*holder, report.CheckedAt)

if _, err := hostclaim.Update(ctx, r.s, obs, hostclaim.TargetReleased(r.g.ID, cfg.Box, cfg.Region, evidence, scope, obligationID)); err != nil {

return fmt.Errorf("the claim half of the release: %w", err)

}

@@ -862,6 +895,7 @@ func (r *rollbackRun) stuck(ctx context.Context, refused *returnRefusal) (bool,

return false, fmt.Errorf("mark the return stuck: %w", err)

}

r.w.logEvent(ctx, r.rec, "target-stuck")

+ r.w.pageStuck(ctx, r.rec, "target", refused.reason)

nudgeSerial(ctx, r.s)

return true, nil

}

@@ -1084,8 +1118,7 @@ func (r *rollbackRun) remote(ctx context.Context, obs hostclaim.Observed, j *mov

if err != nil {

return false, err

}

- scope := hostclaim.ParkScope{Location: moves.ScopeLocation(host, id), Incarnation: obs.Claim.Holder.Incarnation,

- EvidenceAt: staged.Bound().UTC(), RequestLimit: cutRequestLimit}

+ scope := moves.StagingScope(host, id, obs.Claim.Holder.Incarnation, staged.Bound())

if _, err := hostclaim.Update(ctx, r.s, obs, hostclaim.QueueStagingPark(id, cfg.Region, scope, obligationID)); err != nil {

return false, fmt.Errorf("queue the move's lane park: %w", err)

}

diff --git a/internal/host/movesrollback_test.go b/internal/host/movesrollback_test.go

index 022a5461f..43581e789 100644

--- a/internal/host/movesrollback_test.go

+++ b/internal/host/movesrollback_test.go

@@ -42,6 +42,7 @@ type rollbackHarness struct {

tchecks int // the target's final checkpoints

tpos uint64 // the target's local position

events []string

+ pages pagesSeen // the operator pages (moveCfg.Page), the target's and the source's

rbAt string

rbDone bool

rbHits []string

@@ -135,6 +136,7 @@ func (h *rollbackHarness) rollbackCfg(t *testing.T) moveCfg {

return true, nil

}

c.LogEvent = func(_ context.Context, _ provision.Record, event string) { h.events = append(h.events, event) }

+ c.Page = h.pages.hook()

c.rbFault = func(point string) error {

h.rbHits = append(h.rbHits, point)

if h.rbHook != nil {

diff --git a/internal/host/movesstuckpage_test.go b/internal/host/movesstuckpage_test.go

new file mode 100644

index 000000000..786283953

--- /dev/null

+++ b/internal/host/movesstuckpage_test.go

@@ -0,0 +1,115 @@

+package host

+

+import (

+ "context"

+ "os"

+ "path/filepath"

+ "strings"

+ "testing"

+

+ "github.com/carlosframework/platform/internal/provision"

+)

+

+// pagesSeen records the pages a box arm sends through moveCfg.Page.

+type pagesSeen []struct{ level, subject, body string }

+

+func (p *pagesSeen) hook() func(context.Context, string, string, string) error {

+ return func(_ context.Context, level, subject, body string) error {

+ *p = append(*p, struct{ level, subject, body string }{level, subject, body})

+ return nil

+ }

+}

+

+// stuckPageBody is the page the record's stuck mark sends (MV-N314), spelled

+// out here rather than built by the code under test.

+func stuckPageBody(box, role, reason string) string {

+ return "The move " + p1ID + " of " + p1Host + " stopped for an operator on " + box + ", its " + role + ": " + reason +

+ ". Run carlos move resolve --host " + p1Host + " --retry once the cause is fixed."

+}

+

+// pagedOnce asserts exactly one incident page for the stuck mark, carrying

+// no path and no secret.

+func pagedOnce(t *testing.T, pages pagesSeen, box, role, reason string) {

+ t.Helper()

+ if len(pages) != 1 {

+ t.Fatalf("pages %+v, want one", pages)

+ }

+ p := pages[0]

+ if p.level != "incident" || p.subject != "carlos move stuck: "+p1Host || p.body != stuckPageBody(box, role, reason) {

+ t.Fatalf("page %+v\nwant body %q", p, stuckPageBody(box, role, reason))

+ }

+ if strings.Contains(p.subject+p.body, "/") {

+ t.Fatalf("a page carries a path: %+v", p)

+ }

+}

+

+// TestStuckMarksPageAnOperator (moves spec §13, K11): wherever a box arm's

+// MoveMarkStuck lands, the operator is paged once, as an incident, with the

+// recorded reason; a pass over the stuck record pages nothing more. The

+// hook is nil in the other tests and in production until armMovesPass sets

+// it to PageOperator.

+func TestStuckMarksPageAnOperator(t *testing.T) {

+ t.Run("the target's return", func(t *testing.T) {

+ h := newRollbackHarness(t)

+ db := provision.DefaultDBPath(h.tdir, p1Host)

+ if err := os.Rename(db, db+".aside"); err != nil {

+ t.Fatal(err)

+ }

+ h.rollBack(t)

+ h.rollbackPass(t)

+ h.stuckAtReturning(t, provision.StuckReturnTargetDatabase)

+ h.rollbackPass(t)

+ pagedOnce(t, h.pages, "uk-box", "target", provision.StuckReturnTargetDatabase)

+ })

+ t.Run("the source's recovery", func(t *testing.T) {

+ h := newRecoverHarness(t, false)

+ h.released(t)

+ if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("changed on the source"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ h.pages = nil

+ h.srcPass(t)

+ if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck || m.StuckReason != provision.StuckBaselineMismatch {

+ t.Fatalf("not stuck: %+v", m)

+ }

+ h.srcPass(t)

+ pagedOnce(t, h.pages, "ie-box", "source", provision.StuckBaselineMismatch)

+ })

+ t.Run("the source's dirty return install", func(t *testing.T) {

+ h := newRecoverHarness(t, true)

+ h.released(t)

+ sentinel := filepath.Join(h.sdir, p1Host, "sentinel.txt")

+ h.landFault = func(p string) error {

+ if p == "after-"+landStepRetired {

+ if err := os.MkdirAll(filepath.Dir(sentinel), 0o755); err != nil {

+ t.Fatal(err)

+ }

+ return os.WriteFile(sentinel, []byte("not the return's"), 0o644)

+ }

+ return nil

+ }

+ h.pages = nil

+ h.srcPass(t)

+ if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied {

+ t.Fatalf("not stuck: %+v", m)

+ }

+ h.srcPass(t)

+ pagedOnce(t, h.pages, "ie-box", "source", provision.StuckReturnFilesOccupied)

+ })

+ t.Run("the source's revert", func(t *testing.T) {

+ h := newRevertHarness(t)

+ h.fromStopped(t)

+ h.arm(t)

+ var pages pagesSeen

+ h.cfg.Page = pages.hook()

+ if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host, "upload.txt"), []byte("changed while stopped"), 0o644); err != nil {

+ t.Fatal(err)

+ }

+ h.pass(t)

+ if m := h.move(t); m.State != provision.MoveStuck || m.StuckReason != provision.StuckBaselineMismatch {

+ t.Fatalf("not stuck: %+v", m)

+ }

+ h.pass(t)

+ pagedOnce(t, pages, "ie-box", "source", provision.StuckBaselineMismatch)

+ })

+}

diff --git a/internal/hostclaim/move.go b/internal/hostclaim/move.go

index 4215bfcec..4640e7e71 100644

--- a/internal/hostclaim/move.go

+++ b/internal/hostclaim/move.go

@@ -17,7 +17,7 @@ import (

const (

PhaseMoving = "" // moving, before the seal

PhaseSealed = "sealed" // moving, after the cut's staged phase

- PhaseAborting = "aborting" // moving, the interim abort has begun

+ PhaseAborting = "aborting" // moving, the abort has begun (Abort): the lane park is queued

)

// Journal keys the cut and the flip write on the claim's op (moves spec

@@ -133,22 +133,24 @@ func RequireMoveOp(opID, kind, phase string) func(*Claim) error {

}

// ReleaseMove is K8: a move hands the host back to serving. It is legal

-// from the interim abort (moving before the seal, phase "" or "aborting"),

-// and from the end of each move outcome, read from op.phase (J1, J6): a

-// committed landing, an opened rollback (restoring) and an opened revert

-// (reverting). In those three the executor named must be the one executing.

-// It is legal only once no obligation under this op remains. The serving op

-// keeps exec and records which move released it: operation-bound completion

-// evidence for the abort's crash boundary (RequireReleasedMove).

+// from moving before the seal (phase "": the request door's compensation for

+// a move whose record start failed), and from the end of each move outcome,

+// read from op.phase (J1, J6): a committed landing, an opened rollback

+// (restoring) and an opened revert (reverting). In those three the executor

+// named must be the one executing. An aborting op is ReleaseAbort's (Plan 5

+// ruling K7), never this. It is legal only once no obligation under this op

+// remains. The serving op keeps exec and records which move released it:

+// operation-bound completion evidence for the abort's crash boundary

+// (RequireReleasedMove).

func ReleaseMove(opID, execBox string) func(*Claim) error {

return func(c *Claim) error {

if err := RequireOp(opID)(c); err != nil {

return err

}

- abort := c.Op.Kind == KindMoving && (c.Op.Phase == PhaseMoving || c.Op.Phase == PhaseAborting)

+ door := c.Op.Kind == KindMoving && c.Op.Phase == PhaseMoving

ended := (c.Op.Kind == KindLanding && c.Op.Phase == PhaseCommitted) ||

((c.Op.Kind == KindRestoring || c.Op.Kind == KindReverting) && c.Op.Phase == PhaseOpened)

- if !abort && !ended {

+ if !door && !ended {

return fmt.Errorf("%w: op is %s/%q, a release needs moving before the seal, a committed landing or an opened recovery", ErrWrongState, c.Op.Kind, c.Op.Phase)

}

if execBox == "" {

@@ -157,16 +159,93 @@ func ReleaseMove(opID, execBox string) func(*Claim) error {

if ended && c.Op.Exec != execBox {

return fmt.Errorf("%w: exec is %q, the release names %q", ErrWrongState, c.Op.Exec, execBox)

}

- for _, o := range c.Obligations {

- if o.Op == opID {

- return fmt.Errorf("%w: %s %s%s is still owed under this move", ErrPendingChore, o.Kind, o.Region, o.Box)

+ return releaseToServing(c, opID, execBox)

+ }

+}

+

+// releaseToServing is every move release's last step: legal only once no

+// obligation under the op remains; the serving op keeps the executor and

+// records the move it released.

+func releaseToServing(c *Claim, opID, execBox string) error {

+ for _, o := range c.Obligations {

+ if o.Op == opID {

+ return fmt.Errorf("%w: %s %s%s is still owed under this move", ErrPendingChore, o.Kind, o.Region, o.Box)

+ }

+ c.Op = Op{Kind: KindServing, Exec: execBox, Journal: map[string]string{journalReleasedMove: opID}}

+ return nil

+}

+

+// Abort is the claim half of a move's abort (moves spec §7.11, joint N13;

+// Plan 5 Task 1): this move's moving op before the seal, with the source

+// still executing, goes to moving/aborting, and the park of its own staging

+// lane is queued in the same write for the target region, owned by that

+// region (QueueStagingPark's chore, index D6). The source cannot read or

+// park the target's moves/*, so the lane is the target region's to retire;

+// execution stays with the source, which serves throughout.

+//

+// The scope is moves/<host>/<op> with the holder's incarnation and the

+// journalled abort bound the caller computed (moves.AbortScope, ruling K5).

+// A replay finds the op already aborting: it is a no-op when the same lane

+// park is owed, or when none is (the target already acknowledged it), and

+// is refused when a lane park with another scope is owed. It allocates no

+// ids. A cut past requested cannot abort, and neither can an op whose

+// execution was given up (that is a revert).

+func Abort(opID, targetRegion string, scope ParkScope, newID func() string) func(*Claim) error {

+ return func(c *Claim) error {

+ if err := RequireOp(opID)(c); err != nil {

+ return err

+ }

+ if c.Op.Kind != KindMoving || (c.Op.Phase != PhaseMoving && c.Op.Phase != PhaseAborting) || c.Op.Exec == "" {

+ return fmt.Errorf("%w: op is %s/%q exec %q, an abort needs moving before the seal with the source executing", ErrWrongState, c.Op.Kind, c.Op.Phase, c.Op.Exec)

+ }

+ if targetRegion == "" || c.Op.Intent["to"] != targetRegion {

+ return fmt.Errorf("%w: %q is not the move's target region", ErrWrongState, targetRegion)

+ }

+ var err error

+ if scope, err = parkQueueGuard(c, opID, targetRegion, scope, newID); err != nil {

+ return err

+ }

+ // Restates moves.ScopeLocation(host, op), as QueueStagingPark does.

+ if scope.Location != "moves/"+c.Host+"/"+opID {

+ return fmt.Errorf("%w: an abort parks this move's own lane, moves/%s/%s", ErrMalformed, c.Host, opID)

+ }

+ if c.Op.Phase == PhaseAborting {

+ for _, o := range c.Obligations {

+ if o.Op == opID && o.Kind == "park-replica" && o.Region == targetRegion && o.Park != nil && isStagingLocation(o.Park.Location) && !sameParkScope(*o.Park, scope) {

+ return fmt.Errorf("%w: %s already owes its lane park in %s with another scope", ErrWrongState, opID, targetRegion)

+ }

}

+ return nil

+ }

+ if _, err := queueReplicaPark(c, opID, targetRegion, scope, newID); err != nil {

+ return err

}

- c.Op = Op{Kind: KindServing, Exec: execBox, Journal: map[string]string{journalReleasedMove: opID}}

+ c.Op.Phase = PhaseAborting

return nil

}

+// ReleaseAbort is the abort's release (ruling K7): from moving/aborting

+// only, so always after Abort queued the lane park, with the named source

+// still executing, and only once no obligation under the op remains (the

+// target region acknowledged the lane park). The serving op keeps the

+// source as its executor and records the move it released.

+func ReleaseAbort(opID, sourceBox string) func(*Claim) error {

+ return func(c *Claim) error {

+ if err := RequireMoveOp(opID, KindMoving, PhaseAborting)(c); err != nil {

+ return err

+ }

+ if sourceBox == "" {

+ return fmt.Errorf("%w: a release names the executor", ErrMalformed)

+ }

+ if c.Op.Exec != sourceBox {

+ return fmt.Errorf("%w: exec is %q, the abort's release names %q", ErrWrongState, c.Op.Exec, sourceBox)

+ }

+ return releaseToServing(c, opID, sourceBox)

+ }

+}

+

// RequireReleasedMove is the replay rule for a crashed abort: nil when

// the claim is serving and its op records that move opID released it,

// else ErrStaleOp. A plain serving claim, or one released by another

@@ -368,7 +447,10 @@ func sameParkScope(a, b ParkScope) bool {

//

// Each helper changes obligations only, is idempotent per its key (a replay

// allocates no ids), and refuses a replay that names the same chore with a

-// different scope rather than overwriting it.

+// different scope rather than overwriting it. The scopes themselves come

+// from internal/moves (SourceLocationScope, StagingScope, AbortScope,

+// ReturnLaneScope), the one source for the box and the console; moves

+// imports hostclaim, so they cannot be called from here.

// parkQueueGuard is the check every helper shares: the op is current, the

// claim has its holder, the scope is valid for this host and names this

diff --git a/internal/hostclaim/move_abort_test.go b/internal/hostclaim/move_abort_test.go

new file mode 100644

index 000000000..69eb51cc6

--- /dev/null

+++ b/internal/hostclaim/move_abort_test.go

@@ -0,0 +1,188 @@

+package hostclaim

+

+import (

+ "errors"

+ "strings"

+ "testing"

+ "time"

+)

+

+// Abort (moves spec §7.11, Plan 5 Task 1): moving before the seal goes to

+// moving/aborting, and the lane's park-replica is queued in the same write,

+// owned by the target region and scoped to moves/<host>/<op>. Execution

+// stays with the source. A replay allocates no ids; a replay with another

+// scope is refused, not overwritten.

+func TestAbortQueuesTheTargetsLanePark(t *testing.T) {

+ c, scope := parkFixture(t)

+ lane := scope("moves/" + c.Host + "/" + moveA)

+ ids := 0

+ newID := func() string { ids++; return "id" + strings.Repeat("x", ids) }

+ if err := Abort(moveA, "uk", lane, newID)(c); err != nil {

+ t.Fatal(err)

+ }

+ if c.Op.Kind != KindMoving || c.Op.Phase != PhaseAborting || c.Op.Exec != "ie-box" || c.Op.ID != moveA {

+ t.Fatalf("op after the abort: %+v", c.Op)

+ }

+ if len(c.Obligations) != 1 {

+ t.Fatalf("obligations: %+v", c.Obligations)

+ }

+ o := c.Obligations[0]

+ if o.Op != moveA || o.Kind != "park-replica" || o.Region != "uk" || o.Owner != "region:uk" || o.Box != "" || o.Park == nil || !sameParkScope(*o.Park, lane) || o.Retirement == "" {

+ t.Fatalf("the lane park: %+v", o)

+ }

+ c.Gen++

+ if err := c.Validate(); err != nil {

+ t.Fatalf("the claim after the abort must validate: %v", err)

+ }

+ // A replay with the same scope is a no-op and allocates nothing.

+ before, n := string(mustJSON(t, c)), ids

+ if err := Abort(moveA, "uk", lane, newID)(c); err != nil || ids != n || string(mustJSON(t, c)) != before {

+ t.Fatalf("replay: %v, ids %d to %d", err, n, ids)

+ }

+ // Another scope for the same lane is refused, not overwritten.

+ later := lane

+ later.EvidenceAt = lane.EvidenceAt.Add(time.Second)

+ if err := Abort(moveA, "uk", later, newID)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != before {

+ t.Fatalf("replay with another scope: %v", err)

+ }

+ // Once the target acknowledged the park, a late replay queues nothing.

+ if err := CompleteObligation(moveA, o.ID)(c); err != nil {

+ t.Fatal(err)

+ }

+ n = ids

+ if err := Abort(moveA, "uk", lane, newID)(c); err != nil || len(c.Obligations) != 0 || ids != n {

+ t.Fatalf("replay after the acknowledgement: %v, obligations %+v", err, c.Obligations)

+ }

+}

+

+// Abort is legal only on this move's moving op before the seal, with the

+// source still executing, for the move's own target region and its own

+// lane; every refusal leaves the claim as it was and allocates nothing.

+func TestAbortRefusals(t *testing.T) {

+ for name, tc := range map[string]struct {

+ edit func(c *Claim)

+ op string

+ region string

+ loc func(c *Claim) string

+ inc string

+ zero bool

+ nilID bool

+ want error

+ }{

+ "another op": {op: moveB, want: ErrStaleOp},

+ "sealed": {edit: func(c *Claim) { c.Op.Phase = PhaseSealed }, want: ErrWrongState},

+ "execution given up": {edit: func(c *Claim) { c.Op.Exec = "" }, want: ErrWrongState},

+ "a landing": {edit: func(c *Claim) { c.Op.Kind = KindLanding }, want: ErrWrongState},

+ "a revert": {edit: func(c *Claim) { c.Op.Kind = KindReverting }, want: ErrWrongState},

+ "another region": {region: "se", want: ErrWrongState},

+ "no region": {region: "-", want: ErrWrongState},

+ "the live location": {loc: func(c *Claim) string { return "instances/" + c.Host + "/" + c.Holder.Incarnation }, want: ErrMalformed},

+ "the staging prefix": {loc: func(c *Claim) string { return "moves/" + c.Host }, want: ErrMalformed},

+ "another move's lane": {loc: func(c *Claim) string { return "moves/" + c.Host + "/" + moveB }, want: ErrMalformed},

+ "another incarnation": {inc: strings.Repeat("a", 64), want: ErrMalformed},

+ "no bound": {zero: true, want: ErrMalformed},

+ "no way to mint ids": {nilID: true, want: ErrMalformed},

+ "a dirty claim (no move)": {edit: func(c *Claim) { c.Op = Op{Kind: KindServing, Exec: "ie-box"} }, want: ErrStaleOp},

+ } {

+ t.Run(name, func(t *testing.T) {

+ c, scope := parkFixture(t)

+ if tc.edit != nil {

+ tc.edit(c)

+ }

+ location := "moves/" + c.Host + "/" + moveA

+ if tc.loc != nil {

+ location = tc.loc(c)

+ }

+ s := scope(location)

+ if tc.inc != "" {

+ s.Incarnation = tc.inc

+ }

+ if tc.zero {

+ s.EvidenceAt = time.Time{}

+ }

+ op, region := moveA, "uk"

+ if tc.op != "" {

+ op = tc.op

+ }

+ if tc.region == "-" {

+ region = ""

+ } else if tc.region != "" {

+ region = tc.region

+ }

+ ids := 0

+ newID := func() string { ids++; return "x" }

+ if tc.nilID {

+ newID = nil

+ }

+ before := string(mustJSON(t, c))

+ err := Abort(op, region, s, newID)(c)

+ if !errors.Is(err, tc.want) {

+ t.Fatalf("err %v, want %v", err, tc.want)

+ }

+ if string(mustJSON(t, c)) != before || ids != 0 {

+ t.Fatalf("a refused abort changed the claim or minted %d ids", ids)

+ }

+ })

+ }

+}

+

+// ReleaseAbort (ruling K7) is the abort's only release: from moving/aborting,

+// with the source still executing, once no chore of the op remains. The

+// serving op keeps the source as its executor and records the move it

+// released. ReleaseMove's moving/"" stays the door's, and it no longer

+// releases an aborting op.

+func TestReleaseAbortFollowsTheLanePark(t *testing.T) {

+ c, scope := parkFixture(t)

+ lane := scope("moves/" + c.Host + "/" + moveA)

+ // Before the abort's CAS there is nothing to release from.

+ if err := ReleaseAbort(moveA, "ie-box")(c); !errors.Is(err, ErrWrongState) {

+ t.Fatalf("release before the abort: %v", err)

+ }

+ if err := Abort(moveA, "uk", lane, counter())(c); err != nil {

+ t.Fatal(err)

+ }

+ before := string(mustJSON(t, c))

+ if err := ReleaseAbort(moveA, "ie-box")(c); !errors.Is(err, ErrPendingChore) || string(mustJSON(t, c)) != before {

+ t.Fatalf("release with the lane park owed: %v", err)

+ }

+ if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != before {

+ t.Fatalf("ReleaseMove released an aborting op: %v", err)

+ }

+ if err := CompleteObligation(moveA, c.Obligations[0].ID)(c); err != nil {

+ t.Fatal(err)

+ }

+ for name, rel := range map[string]func(*Claim) error{

+ "another box": ReleaseAbort(moveA, "uk-box"),

+ "no box": ReleaseAbort(moveA, ""),

+ "another op": ReleaseAbort(moveB, "ie-box"),

+ "ReleaseMove's": ReleaseMove(moveA, "ie-box"),

+ } {

+ mid := string(mustJSON(t, c))

+ if err := rel(c); err == nil || string(mustJSON(t, c)) != mid {

+ t.Fatalf("%s: released (%v)", name, err)

+ }

+ if err := ReleaseAbort(moveA, "ie-box")(c); err != nil {

+ t.Fatal(err)

+ }

+ if c.Op.Kind != KindServing || c.Op.Exec != "ie-box" || c.Op.ID != "" || c.Op.Phase != "" || len(c.Op.Participants) != 0 || len(c.Op.Journal) != 1 {

+ t.Fatalf("op after the release: %+v", c.Op)

+ }

+ if err := RequireReleasedMove(moveA)(c); err != nil {

+ t.Fatal(err)

+ }

+ c.Gen++

+ if err := c.Validate(); err != nil {

+ t.Fatalf("the claim after the release must validate: %v", err)

+ }

+ // Execution moved away (it never does in an abort) refuses.

+ d, scope := parkFixture(t)

+ if err := Abort(moveA, "uk", scope("moves/"+d.Host+"/"+moveA), counter())(d); err != nil {

+ t.Fatal(err)

+ }

+ d.Obligations = nil

+ d.Op.Exec = "uk-box"

+ if err := ReleaseAbort(moveA, "ie-box")(d); !errors.Is(err, ErrWrongState) {

+ t.Fatalf("exec is not the source: %v", err)

+ }

+}

diff --git a/internal/hostclaim/move_test.go b/internal/hostclaim/move_test.go

index 465dff847..078a828e0 100644

--- a/internal/hostclaim/move_test.go

+++ b/internal/hostclaim/move_test.go

@@ -147,7 +147,7 @@ func TestTouchAppendsOnceUnderTheOp(t *testing.T) {

func TestReleaseMoveFromMovingNeedsNoChores(t *testing.T) {

now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)

- for _, phase := range []string{PhaseMoving, PhaseAborting} {

+ for _, phase := range []string{PhaseMoving} {

c := movingTestClaim(t)

if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil {

t.Fatal(err)

@@ -176,6 +176,14 @@ func TestReleaseMoveFromMovingNeedsNoChores(t *testing.T) {

t.Fatalf("claim after release must validate: %v", err)

}

+ // Not from an aborting move (ruling K7: that is ReleaseAbort's, after

+ // the lane park the abort queued).

+ a := movingTestClaim(t)

+ _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(a)

+ a.Op.Phase = PhaseAborting

+ if err := ReleaseMove(moveA, "ie-box")(a); !errors.Is(err, ErrWrongState) {

+ t.Fatalf("release from aborting: %v", err)

+ }

// Not from serving, and not from a sealed move.

s := movingTestClaim(t)

if err := ReleaseMove(moveA, "ie-box")(s); !errors.Is(err, ErrStaleOp) {

@@ -874,7 +882,7 @@ func TestLocationParkOnlyWhereTheRegionIsRetired(t *testing.T) {

"rolled-back target after the source": {KindRestoring, "claimed", "uk", "uk-box", true},

"source before the cut": {KindMoving, PhaseMoving, "ie", "ie-box", false},

"source sealed": {KindMoving, PhaseSealed, "ie", "ie-box", false},

- "source in the interim abort": {KindMoving, PhaseAborting, "ie", "ie-box", false},

+ "source in the abort": {KindMoving, PhaseAborting, "ie", "ie-box", false},

"source while the landing checks": {KindLanding, "checking", "ie", "ie-box", false},

"source while the landing probes": {KindLanding, "probing", "ie", "ie-box", false},

"source while reverting": {KindReverting, "evidence", "ie", "ie-box", false},

@@ -1782,11 +1790,10 @@ func TestReleaseMoveWaitsForObligations(t *testing.T) {

states := map[string]struct {

kind, phase, exec string

}{

- "committed landing": {KindLanding, PhaseCommitted, "uk-box"},

- "opened rollback": {KindRestoring, PhaseOpened, "ie-box"},

- "opened revert": {KindReverting, PhaseOpened, "ie-box"},

- "interim abort": {KindMoving, PhaseAborting, "ie-box"},

- "interim abort before cut": {KindMoving, PhaseMoving, "ie-box"},

+ "committed landing": {KindLanding, PhaseCommitted, "uk-box"},

+ "opened rollback": {KindRestoring, PhaseOpened, "ie-box"},

+ "opened revert": {KindReverting, PhaseOpened, "ie-box"},

+ "the door, before the seal": {KindMoving, PhaseMoving, "ie-box"},

}

for name, st := range states {

c := targetWroteLanding(t)

@@ -1832,6 +1839,7 @@ func TestReleaseMoveWaitsForObligations(t *testing.T) {

"reverting probed": {KindReverting, PhaseProbed},

"reverting, rev only": {KindReverting, PhaseSealed},

"moving sealed": {KindMoving, PhaseSealed},

+ "moving aborting": {KindMoving, PhaseAborting},

} {

c := targetWroteLanding(t)

c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, "ie-box"

diff --git a/internal/moves/abort.go b/internal/moves/abort.go

new file mode 100644

index 000000000..60c136e20

--- /dev/null

+++ b/internal/moves/abort.go

@@ -0,0 +1,39 @@

+package moves

+

+import (

+ "time"

+

+ "github.com/carlosframework/platform/internal/provision"

+)

+

+// AbortAfter is how long a Phase 1 move may stay preparing before it aborts

+// itself (spec §7.11: "a move whose readiness never comes aborts itself

+// after a bound, so an unreachable target never needs a human"; Plan 5

+// ruling K18). §13 pages an operator at 60 minutes with no promotable round;

+// the abort comes at six hours. The console's sweep (Plan 5 Task 7) CASes

+// provision.MoveGuardedAbort when AbortDue says so; nothing here writes.

+const AbortAfter = 6 * time.Hour

+

+// AbortDue reports whether rec's move is due its automatic abort at now: a

+// Phase 1 move (not legacy) still preparing, AbortAfter or more after the

+// later of its RequestedAt and its ReadyAt.

+//

+// The bound is measured from max(RequestedAt, ReadyAt) (ruling K18), not

+// from Move.UpdatedAt, which every record write stamps and so says nothing

+// about readiness. Today a preparing move has no ReadyAt (nothing demotes a

+// ready move to preparing), so the bound runs from the request; taking the

+// later of the two keeps it right should a demotion ever stamp one. A ready

+// move is not due: readiness came, and the cut is the operator's (or the

+// automatic cut's). A move with no request time is never due: there is no

+// evidence to measure from.

+func AbortDue(rec provision.Record, now time.Time) bool {

+ m := rec.Move

+ if m == nil || m.Legacy() || m.State != provision.MovePreparing || m.RequestedAt.IsZero() {

+ return false

+ }

+ from := m.RequestedAt

+ if m.ReadyAt != nil && m.ReadyAt.After(from) {

+ from = *m.ReadyAt

+ }

+ return !now.Before(from.Add(AbortAfter))

+}

diff --git a/internal/moves/abort_test.go b/internal/moves/abort_test.go

new file mode 100644

index 000000000..af8d45594

--- /dev/null

+++ b/internal/moves/abort_test.go

@@ -0,0 +1,46 @@

+package moves

+

+import (

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/provision"

+)

+

+// The auto-abort bound (spec §7.11, ruling K18): a Phase 1 move still

+// preparing AbortAfter after max(RequestedAt, ReadyAt) is due; one that is

+// not yet there, or is in any other state, or is a legacy move, is not.

+func TestAbortDue(t *testing.T) {

+ if AbortAfter != 6*time.Hour {

+ t.Fatalf("AbortAfter = %v, want 6h", AbortAfter)

+ }

+ req := time.Date(2026, 10, 9, 8, 0, 0, 0, time.UTC)

+ rec := func(state string, ready *time.Time) provision.Record {

+ return provision.Record{Host: "a.go.example.com", Move: &provision.MoveState{ID: "mv_abcdefghijklmnopqrstuvwxyz", State: state, RequestedAt: req, ReadyAt: ready}}

+ }

+ at := func(d time.Duration) time.Time { return req.Add(d) }

+ later := req.Add(2 * time.Hour)

+ for name, c := range map[string]struct {

+ rec provision.Record

+ now time.Time

+ want bool

+ }{

+ "preparing, just short": {rec(provision.MovePreparing, nil), at(AbortAfter - time.Second), false},

+ "preparing, at the bound": {rec(provision.MovePreparing, nil), at(AbortAfter), true},

+ "preparing, well past": {rec(provision.MovePreparing, nil), at(3 * AbortAfter), true},

+ "the bound is from the later time": {rec(provision.MovePreparing, &later), at(AbortAfter + time.Hour), false},

+ "and reached from it": {rec(provision.MovePreparing, &later), later.Add(AbortAfter), true},

+ "a non-UTC clock": {rec(provision.MovePreparing, nil), at(AbortAfter).In(time.FixedZone("x", -7*3600)), true},

+ "ready": {rec(provision.MoveReady, &later), at(10 * AbortAfter), false},

+ "cutting": {rec(provision.MoveCutting, &later), at(10 * AbortAfter), false},

+ "aborting": {rec(provision.MoveAborting, nil), at(10 * AbortAfter), false},

+ "landing": {rec(provision.MoveLanding, nil), at(10 * AbortAfter), false},

+ "legacy": {provision.Record{Move: &provision.MoveState{State: provision.MovePreparing, RequestedAt: req}}, at(10 * AbortAfter), false},

+ "no move": {provision.Record{}, at(10 * AbortAfter), false},

+ "no request time": {provision.Record{Move: &provision.MoveState{ID: "mv_abcdefghijklmnopqrstuvwxyz", State: provision.MovePreparing}}, at(10 * AbortAfter), false},

+ } {

+ if got := AbortDue(c.rec, c.now); got != c.want {

+ t.Errorf("%s: AbortDue = %v, want %v", name, got, c.want)

+ }

+}

diff --git a/internal/moves/parkscope.go b/internal/moves/parkscope.go

new file mode 100644

index 000000000..f1611d6f0

--- /dev/null

+++ b/internal/moves/parkscope.go

@@ -0,0 +1,74 @@

+package moves

+

+import (

+ "strings"

+ "time"

+

+ "github.com/carlosframework/platform/internal/hostclaim"

+)

+

+// The park scopes a move's chores carry (claim rev 6.23 §2, INV-22; Plan 5

+// ruling K8). These helpers are the one place the box and the console build

+// them. Each takes its genuine bound as an argument and never reads the

+// clock: the scope is bound once, in the CAS that queues the chore, and a

+// retry or a restart that computes a later bound is refused

+// (hostclaim.QueueStagingPark and friends, park.Engine.Bind), so the

+// horizon is the evidence's, never the cleanup's start. The bound is stored

+// in UTC.

+

+// ParkRequestLimit is every moves park's request limit: the longest a

+// staging request runs (lease's stagingRequestTimeout).

+const ParkRequestLimit = 120 * time.Second

+

+// SourceLocationScope is the source region's location park a commit queues:

+// holder.location, in the holder's form without a trailing slash, and the

+// holder's incarnation. Its horizon starts at the flip (ruling H7): the flip

+// comes after ExecToNone, after which the source never writes its location

+// again.

+func SourceLocationScope(holder hostclaim.Holder, flippedAt time.Time) hostclaim.ParkScope {

+ return locationScope(holder, flippedAt)

+}

+

+// TargetLocationScope is a rolled-back target region's location park, queued

+// when the target releases (hostclaim.TargetReleased): holder.location and

+// the holder's incarnation. Its horizon is the target's stop report's

+// CheckedAt, after which the target writes its location no more.

+func TargetLocationScope(holder hostclaim.Holder, stopCheckedAt time.Time) hostclaim.ParkScope {

+ return locationScope(holder, stopCheckedAt)

+}

+

+// ResidueScope is the location a landing's earlier-stay residue parks

+// (hostclaim.QueueResidueParks) are scoped to: holder.location and the

+// holder's incarnation. Its horizon starts at the flip, like the source's.

+func ResidueScope(holder hostclaim.Holder, flippedAt time.Time) hostclaim.ParkScope {

+ return locationScope(holder, flippedAt)

+}

+

+func locationScope(holder hostclaim.Holder, bound time.Time) hostclaim.ParkScope {

+ return hostclaim.ParkScope{Location: strings.TrimSuffix(holder.Location, "/"), Incarnation: holder.Incarnation, EvidenceAt: bound.UTC(), RequestLimit: ParkRequestLimit}

+}

+

+// StagingScope is the park of a move's own staging lane, moves/<host>/<op>,

+// after a seal: its horizon is the seal's bound (a staged seal's for a

+// rollback's target, the journalled revert bound for a revert).

+func StagingScope(host, op, inc string, sealBound time.Time) hostclaim.ParkScope {

+ return laneScope(host, op, inc, sealBound)

+}

+

+// AbortScope is the park of an aborted move's staging lane. No seal exists

+// before an abort, so its horizon is the journalled abort bound (Plan 5

+// ruling K5): the later of the decision's time plus a signature's life and

+// the skew, and any seal's bound.

+func AbortScope(host, op, inc string, abortBound time.Time) hostclaim.ParkScope {

+ return laneScope(host, op, inc, abortBound)

+}

+

+// ReturnLaneScope is the park of a dirty rollback's return lane,

+// moves/<host>/<rb>: its horizon is the return seal's bound.

+func ReturnLaneScope(host, rbID, inc string, returnSealBound time.Time) hostclaim.ParkScope {

+ return laneScope(host, rbID, inc, returnSealBound)

+}

+

+func laneScope(host, lane, inc string, bound time.Time) hostclaim.ParkScope {

+ return hostclaim.ParkScope{Location: ScopeLocation(host, lane), Incarnation: inc, EvidenceAt: bound.UTC(), RequestLimit: ParkRequestLimit}

+}

diff --git a/internal/moves/parkscope_test.go b/internal/moves/parkscope_test.go

new file mode 100644

index 000000000..5c3ee56e1

--- /dev/null

+++ b/internal/moves/parkscope_test.go

@@ -0,0 +1,58 @@

+package moves

+

+import (

+ "strings"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/hostclaim"

+)

+

+// Every scope helper's EvidenceAt is the bound it was handed, never the

+// clock (claim rev 6.23 §2, INV-22): a bound years from now and one years

+// ago, in a zone that is not UTC, both come back as the same instant in UTC.

+// Each scope is a valid park for the host, names the incarnation it was

+// given, and carries the staging request limit.

+func TestScopeHelpersUseTheGenuineBound(t *testing.T) {

+ const host = "shop.bab.example.com"

+ inc := strings.Repeat("ab", 32)

+ zone := time.FixedZone("plus-one", 3600)

+ for _, bound := range []time.Time{

+ time.Date(2031, 3, 4, 5, 6, 7, 8, zone),

+ time.Date(2001, 9, 10, 11, 12, 13, 14, zone),

+ } {

+ for _, c := range []struct {

+ name string

+ got hostclaim.ParkScope

+ location string

+ }{

+ {"source location, holder with a slash", SourceLocationScope(hostclaim.Holder{Location: "instances/" + host + "/" + inc + "/", Incarnation: inc}, bound), "instances/" + host + "/" + inc},

+ {"source location, holder without one", SourceLocationScope(hostclaim.Holder{Location: "instances/" + host, Incarnation: inc}, bound), "instances/" + host},

+ {"target location, holder with a slash", TargetLocationScope(hostclaim.Holder{Location: "instances/" + host + "/" + inc + "/", Incarnation: inc}, bound), "instances/" + host + "/" + inc},

+ {"target location, holder without one", TargetLocationScope(hostclaim.Holder{Location: "instances/" + host, Incarnation: inc}, bound), "instances/" + host},

+ {"residue, holder with a slash", ResidueScope(hostclaim.Holder{Location: "instances/" + host + "/" + inc + "/", Incarnation: inc}, bound), "instances/" + host + "/" + inc},

+ {"residue, holder without one", ResidueScope(hostclaim.Holder{Location: "instances/" + host, Incarnation: inc}, bound), "instances/" + host},

+ {"staging", StagingScope(host, "move-1", inc, bound), "moves/" + host + "/move-1"},

+ {"abort", AbortScope(host, "move-1", inc, bound), "moves/" + host + "/move-1"},

+ {"return lane", ReturnLaneScope(host, "rb-1", inc, bound), "moves/" + host + "/rb-1"},

+ } {

+ t.Run(c.name+"/"+bound.Format("2006"), func(t *testing.T) {

+ if !c.got.EvidenceAt.Equal(bound) || c.got.EvidenceAt.Location() != time.UTC {

+ t.Fatalf("EvidenceAt = %v, want %v in UTC", c.got.EvidenceAt, bound)

+ }

+ if c.got.RequestLimit != 120*time.Second {

+ t.Fatalf("RequestLimit = %v, want 120s", c.got.RequestLimit)

+ }

+ if c.got.Location != c.location || c.got.Incarnation != inc {

+ t.Fatalf("scope = %q/%q, want %q/%q", c.got.Location, c.got.Incarnation, c.location, inc)

+ }

+ if err := c.got.Validate(host); err != nil {

+ t.Fatalf("Validate(%q): %v", host, err)

+ }

+ })

+ }

+ if zero := StagingScope(host, "move-1", inc, time.Time{}); zero.Validate(host) == nil {

+ t.Fatal("a zero bound made a valid scope; the helper must not fill one in")

+ }

+}

diff --git a/internal/park/local_linux_test.go b/internal/park/local_linux_test.go

index aae79d7f9..fcf56deb4 100644

--- a/internal/park/local_linux_test.go

+++ b/internal/park/local_linux_test.go

@@ -27,6 +27,13 @@ type localFixture struct {

}

func localFixtureFor(t *testing.T, paths map[string]uint64) localFixture {

+ t.Helper()

+ return localFixtureForOp(t, paths, hostclaim.Op{Kind: hostclaim.KindParking, ID: "delete", Exec: "box"})

+}

+

+// localFixtureForOp is localFixtureFor under the claim op given; the

+// park-local chore is that op's.

+func localFixtureForOp(t *testing.T, paths map[string]uint64, op hostclaim.Op) localFixture {

t.Helper()

host := "shop.bab.oncarlos.com"

data, locks := t.TempDir(), t.TempDir()

@@ -43,9 +50,9 @@ func localFixtureFor(t *testing.T, paths map[string]uint64) localFixture {

}

c := &hostclaim.Claim{V: 2, Host: host, Gen: 5,

Holder: &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", CreatedAt: at, Incarnation: inc, Location: "instances/" + host + "/" + inc, Stamp: 5},

- Op: hostclaim.Op{Kind: hostclaim.KindParking, ID: "delete", Exec: "box"},

+ Op: op,

Inventory: map[string]hostclaim.InventoryEntry{"box": {Layout: "flat", Paths: paths}},

- Obligations: []hostclaim.Obligation{{ID: "local", Op: "delete", Kind: "park-local", Box: "box", Owner: "box:box", Retirement: "retirement"}}}

+ Obligations: []hostclaim.Obligation{{ID: "local", Op: op.ID, Kind: "park-local", Box: "box", Owner: "box:box", Retirement: "retirement"}}}

if err := c.Validate(); err != nil {

t.Fatal(err)

}

@@ -671,3 +678,53 @@ func TestLocalUntouchedBoxAcknowledgement(t *testing.T) {

})

}

+

+// TestLocalRetirementUnderAMoveOpIsFindableByOp (moves spec §6.4, D6, K20):

+// a park-local whose op is a move's id, under each move kind that queues

+// one, lands under the engine's own layout,

+// .parked/<incarnation>/<host>/<retirement>/artifacts/, and its journal names

+// the move's op. That is park.Local's own contract; how F3 selects a

+// move's local retirements by op across both receipts (this journal and

+// the moves-owned rename's) is internal/host's

+// TestMoveLocalRetirementsAreFindableByOp.

+func TestLocalRetirementUnderAMoveOpIsFindableByOp(t *testing.T) {

+ const move = "mv_abcdefghijklmnopqrstuvwxyz"

+ for _, op := range []hostclaim.Op{

+ {Kind: hostclaim.KindLanding, ID: move, Phase: hostclaim.PhaseCommitted, Exec: "box"},

+ {Kind: hostclaim.KindRestoring, ID: move, Phase: hostclaim.PhaseClaimed, Exec: "box"},

+ } {

+ for _, mode := range []string{"artifacts", "installation"} {

+ t.Run(op.Kind+"/"+mode, func(t *testing.T) {

+ f := localFixtureForOp(t, map[string]uint64{"old.db": 3, "nested/state": 3}, op)

+ for _, p := range []string{"old.db", "nested/state"} {

+ f.write(t, p, p)

+ }

+ var err error

+ if mode == "artifacts" {

+ err = f.l.RetireArtifacts(t.Context(), f.host, f.captured, "local", move, 3)

+ } else {

+ err = f.l.RetireInstallation(t.Context(), f.host, f.captured, "local", move)

+ }

+ if err != nil {

+ t.Fatal(err)

+ }

+ for _, p := range []string{"old.db", "nested/state"} {

+ if b, err := os.ReadFile(f.parked(p)); err != nil || string(b) != p {

+ t.Fatalf("not under the engine's layout: %s: %q %v", p, b, err)

+ }

+ b, err := os.ReadFile(filepath.Join(f.l.DataDir, ".parked", f.captured.Incarnation, f.host, "retirement", ".journal.json"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ var j localJournal

+ if err := json.Unmarshal(b, &j); err != nil {

+ t.Fatal(err)

+ }

+ if j.Op != move || j.Host != f.host || j.Incarnation != f.captured.Incarnation || j.Retirement != "retirement" || j.Obligation != "local" || j.Mode != mode {

+ t.Fatalf("the journal does not name the move: %+v", j)

+ }

+ })

+ }

+}

diff --git a/internal/park/scope_restart_test.go b/internal/park/scope_restart_test.go

new file mode 100644

index 000000000..f8d39e654

--- /dev/null

+++ b/internal/park/scope_restart_test.go

@@ -0,0 +1,106 @@

+package park

+

+import (

+ "context"

+ "errors"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/hostclaim"

+ "github.com/carlosframework/platform/internal/moves"

+)

+

+// TestParkScopeSurvivesRestart is witness wParkScopeSurvivesRestart (claim

+// rev 6.23 §2, INV-22, Plan 5 ruling K8). A moves chore is queued with its

+// scope bound in the queuing CAS. After a restart the caller computes its

+// bound again, and the new one is later; neither a second queue nor a

+// park.Engine.Bind with it moves the scope: both are refused, the claim is

+// not written, and the first scope stands. A fresh engine then takes its

+// deadline from the persisted scope: pending before it, acknowledged after.

+func TestParkScopeSurvivesRestart(t *testing.T) {

+ f := newFixture(t)

+ ctx := t.Context()

+ obs, err := hostclaim.Read(ctx, f.e.Claims, f.w.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ inc := obs.Claim.Holder.Incarnation

+ sealBound := f.ob.Park.EvidenceAt

+ first := moves.StagingScope(f.w.Host, f.w.Op, inc, sealBound)

+ obs, err = hostclaim.Update(ctx, f.e.Claims, obs, func(c *hostclaim.Claim) error {

+ c.Obligations = nil

+ return hostclaim.QueueStagingPark(f.w.Op, "ie", first, f.e.NewID)(c)

+ })

+ if err != nil {

+ t.Fatalf("queue the lane park: %v", err)

+ }

+ if len(obs.Claim.Obligations) != 1 {

+ t.Fatalf("obligations = %+v, want the lane park alone", obs.Claim.Obligations)

+ }

+ lane := obs.Claim.Obligations[0]

+ w := Work{Host: f.w.Host, ID: lane.ID, Op: lane.Op}

+

+ // The restart: the bound is computed again, a minute later.

+ later := moves.StagingScope(f.w.Host, f.w.Op, inc, sealBound.Add(time.Minute))

+ stands := func(when string) {

+ t.Helper()

+ now, err := hostclaim.Read(ctx, f.e.Claims, f.w.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if now.ETag != obs.ETag {

+ t.Fatalf("%s: the claim was written", when)

+ }

+ if len(now.Claim.Obligations) != 1 || now.Claim.Obligations[0].Park == nil || !now.Claim.Obligations[0].Park.EvidenceAt.Equal(sealBound) {

+ t.Fatalf("%s: the first scope does not stand: %+v", when, now.Claim.Obligations)

+ }

+ if _, err := hostclaim.Update(ctx, f.e.Claims, obs, hostclaim.QueueStagingPark(f.w.Op, "ie", later, f.e.NewID)); !errors.Is(err, hostclaim.ErrWrongState) {

+ t.Fatalf("a second queue with a later bound: %v, want ErrWrongState", err)

+ }

+ stands("after the second queue")

+ if err := f.e.Bind(ctx, w, later); !errors.Is(err, ErrChanged) {

+ t.Fatalf("a bind with a later bound: %v, want ErrChanged", err)

+ }

+ stands("after the bind")

+ if err := f.e.Bind(ctx, w, first); err != nil {

+ t.Fatalf("a bind with the persisted scope: %v", err)

+ }

+ stands("after the same bind")

+

+ // A fresh engine (the agent restarted) reads its deadline from the claim.

+ restarted := f.e

+ restarted.View = func(_ context.Context, o hostclaim.Observed) (View, error) {

+ return View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: "unrelated", GenerationOf: func(Ref) (string, bool) { return "live", true }}, nil

+ }

+ f.b.Put("data", first.Location+"/replica/0000/data", []byte("staged"))

+ deadline := sealBound.Add(SignatureAge + first.RequestLimit)

+ f.now = deadline.Add(-time.Second)

+ restarted.After = func(step string) error {

+ if step == "verify" {

+ f.now = deadline.Add(time.Hour)

+ }

+ return nil

+ }

+ if err := restarted.Run(ctx, w); !errors.Is(err, ErrPending) {

+ t.Fatalf("a run whose first listing is before the persisted deadline: %v, want ErrPending", err)

+ }

+ after, err := hostclaim.Read(ctx, f.e.Claims, f.w.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if len(after.Claim.Obligations) != 1 || !after.Claim.Obligations[0].Park.EvidenceAt.Equal(sealBound) {

+ t.Fatalf("the run moved the scope: %+v", after.Claim.Obligations)

+ }

+ restarted.After = nil

+ if err := restarted.Run(ctx, w); err != nil {

+ t.Fatalf("a run after the persisted deadline: %v", err)

+ }

+ after, err = hostclaim.Read(ctx, f.e.Claims, f.w.Host)

+ if err != nil {

+ t.Fatal(err)

+ }

+ if len(after.Claim.Obligations) != 0 || !after.Claim.QuietAfter["ie"].Equal(deadline) {

+ t.Fatalf("after the deadline: obligations %+v, quiet_after %v, want none and %v", after.Claim.Obligations, after.Claim.QuietAfter["ie"], deadline)

+ }

+}

diff --git a/internal/provision/move_test.go b/internal/provision/move_test.go

index 4a692bd50..b28f67c22 100644

--- a/internal/provision/move_test.go

+++ b/internal/provision/move_test.go

@@ -1621,3 +1621,22 @@ func TestRevertInputsRecordedBeforeTheRender(t *testing.T) {

t.Fatalf("recovery_inputs missing: %s", raw)

}

+

+// The legacy abort completion is the legacy record's only (Plan 5 Task 1):

+// it refuses a per-op move, which completes through MoveGuardedAbortComplete

+// once the target region has parked its lane. A legacy aborting record still

+// completes, unchanged.

+func TestLegacyAbortCompleteRefusesAPerOpMove(t *testing.T) {

+ g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}

+ rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveAborting}}

+ if err := MoveAbortComplete()(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {

+ t.Fatalf("legacy completion of a per-op abort: %v %+v", err, rec.Move)

+ }

+ if err := MoveGuardedAbortComplete(g, time.Now())(rec); err != nil || rec.Move != nil {

+ t.Fatalf("guarded completion: %v %+v", err, rec.Move)

+ }

+ legacy := &Record{Move: &MoveState{To: "uk", State: MoveAborting}}

+ if err := MoveAbortComplete()(legacy); err != nil || legacy.Move != nil {

+ t.Fatalf("legacy completion of a legacy abort: %v %+v", err, legacy.Move)

+ }

+}

diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go

index a147485c9..26d0fe6c8 100644

--- a/internal/provision/plan9_b8_census_test.go

+++ b/internal/provision/plan9_b8_census_test.go

@@ -508,7 +508,7 @@ var plan9B8CensusInventory = []censusEntry{

// Records probe inputs through SetRecordMove; phase and stuck helpers use the same writer.

{"internal/host/movesrecover.go", "recoverRun.probing", "record-writer", "key:SetRecordMove", "7184b393fc356deb85181e25e4cafd61079598bf7ea4dafc28f90cbe26d50a31"},

// Marks recovery stuck through SetRecordMove; event and serial writes are not record objects.

- {"internal/host/movesrecover.go", "recoverRun.stuck", "record-writer", "key:SetRecordMove", "e2942947ddb1559e44b494e9900fa82ebbe3782a86333167797f4a94375a4a6c"},

+ {"internal/host/movesrecover.go", "recoverRun.stuck", "record-writer", "key:SetRecordMove", "4ea620140bbc75b127a893926eefcf58685b40c8362f86ba94f22931a05b449a"},

// Records revert inputs through SetRecordMove; stuck uses the reviewed record helper and journals are local.

{"internal/host/movesrevert.go", "revertRun.probing", "record-writer", "key:SetRecordMove", "67f15935f08fe9111b713641724093661e8c9dd2ee421287a1fc6cbe0509faf4"},

// Applies revert record mutations only through SetRecordMove and its sequenced writer.

@@ -520,7 +520,7 @@ var plan9B8CensusInventory = []censusEntry{

// Advances the target rollback phase through SetRecordMove and its sequenced writer.

{"internal/host/movesrollback.go", "rollbackRun.phase", "record-writer", "key:SetRecordMove", "76f1efb1076029e5da4c0cc0d750fbac5e1c94a6014c2b1e4af241797350827b"},

// Marks the return stuck through SetRecordMove; event and serial writes are not record objects.

- {"internal/host/movesrollback.go", "rollbackRun.stuck", "record-writer", "key:SetRecordMove", "8421f9feb384491bdb9d9cf3f722887b522bb70c41892526f7578e85654b0541"},

+ {"internal/host/movesrollback.go", "rollbackRun.stuck", "record-writer", "key:SetRecordMove", "84fcea399a55d77ceee17e35652ef269e5811453ffff62dff43208b68538b1ed"},

// Validates the pointer-digest marker key and Put/Delete method names without writing.

{"internal/hostops/mutation_classifier.go", "classifyPointerDigestMarker", "reference", "str:Put", "08111b5db5d98e788d1eef5861ad9c861cf926c5e1d5d54717fd650d2dcf9fa8"},

// Renders systemd paths under /data/instances and /run/carlos/instances; performs no store write.

@@ -546,7 +546,7 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/boxid/record.go", "Publish", "nonrecord", "sel:Put", "c60d1cf14e758de490c0824b635eb5d6ac54e24b5dd05c99a576a8a03d46ead7"},

{"internal/boxvolumes/volumes.go", "Write", "nonrecord", "sel:Put", "70ec487d1180d4abe18031d07f49ea98ff7d7fb222403c1ed9304c00429e7825"},

{"internal/console/account.go", "WriteAccountManifest", "nonrecord", "sel:Put", "f6f1ec0f2aa488ab51850ccf47753e37736d65b94708a1cf25ee0af1152ce251"},

- {"internal/console/apidbclaim.go", "Server.dbClaim", "key-site", "key:RecordKey", "9901c67966f4f932b0dd4acff8b243be5d91bcef55fff0c34ec03ba080b87983"},

+ {"internal/console/apidbclaim.go", "Server.dbClaim", "key-site", "key:RecordKey", "ba56b1742d07a47e9ccb17ff1f7a4def9722c2ef0ed51eecf6fd8550d292fa31"},

{"internal/console/apidomains.go", "Server.handleAPIDomainAttach", "key-site", "lit:instances", "f235eb06f14a7a2d74b69214076329b7d506eb5d49532ba74aa78b61e90fc5e2"},

{"internal/console/apidoorkeys.go", "Server.publishDoorKeysLocked", "foreign-recordkey", "key:RecordKey,sel:Put", "0fc498cd4af50570a0b7ed3ff71994213cab40040f6701a303bdf85d94f958da"},

{"internal/console/apiinstancemove.go", "Server.moveTransitionSaying", "record-writer", "key:SetRecordMove", "d1f7162eecdc234607c708a83e9633cdf77cc6931bac6754fb51042dd8409377"},

@@ -627,6 +627,7 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/host/fleetbind.go", "WriteFleetBinding", "nonrecord", "sel:Put", "44ab984c2c9b30abcc938132118f61a6922ccc0cd40ed2b6cb61c7654aa0a1c3"},

{"internal/host/logship.go", "logshipFlush", "nonrecord", "sel:Put", "a2fa594c1206028af170a2d0cbedef55ba00a2f1c698a6853e1214cc3b172dd9"},

{"internal/host/metrics_loop.go", "putAlertState", "nonrecord", "sel:Put", "d5b6e8b34dc5a731d6bbafeac0b7d0c5c20706d6f413dff38088b3321b35a26c"},

+ {"internal/host/movesabort.go", "movesWorker.abortSource", "record-writer", "key:SetRecordMove", "70f2cd7ec7709045043081c078f1abe63ef822325eef7016ab9f595a16cabad5"},

{"internal/host/movesarchive.go", "cutRun.publishCutArchive", "nonrecord", "sel:CopyVersioned,sel:PutIfAbsent", "6c7fcfd2d4b2a0319ac15ecbfe003ed0292fe160457fd5704be508c2ab302a1b"},

{"internal/host/movescut.go", "cutRun.cutTransition", "record-writer", "key:SetRecordMove", "d0d67399aa784036de43b3c143e88c3d85f8b3d9e6345f84908d3b58ff56191e"},

{"internal/host/movescut.go", "publishStoppedReport", "nonrecord", "sel:Put", "c710682f31d99127ad76524add246fda9f640c63934940e45c79ea25ba36eb3c"},

@@ -645,7 +646,6 @@ var plan9B8CensusInventory = []censusEntry{

{"internal/host/movesphase1.go", "movesWorker.refreshCrossRate", "nonrecord", "sel:CopyVersioned", "7e03e680bd61be26dddcf168cacd2427bb535571dfd326c4a3130ebe65ea1968"},

{"internal/host/movesphase1.go", "movesWorker.writeSourceReport", "nonrecord", "sel:PutReplace", "370407cca2c1b91f6e5e7f135238f04c0dce9d7986e6e1ee9bebbf88cad0b9e0"},

{"internal/host/movesphase1.go", "movesWorker.decideReady", "record-writer", "key:SetRecordMove", "c228e6f38eca6ed10501538031d3260f016a3f26ce501d8544f1581ae7668d8f"},

- {"internal/host/movesphase1.go", "movesWorker.interimAbort", "record-writer", "key:SetRecordMove", "9e875647dfdd2531c6857a4730245526c90c000986b2c9ea655bfcc4df43c02e"},

{"internal/host/movesreport.go", "movesWorker.targetReport", "nonrecord", "sel:PutReplace", "3f6aec125f1bd40271adf78833ac3268e536cbc5e8af3731b16f382669cd955b"},

{"internal/host/movesstore.go", "laneProbe", "nonrecord", "sel:PutIfAbsent", "f4a17a1143e7370b9039b2e66e736606015c46619bab92c553ce7b1ad6b7918b"},

{"internal/host/snapshot_loop.go", "registrySnapshotPass", "nonrecord", "sel:Put", "6b01b46b361c3a2d1d432659a0d234c01975849b9a8ac5571858fdd78d376849"},

diff --git a/internal/provision/provision.go b/internal/provision/provision.go

index cda7ff114..0154b3918 100644

--- a/internal/provision/provision.go

+++ b/internal/provision/provision.go

@@ -2416,13 +2416,19 @@ func MoveAbort() func(*Record) error {

}

-// MoveAbortComplete is the source box's half of an abort: staging is gone,

-// the move field clears.

+// MoveAbortComplete is the source box's half of a legacy abort: staging is

+// gone, the move field clears. It refuses a per-op move (Plan 5 Task 1): that

+// completes through MoveGuardedAbortComplete once the target region has

+// parked its lane and the claim is released, and its source never deletes

+// the target's staging.

func MoveAbortComplete() func(*Record) error {

return func(rec *Record) error {

if rec.Move == nil || rec.Move.State != MoveAborting {

return fmt.Errorf("%w: abort-complete is only legal from aborting", ErrMoveConflict)

}

+ if !rec.Move.Legacy() {

+ return fmt.Errorf("%w: a move with an id completes its abort through the guarded transition", ErrMoveConflict)

+ }

rec.Move = nil

return nil

}

carlos / platform Public

moves-phase1-5