amadan

carlos / platform Public

moves-phase1-5e

Squashed from moves-phase1-5e into main.
claude 5157c1e702da0d5f318f52c825de4e1de3f28288 parent e79d71b
38 files changed, +4048 −296
  • approved_moves_copy_test.go +5 −0
  • docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +13 −0
  • internal/appconfig/renderheld.go +20 −4
  • internal/appconfig/renderheld_before_test.go +62 −0
  • internal/appconfig/syncexec.go +16 −0
  • internal/host/agent.go +16 −2
  • internal/host/claimproduction.go +39 −14
  • internal/host/movesabort_test.go +25 −10
  • internal/host/movescarries_test.go +503 −0
  • internal/host/movesfileslive_test.go +610 −0
  • internal/host/movesfinal.go +38 −18
  • internal/host/movesfinal_test.go +11 −3
  • internal/host/movesinstall.go +236 −12
  • internal/host/movesinstall_test.go +30 −1
  • internal/host/movesland.go +32 −13
  • internal/host/movesland_test.go +65 −39
  • internal/host/moveslandcommit.go +2 −1
  • internal/host/moveslandpark.go +64 −64
  • internal/host/movesparkfakes_test.go +76 −0
  • internal/host/movesparks.go +2 −2
  • internal/host/movespass.go +48 −27
  • internal/host/movespass_test.go +2 −2
  • internal/host/movesphase1_test.go +1 −1
  • internal/host/movesrecover.go +10 −8
  • internal/host/movesrecover_test.go +5 −1
  • internal/host/movesrefusals.go +225 −0
  • internal/host/movesrefusals_test.go +226 −0
  • internal/host/movesrevert.go +21 −30
  • internal/host/movesrevert_test.go +1 −1
  • internal/host/movesrollback.go +163 −39
  • internal/host/movesrollback_finalfix_test.go +2 −2
  • internal/host/movessources.go +648 −0
  • internal/host/movessources_test.go +712 −0
  • internal/park/live_generations_test.go +39 −0
  • internal/park/sweep.go +14 −0
  • internal/provision/move_test.go +44 −0
  • internal/provision/plan9_b8_census_test.go +8 −2
  • internal/provision/provision.go +14 −0

CI passed — run details

CI log 
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.011s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	301.314s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	2.974s
ok  	github.com/carlosframework/platform/internal/activator	142.400s
ok  	github.com/carlosframework/platform/internal/agentsock	0.020s
ok  	github.com/carlosframework/platform/internal/alerting	0.004s
ok  	github.com/carlosframework/platform/internal/analytics	0.025s
ok  	github.com/carlosframework/platform/internal/appconfig	2.128s
ok  	github.com/carlosframework/platform/internal/apperr	0.092s
ok  	github.com/carlosframework/platform/internal/applog	0.147s
ok  	github.com/carlosframework/platform/internal/awsinv	0.007s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.009s
ok  	github.com/carlosframework/platform/internal/boxid	0.652s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.009s
ok  	github.com/carlosframework/platform/internal/caps	0.013s
ok  	github.com/carlosframework/platform/internal/certwarm	0.003s
ok  	github.com/carlosframework/platform/internal/cgroup	0.189s
ok  	github.com/carlosframework/platform/internal/changelog	0.249s
ok  	github.com/carlosframework/platform/internal/clamd	0.058s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	317.131s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	6.474s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.010s
ok  	github.com/carlosframework/platform/internal/dbverify	0.006s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.006s
ok  	github.com/carlosframework/platform/internal/docsite	0.006s
ok  	github.com/carlosframework/platform/internal/domains	1.929s
ok  	github.com/carlosframework/platform/internal/doorkey	0.006s
ok  	github.com/carlosframework/platform/internal/economics	0.006s
ok  	github.com/carlosframework/platform/internal/edge	25.685s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.137s
ok  	github.com/carlosframework/platform/internal/gate	1.654s
ok  	github.com/carlosframework/platform/internal/genselect	0.018s
ok  	github.com/carlosframework/platform/internal/geo	0.018s
ok  	github.com/carlosframework/platform/internal/geoip	0.034s
ok  	github.com/carlosframework/platform/internal/host	45.078s
ok  	github.com/carlosframework/platform/internal/hostactors	0.452s
ok  	github.com/carlosframework/platform/internal/hostbox	1.177s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.221s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.006s
ok  	github.com/carlosframework/platform/internal/hostexec	0.310s
ok  	github.com/carlosframework/platform/internal/hostfence	0.042s
ok  	github.com/carlosframework/platform/internal/hostfiles	4.086s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.016s
ok  	github.com/carlosframework/platform/internal/hostguard	0.133s
ok  	github.com/carlosframework/platform/internal/hostops	1.227s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.004s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.134s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.002s
ok  	github.com/carlosframework/platform/internal/infraview	0.002s
ok  	github.com/carlosframework/platform/internal/instancectl	1.702s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.685s
ok  	github.com/carlosframework/platform/internal/instancefs	0.004s
ok  	github.com/carlosframework/platform/internal/layout	1.276s
ok  	github.com/carlosframework/platform/internal/lease	9.447s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.010s
ok  	github.com/carlosframework/platform/internal/ledger	0.020s
ok  	github.com/carlosframework/platform/internal/mail	0.155s
ok  	github.com/carlosframework/platform/internal/maintenance	0.082s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.527s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.073s
ok  	github.com/carlosframework/platform/internal/metrics	0.004s
ok  	github.com/carlosframework/platform/internal/moves	20.945s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.726s
ok  	github.com/carlosframework/platform/internal/objstore	0.179s
ok  	github.com/carlosframework/platform/internal/ops	0.153s
ok  	github.com/carlosframework/platform/internal/park	0.171s
ok  	github.com/carlosframework/platform/internal/privatefile	0.004s
ok  	github.com/carlosframework/platform/internal/provision	2.963s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.009s
ok  	github.com/carlosframework/platform/internal/registry	3.542s
ok  	github.com/carlosframework/platform/internal/relay	7.223s
ok  	github.com/carlosframework/platform/internal/release	18.426s
ok  	github.com/carlosframework/platform/internal/rollout	1.455s
ok  	github.com/carlosframework/platform/internal/rootaws	0.006s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.003s
ok  	github.com/carlosframework/platform/internal/schedule	2.174s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	2.324s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	4.105s
ok  	github.com/carlosframework/platform/internal/status	0.036s
ok  	github.com/carlosframework/platform/internal/steering	0.005s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.128s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.004s
ok  	github.com/carlosframework/platform/internal/termfence	0.003s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.005s
ok  	github.com/carlosframework/platform/internal/unixdial	0.003s
ok  	github.com/carlosframework/platform/internal/updatewords	0.002s
ok  	github.com/carlosframework/platform/internal/usage	0.004s
ok  	github.com/carlosframework/platform/internal/version	0.001s
ok  	github.com/carlosframework/platform/internal/vet	1.549s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.022s
=== RUN   TestT4StorageProbeMinio
2026/10/11 09:38:43 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18DD70235CD6DC13, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.014s
minio contract: PASS
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go
index f0e8f1820..746e2bd93 100644
--- a/approved_moves_copy_test.go
+++ b/approved_moves_copy_test.go
@@ -76,6 +76,7 @@ var movesCopyIDs = []string{
"MV-N313", "MV-N314",
"MV-N315", "MV-N316", "MV-N317", "MV-N318",
"MV-N325", "MV-N326", "MV-N327", "MV-N328", "MV-N329", "MV-N330", "MV-N331", "MV-N332", "MV-N333", "MV-N334",
+ "MV-N335", "MV-N336", "MV-N337",
}
// copyRow is one row of the record. The File cell is the file the sentence is
@@ -544,6 +545,8 @@ func TestMovesTextAppliedPlaceholders(t *testing.T) {
// copy" task appends the files it touched; TestMovesScanFilesCoverTheDiff
// fails when the list falls behind.
var movesScanFiles = []string{
+ "internal/host/movessources.go",
+ "internal/host/movesrefusals.go",
"internal/console/movesrecovery.go",
"internal/console/claim_runtime.go",
"internal/alerting/alerting.go",
@@ -750,6 +753,8 @@ var movesReturnedSentences = map[string]map[string]bool{
"internal/host/movesreport.go": {"t1Below": true, "t3Differs": true, "t3Shadowed": true, "t4Failed": true,
"t6NoRoom": true, "t8Count": true, "t12NotQuiet": true, "replicaReason": true},
"internal/host/movesrollback.go": {"stuckPage": true},
+ // The moves evidence's anomaly page (Plan 5 Task 9a, MV-N336 and MV-N337).
+ "internal/host/movesrefusals.go": {"movesAnomalyPage": true},
// The console's move pass pages (Plan 5 Task 7, MV-N325 to MV-N334).
"internal/console/movesrecovery.go": {"moveWaitingPage": true, "movePreparingPageText": true, "moveOrphanPage": true, "moveCloseOverduePage": true, "moveAbortOverduePage": true},
}
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
index 4651f5fb7..d70407767 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
@@ -286,6 +286,19 @@ Task 7's pages, the console's move pass (`internal/console/movesrecovery.go`, §
| MV-N333 | internal/console/movesrecovery.go | carlos move abort not closed: <host> | pending |
| MV-N334 | internal/console/movesrecovery.go | The abort of the move <id> of <host> has nothing left to park in <to>, and <box> has not closed it. <box> closes it once it has removed the move's snapshots from its own bucket. Check <box>. | pending |
+Task 9a's rows (the production park wiring). MV-N335 is the stuck reason a rollback records when the target's files cannot be archived at its release; it reaches `carlos move status` through Move.StuckReason and the stuck page (MV-N314).
+
+| id | file | text | status |
+|---|---|---|---|
+| MV-N335 | internal/provision/provision.go | the target's files cannot be archived now that it has stopped | pending |
+
+The anomaly page (Task 9a's quiet refusals, `internal/host/movesrefusals.go`): a box pages once when the moves evidence for a host's at-rest cleanup contradicts itself. A warning; the reason stays in the agent's log.
+
+| id | file | text | status |
+|---|---|---|---|
+| MV-N336 | internal/host/movesrefusals.go | carlos move cleanup refused: <host> | pending |
+| MV-N337 | internal/host/movesrefusals.go | The cleanup of <host> on <box> stopped: its move history does not add up, or something wrote where no move could. Nothing of <host> is cleaned up in this region until an operator checks the agent's log on <box>. | pending |
+
## Floor gate fix (#267 review)
(root's floor gate for the system units, `carlos system floor-allows-prev`; its stderr reaches the unit's journal)
diff --git a/internal/appconfig/renderheld.go b/internal/appconfig/renderheld.go
index 1258805b0..2babb92ca 100644
--- a/internal/appconfig/renderheld.go
+++ b/internal/appconfig/renderheld.go
@@ -52,7 +52,23 @@ func RenderHeld(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, dat
if hostwrite.Enabled(os.Getenv) {
return HeldRender{}, ErrHeldClaimWired
}
- return renderHeld(ctx, s, priv, dataDir, rt, op, box, func(HeldRender) (hostwrite.Work, error) { return legacyHostWork{}, nil })
+ return renderHeld(ctx, s, priv, dataDir, rt, op, box, func(HeldRender) (hostwrite.Work, error) { return legacyHostWork{}, nil }, nil)
+}
+
+// RenderHeldBefore is RenderHeld with before called once the rendered bytes
+// are known and before anything is written: before receives the hex sha256
+// of exactly the bytes the render then places (or finds already in place),
+// so a caller can make its ownership of the file durable first (moves Plan 5
+// Task 9a, carry (c)). An error from before ends the render: nothing is
+// written and no ack is sent. The write itself is RenderHeld's, unchanged.
+func RenderHeldBefore(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, dataDir string, rt registry.Route, op string, box string, before func(sha256hex string) error) (HeldRender, error) {
+ if before == nil {
+ return HeldRender{}, hostwrite.ErrUnavailable
+ }
+ if hostwrite.Enabled(os.Getenv) {
+ return HeldRender{}, ErrHeldClaimWired
+ }
+ return renderHeld(ctx, s, priv, dataDir, rt, op, box, func(HeldRender) (hostwrite.Work, error) { return legacyHostWork{}, nil }, before)
}
// RenderHeldUnder is RenderHeld for a held row under the host claim (moves
@@ -68,10 +84,10 @@ func RenderHeldUnder(ctx context.Context, s release.Store, priv *ecdh.PrivateKey
if work == nil {
return HeldRender{}, hostwrite.ErrUnavailable
}
- return renderHeld(ctx, s, priv, dataDir, rt, op, box, work)
+ return renderHeld(ctx, s, priv, dataDir, rt, op, box, work, nil)
}
-func renderHeld(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, dataDir string, rt registry.Route, op string, box string, work func(HeldRender) (hostwrite.Work, error)) (HeldRender, error) {
+func renderHeld(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, dataDir string, rt registry.Route, op string, box string, work func(HeldRender) (hostwrite.Work, error), before func(string) error) (HeldRender, error) {
if err := heldRenderable(rt, op, box); err != nil {
return HeldRender{}, err
}
@@ -87,7 +103,7 @@ func renderHeld(ctx context.Context, s release.Store, priv *ecdh.PrivateKey, dat
if w == nil {
return HeldRender{}, hostwrite.ErrUnavailable
}
- r, err := renderExecConfig(ctx, w, rt, m, priv, dataDir, "", v.Overlay)
+ r, err := renderExecConfigBefore(ctx, w, rt, m, priv, dataDir, "", v.Overlay, before)
if err != nil {
return HeldRender{}, err
}
diff --git a/internal/appconfig/renderheld_before_test.go b/internal/appconfig/renderheld_before_test.go
new file mode 100644
index 000000000..fd4008713
--- /dev/null
+++ b/internal/appconfig/renderheld_before_test.go
@@ -0,0 +1,62 @@
+package appconfig
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "os"
+ "testing"
+)
+
+// TestRenderHeldBeforeAnswersTheHashBeforeTheWrite (moves Plan 5 Task 9a,
+// fix round 1, I1): the caller learns the sha256 of the exact bytes the
+// render is about to place, before anything is written, so it can journal
+// its ownership first; the write that follows places those bytes; a refusal
+// from the callback writes nothing and sends no ack.
+func TestRenderHeldBeforeAnswersTheHashBeforeTheWrite(t *testing.T) {
+ ctx, s, reg, dd, privB64, _ := execKit(t)
+ priv, err := ParsePrivateKey(privB64, "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ heldFixture(t, ctx, s)
+ rt := landingRow()
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ path := ConfigEnvPath(dd, rt)
+
+ refused := errors.New("the journal refused")
+ if _, err := RenderHeldBefore(ctx, s, priv, dd, rt, heldOp, heldBox, func(string) error { return refused }); !errors.Is(err, refused) {
+ t.Fatalf("a refusing callback: %v", err)
+ }
+ if _, err := os.Stat(path); !os.IsNotExist(err) {
+ t.Fatalf("a refused render wrote the file: %v", err)
+ }
+ if _, _, err := s.Get(ctx, ConfigAckKey(heldBox, heldHost)); err == nil {
+ t.Fatal("a refused render wrote the ack")
+ }
+
+ var told string
+ got, err := RenderHeldBefore(ctx, s, priv, dd, rt, heldOp, heldBox, func(sha string) error {
+ if _, err := os.Stat(path); !os.IsNotExist(err) {
+ t.Errorf("the file was there before the callback: %v", err)
+ }
+ told = sha
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(body)
+ if want := hex.EncodeToString(sum[:]); told != want || got.SHA256 != want {
+ t.Fatalf("callback %q, render %q, file %q", told, got.SHA256, want)
+ }
+ if a := readAck(t, ctx, s); a.FileSHA256 != told {
+ t.Fatalf("ack %q, callback %q", a.FileSHA256, told)
+ }
+}
diff --git a/internal/appconfig/syncexec.go b/internal/appconfig/syncexec.go
index cca52b4ea..cc3d95287 100644
--- a/internal/appconfig/syncexec.go
+++ b/internal/appconfig/syncexec.go
@@ -37,6 +37,8 @@ import (
"bytes"
"context"
"crypto/ecdh"
+ "crypto/sha256"
+ "encoding/hex"
"errors"
"fmt"
"os"
@@ -1325,6 +1327,14 @@ type execRender struct {
// them. The bare-hostname check stays with each caller, before any path is
// derived.
func renderExecConfig(ctx context.Context, work hostwrite.Work, rt registry.Route, m *Manifest, priv *ecdh.PrivateKey, dataDir, header, instETag string) (execRender, error) {
+ return renderExecConfigBefore(ctx, work, rt, m, priv, dataDir, header, instETag, nil)
+}
+
+// renderExecConfigBefore is renderExecConfig with before, when not nil,
+// called with the hex sha256 of the rendered bytes once they are known and
+// before the file is written or found up to date (RenderHeldBefore). An
+// error from it ends the render with nothing written.
+func renderExecConfigBefore(ctx context.Context, work hostwrite.Work, rt registry.Route, m *Manifest, priv *ecdh.PrivateKey, dataDir, header, instETag string, before func(string) error) (execRender, error) {
path := ConfigEnvPath(dataDir, rt)
// A nested row's current file lives on tenant-owned ground, so its read
// goes through instancefs the same way its write below does: a plain
@@ -1378,6 +1388,12 @@ func renderExecConfig(ctx context.Context, work hostwrite.Work, rt registry.Rout
rendered = append([]byte(header), rendered...)
}
out := execRender{path: path, rendered: rendered}
+ if before != nil {
+ sum := sha256.Sum256(rendered)
+ if err := before(hex.EncodeToString(sum[:])); err != nil {
+ return execRender{}, err
+ }
+ }
if existed && bytes.Equal(cur, rendered) {
return out, nil // up to date — stay quiet on the periodic tick
}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 7457ed444..e71dd6fb0 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -187,6 +187,7 @@ import (
"github.com/carlosframework/platform/internal/instancefs"
"github.com/carlosframework/platform/internal/layout"
"github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
@@ -276,6 +277,15 @@ func runAgent(args []string, buildClaims ClaimFactory) {
log.Fatalf("agent: %v", err)
}
claimCtx = withClaimRuntimeResources(claimCtx, ClaimRuntimeResources{Mode: "agent", Registry: reg, Manager: cgm, DataDir: dd, OptDir: release.OptDir(), UnitsDir: "/etc/systemd/system"})
+ // The moves park providers (Plan 5 Task 9a) register before the claim
+ // assembly reads them; only with both switches on, and behind the claim
+ // Plan 12 guard, so the parks still refuse by name until it lands.
+ // Task 5's expected refusals are quiet in the regional worker's log and
+ // its anomalies page once (movesrefusals.go), with both switches on.
+ movesRefusalLog := newMovesRefusals(metricsBoxLabel(os.Getenv, readMachineID()), PageOperator, log.Printf, time.Now)
+ if err := registerProductionMovesParks(claimCtx, claimEntryEnv(claimCtx), claimStore(claimCtx), *cfg, productionRosterRegion(claimStore(claimCtx), os.Getenv), movesRefusalLog); err != nil {
+ log.Printf("agent: moves: the park providers are not registered: %v", err)
+ }
claims, err := newClaimWiring(claimCtx, claimEntryEnv(claimCtx), buildClaims)
if err != nil {
log.Fatalf("agent: host claims: %v", err)
@@ -447,7 +457,7 @@ func runAgent(args []string, buildClaims ClaimFactory) {
} else {
eb.Mirror = m
}
- armMovesPass(*cfg, dd, reg, svc, eb, claims.Writes)
+ armMovesPass(*cfg, dd, reg, svc, eb, claims.Writes, claims.Providers)
// The startup sweep (design §6.7): a leftover instance unit from a
// previous agent process, one this box's registry no longer holds
// a live wake for, is stopped before this process starts spawning
@@ -578,7 +588,11 @@ func runAgent(args []string, buildClaims ClaimFactory) {
sd.runner.Start(context.Background()) // runs until the process ends, like every loop below
}
serialClaims := make(chan struct{}, 1)
- if err := claims.startRegional(agentCtx, serialClaims, func(err error) { log.Printf("agent: regional claims: %v", err) }); err != nil {
+ regionalReport := func(err error) { log.Printf("agent: regional claims: %v", err) }
+ if hostwrite.Enabled(claimEntryEnv(claimCtx)) && moves.Enabled(os.Getenv) {
+ regionalReport = movesRefusalLog.report
+ }
+ if err := claims.startRegional(agentCtx, serialClaims, regionalReport); err != nil {
log.Fatalf("agent: regional claims: %v", err)
}
defer func() {
diff --git a/internal/host/claimproduction.go b/internal/host/claimproduction.go
index 82cdfa57c..9f903b913 100644
--- a/internal/host/claimproduction.go
+++ b/internal/host/claimproduction.go
@@ -10,6 +10,7 @@ import (
"time"
"crypto/ecdh"
+ "crypto/ed25519"
"github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/appconfig"
"github.com/carlosframework/platform/internal/boxid"
@@ -59,32 +60,56 @@ func RegisterProductionClaimMoveSources(s ClaimProductionMoveSources) error {
return nil
}
-// ProductionClaimOwnerProvider constructs claim-owned authority directly. Only
-// C8 and the moves regional snapshot/lease inventory come from the moves owner.
-type ProductionClaimOwnerProvider struct{}
+// productionRosterRow is this box's verified roster row and what read it:
+// the release key, the box's host-actor enrollment, the signed directory,
+// and the directory's row for the box, whose enrollment and cloud instance
+// are the enrollment's.
+type productionRosterRow struct {
+ pub ed25519.PublicKey
+ enrollment hostfloor.Enrollment
+ directory hostactors.Directory
+ row hostactors.Box
+}
-func (ProductionClaimOwnerProvider) OpenClaimOwners(ctx context.Context, store release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) {
- var out ClaimOwnerBindings
- if store == nil || r.Registry == nil || (r.Mode != "agent" && r.Mode != "adopt" && r.Mode != "instance-control") {
- return out, hostwrite.ErrUnavailable
- }
- pub, err := release.ParsePublicKey(os.Getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+// readProductionRosterRow is OpenClaimOwners' roster read, shared with the
+// moves park view's region (productionRosterRegion, Plan 5 Task 9a fix
+// round 1) so the two cannot disagree.
+func readProductionRosterRow(ctx context.Context, store release.Store, getenv func(string) string) (productionRosterRow, error) {
+ var rr productionRosterRow
+ pub, err := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
if err != nil {
- return out, err
+ return rr, err
}
enrollment, err := hostfloor.ReadEnrollment("/etc/carlos/host-actor.json")
if err != nil {
- return out, err
+ return rr, err
}
- directory := hostactors.Directory{Store: store, PublicKey: pub, Bucket: os.Getenv("CARLOS_DEPLOYMENT_BUCKET")}
+ directory := hostactors.Directory{Store: store, PublicKey: pub, Bucket: getenv("CARLOS_DEPLOYMENT_BUCKET")}
roster, err := directory.Read(ctx)
if err != nil {
- return out, err
+ return rr, err
}
row, ok := roster.Roster.Boxes[enrollment.Box]
if !ok || row.Enrollment != enrollment.Enrollment || row.Cloud.InstanceID != enrollment.InstanceID {
- return out, hostbox.ErrChanged
+ return rr, hostbox.ErrChanged
+ }
+ return productionRosterRow{pub: pub, enrollment: enrollment, directory: directory, row: row}, nil
+}
+
+// ProductionClaimOwnerProvider constructs claim-owned authority directly. Only
+// C8 and the moves regional snapshot/lease inventory come from the moves owner.
+type ProductionClaimOwnerProvider struct{}
+
+func (ProductionClaimOwnerProvider) OpenClaimOwners(ctx context.Context, store release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) {
+ var out ClaimOwnerBindings
+ if store == nil || r.Registry == nil || (r.Mode != "agent" && r.Mode != "adopt" && r.Mode != "instance-control") {
+ return out, hostwrite.ErrUnavailable
+ }
+ rr, err := readProductionRosterRow(ctx, store, os.Getenv)
+ if err != nil {
+ return out, err
}
+ pub, enrollment, directory, row := rr.pub, rr.enrollment, rr.directory, rr.row
certs, err := boxid.EmbeddedCerts()
if err != nil {
return out, err
diff --git a/internal/host/movesabort_test.go b/internal/host/movesabort_test.go
index 44b728d79..871feb728 100644
--- a/internal/host/movesabort_test.go
+++ b/internal/host/movesabort_test.go
@@ -67,7 +67,7 @@ func (h *abortHarness) abort(t *testing.T) {
func (h *abortHarness) laneCfg() moveCfg {
c := h.targetCfg()
c.DataDir = h.tdir
- c.ParkLeases = h.tleases
+ armParkFakes(&c, h.s, h.tleases, c.Region, roleTarget)
return c
}
@@ -357,15 +357,30 @@ func TestAbortWaitsForAnUnreachableTargetRegion(t *testing.T) {
if err := hostclaim.StartGuard(c, []string{"ie"}, []string{"ie-box"}); err == nil {
t.Fatal("a new operation could start over the waiting abort")
}
- // An unarmed target box (production until Task 9) parks nothing either.
- unarmed := h.laneCfg()
- unarmed.ParkLeases = nil
- w := &movesWorker{cfg: unarmed, mirrors: map[string]lease.MirrorStore{}}
- if err := movesPass(context.Background(), h.s, w); err != nil {
- t.Fatal(err)
- }
- if h.laneChore(t) == nil || len(h.laneKeys()) == 0 {
- t.Fatal("an unarmed target box parked the lane")
+ // An unarmed target box parks nothing either: the abort's lane park goes
+ // through laneParkTarget (movesfinal.go's dispatch for an aborting
+ // target), whose arming is the shared lane predicate, revertLaneArmed,
+ // which requires all three of the claim's park hooks (Plan 5 Task 9a,
+ // M17). Each one missing leaves the arm unarmed, with its standing
+ // notice, and the lane untouched.
+ for name, unarm := range map[string]func(*moveCfg){
+ "no park leases": func(c *moveCfg) { c.ParkLeases = nil },
+ "no park view": func(c *moveCfg) { c.ParkView = nil },
+ "no park authorizer": func(c *moveCfg) { c.ParkAuthorize = nil },
+ } {
+ unarmed := h.laneCfg()
+ unarm(&unarmed)
+ logs := captureLog(t)
+ w := &movesWorker{cfg: unarmed, mirrors: map[string]lease.MirrorStore{}}
+ if err := movesPass(context.Background(), h.s, w); err != nil {
+ t.Fatal(err)
+ }
+ if h.laneChore(t) == nil || len(h.laneKeys()) == 0 {
+ t.Fatalf("%s: an unarmed target box parked the lane", name)
+ }
+ if !strings.Contains(logs(), "this box's park engine is not armed") {
+ t.Fatalf("%s: the abort's lane park was tried by an unarmed box:\n%s", name, logs())
+ }
}
// Once the region returns, the abort completes.
h.tPass(t)
diff --git a/internal/host/movescarries_test.go b/internal/host/movescarries_test.go
new file mode 100644
index 000000000..4825ed410
--- /dev/null
+++ b/internal/host/movescarries_test.go
@@ -0,0 +1,503 @@
+package host
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// The Task 9 carries (Plan 5 Task 9a, M16): (b) a foreign database parked by
+// F's retirement, (c) a rendered config left behind when its ack fails.
+
+// parkedLocally lists the base names under the target's .parked/.
+func parkedLocally(t *testing.T, dir string) map[string][]byte {
+ t.Helper()
+ out := map[string][]byte{}
+ filepath.WalkDir(filepath.Join(dir, ".parked"), func(p string, d os.DirEntry, err error) error {
+ if err == nil && !d.IsDir() && d.Type().IsRegular() {
+ b, _ := os.ReadFile(p)
+ out[filepath.Base(p)] = b
+ }
+ return nil
+ })
+ return out
+}
+
+// cleanAfterRollback rolls the landing back, recovers the source and runs
+// the target's cleaning to cleaned.
+func (h *rollbackHarness) cleanAfterRollback(t *testing.T) {
+ t.Helper()
+ h.rollBack(t)
+ h.rollbackPass(t)
+ h.recovered(t)
+ for i := 0; i < 4 && h.rb(t).TargetPhase != provision.RbTargetCleaned; i++ {
+ h.rollbackPass(t)
+ }
+ if rb := h.rb(t); rb.TargetPhase != provision.RbTargetCleaned {
+ t.Fatalf("the target did not clean: %+v", rb)
+ }
+}
+
+// TestForeignDatabaseSurvivesRollbackRetirement (carry (b)): the landing
+// registers only what it placed at local, the files and the env; the
+// database and its marker are registered once the install proves them the
+// build's. A database that is not the build, standing at DBPath when a
+// landing stopped at local rolls back, is neither retired with F nor taken
+// by the moves-owned rename.
+func TestForeignDatabaseSurvivesRollbackRetirement(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ h.landAt = "after-inventory"
+ h.crashLand(t)
+ if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandLocal {
+ t.Fatalf("fixture: the landing is at %s", m.LandingPhase)
+ }
+ paths := h.claim(t).Inventory["uk-box"].Paths
+ if _, ok := paths[p1Host+".db"]; ok || len(paths) != 2 {
+ t.Fatalf("local registered %v, want the files and the env only", paths)
+ }
+ db := provision.DefaultDBPath(h.tdir, p1Host)
+ foreign := []byte("a database this landing never installed")
+ if err := os.WriteFile(db, foreign, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(db+".gen", []byte("someone-else\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ h.cleanAfterRollback(t)
+ if got, err := os.ReadFile(db); err != nil || !bytes.Equal(got, foreign) {
+ t.Fatalf("the foreign database: %q %v", got, err)
+ }
+ if _, err := os.Stat(db + ".gen"); err != nil {
+ t.Fatalf("the foreign marker: %v", err)
+ }
+ for name, body := range parkedLocally(t, h.tdir) {
+ if bytes.Equal(body, foreign) {
+ t.Fatalf("the foreign database was parked as %s", name)
+ }
+ }
+ if _, err := os.Lstat(filepath.Join(h.tdir, p1Host)); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the landing's files directory is still live: %v", err)
+ }
+}
+
+// TestInstalledDatabaseIsRegisteredAfterTheInstall (carry (b)): once
+// configured installs the build, the database and its marker are registered
+// under F, and a crash between that install and the record's stamp still
+// leaves them the op's by content: the cleaning parks the build.
+func TestInstalledDatabaseIsRegisteredAfterTheInstall(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ h.landAt = "after-db-rename"
+ h.crashLand(t)
+ if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured || (m.Landing != nil && m.Landing.InstalledDBSHA256 != "") {
+ t.Fatalf("fixture: %+v", m)
+ }
+ if _, ok := h.claim(t).Inventory["uk-box"].Paths[p1Host+".db"]; ok {
+ t.Fatal("the database was registered before the install finished")
+ }
+ db := provision.DefaultDBPath(h.tdir, p1Host)
+ built, err := os.ReadFile(db)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h.cleanAfterRollback(t)
+ if _, err := os.Lstat(db); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the build is still at DBPath: %v", err)
+ }
+ if got, ok := parkedLocally(t, h.tdir)[p1Host+".db"]; !ok || !bytes.Equal(got, built) {
+ t.Fatal("the build was not parked")
+ }
+
+ h2 := newLandingHarness(t)
+ h2.flip(t)
+ h2.land(t)
+ h2.installed(t)
+ paths := h2.claim(t).Inventory["uk-box"].Paths
+ for _, p := range []string{p1Host + ".db", p1Host + ".db.gen", p1Host, p1Host + ".env"} {
+ if _, ok := paths[p]; !ok {
+ t.Fatalf("after configured the inventory is %v, missing %s", paths, p)
+ }
+ }
+}
+
+// TestFailedRenderAckIsParkedByCleaning (carry (c)): the held render wrote
+// <host>.config.env and its config ack failed. The journal recorded the
+// file's state before the render and the rendered sha256, so the target's
+// cleaning parks the file; a file whose content is not the render's is
+// left.
+func TestFailedRenderAckIsParkedByCleaning(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ replace []byte // written over the rendered file before the cleaning
+ parked bool
+ }{
+ {"the rendered file", nil, true},
+ {"another file at the path", []byte("SOMEONE=else\n"), false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ path := filepath.Join(h.tdir, p1Host+".config.env")
+ rendered := []byte("SECRET=landing\n")
+ sum := sha256.Sum256(rendered)
+ ackFailed := errors.New("held config ack: the store refused the write")
+ h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
+ if err := before(hex.EncodeToString(sum[:])); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ if err := os.WriteFile(path, rendered, 0o600); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ in := h.read(t, p1Host).Move.Inputs
+ return appconfig.HeldRender{Path: path, SHA256: hex.EncodeToString(sum[:]), BundleETags: in.BundleETags, OverlayETag: in.OverlayETag}, ackFailed
+ }
+ h.land(t)
+ if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandConfigured {
+ t.Fatalf("fixture: the landing is at %s", m.LandingPhase)
+ }
+ j := h.landJournal(t)
+ want := "sha256:" + hex.EncodeToString(sum[:])
+ if pre := journalledPreRender(t, j); pre.Before != "absent" || len(pre.Intended) != 1 || pre.Intended[0] != want || j.Steps[landStepRendered] != want {
+ t.Fatalf("the render's journal: pre %q rendered %q", j.Steps[landStepPreRender], j.Steps[landStepRendered])
+ }
+ if tc.replace != nil {
+ if err := os.WriteFile(path, tc.replace, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ h.cleanAfterRollback(t)
+ _, err := os.Lstat(path)
+ if tc.parked {
+ if !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the rendered config is still live: %v", err)
+ }
+ if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, rendered) {
+ t.Fatal("the rendered config was not parked")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("a file that is not the render's was taken: %v", err)
+ }
+ })
+ }
+}
+
+// TestReleaseTimeArchiveRefusalGoesStuck (M16): a target that served stops,
+// and its files directory holds something the archive can never carry. The
+// release records the stuck reason at releasing (not a retry forever), the
+// operator is paged once through moveCfg.Page, and once the cause is
+// removed a retry runs the release again.
+func TestReleaseTimeArchiveRefusalGoesStuck(t *testing.T) {
+ h := newRollbackHarness(t)
+ h.rollBack(t)
+ link := filepath.Join(h.tdir, p1Host, "link")
+ if err := os.Symlink("/etc/passwd", link); err != nil {
+ t.Fatal(err)
+ }
+ h.rollbackPass(t)
+ m := h.read(t, p1Host).Move
+ if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReleaseTargetFiles || m.Rollback.TargetPhase != provision.RbTargetReleasing ||
+ m.Rollback.StuckAt != provision.RbSourceWaiting {
+ t.Fatalf("not stuck at releasing: %+v %+v", m, m.Rollback)
+ }
+ pagedOnce(t, h.pages, "uk-box", "target", provision.StuckReleaseTargetFiles)
+ // A stuck record is not the target's to work on, and pages no more.
+ h.rollbackPass(t)
+ if got := h.read(t, p1Host).Move; got.State != provision.MoveStuck || len(h.pages) != 1 {
+ t.Fatalf("a pass over the stuck record: %+v, %d pages", got, len(h.pages))
+ }
+ if err := os.Remove(link); err != nil {
+ t.Fatal(err)
+ }
+ h.retry(t)
+ h.rollbackPass(t)
+ if rb := h.rb(t); h.read(t, p1Host).Move.State != provision.MoveRollingBack || rb.TargetPhase == provision.RbTargetReleasing {
+ t.Fatalf("after the retry: %+v", rb)
+ }
+}
+
+// journalledPreRender decodes the landing journal's pre_render step.
+func journalledPreRender(t *testing.T, j *moves.Journal) preRender {
+ t.Helper()
+ var pre preRender
+ if err := json.Unmarshal([]byte(j.Steps[landStepPreRender]), &pre); err != nil {
+ t.Fatalf("pre_render %q: %v", j.Steps[landStepPreRender], err)
+ }
+ return pre
+}
+
+// TestRenderCrashBeforeItsRecordIsParkedByCleaning (fix round 1, I1): the
+// render journals the hash of the bytes it is about to place before it
+// writes them, so a crash after the file is written and before either the
+// ack or the rendered record lands still leaves the file the op's. The
+// cleaning parks it by that intended hash; other content at the path is
+// left.
+func TestRenderCrashBeforeItsRecordIsParkedByCleaning(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ replace []byte
+ parked bool
+ }{
+ {"the file the render was placing", nil, true},
+ {"other content at the path", []byte("SOMEONE=else\n"), false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ path := filepath.Join(h.tdir, p1Host+".config.env")
+ rendered := []byte("SECRET=landing\n")
+ sum := sha256.Sum256(rendered)
+ crashed := errors.New("the agent died after the write")
+ h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
+ if err := before(hex.EncodeToString(sum[:])); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ if err := os.WriteFile(path, rendered, 0o600); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ return appconfig.HeldRender{}, crashed // no ack, no answer
+ }
+ h.land(t)
+ j := h.landJournal(t)
+ if pre := journalledPreRender(t, j); len(pre.Intended) != 1 || pre.Intended[0] != "sha256:"+hex.EncodeToString(sum[:]) || pre.Before != "absent" {
+ t.Fatalf("pre_render %+v", pre)
+ }
+ if _, ok := j.Steps[landStepRendered]; ok {
+ t.Fatal("fixture: the rendered record landed")
+ }
+ if tc.replace != nil {
+ if err := os.WriteFile(path, tc.replace, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ h.cleanAfterRollback(t)
+ _, err := os.Lstat(path)
+ if tc.parked {
+ if !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the rendered config is still live: %v", err)
+ }
+ if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, rendered) {
+ t.Fatal("the rendered config was not parked")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("a file that is not the render's was taken: %v", err)
+ }
+ })
+ }
+}
+
+// TestRenderReplayKeepsEveryIntendedHash (fix round 2): render A is written
+// and the agent dies before the ack or the rendered record; the retry's
+// inputs have changed, so it renders B, journals B and fails to write it. A
+// is still the file on disk. The intended hashes are an append-only set
+// across the op's attempts, so A's stays recorded and the target's cleaning
+// still parks the file.
+func TestRenderReplayKeepsEveryIntendedHash(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ path := filepath.Join(h.tdir, p1Host+".config.env")
+ a, b := []byte("SECRET=a\n"), []byte("SECRET=b\n")
+ sumA, sumB := sha256.Sum256(a), sha256.Sum256(b)
+ attempt := 0
+ h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
+ attempt++
+ if attempt == 1 {
+ if err := before(hex.EncodeToString(sumA[:])); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ if err := os.WriteFile(path, a, 0o600); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ return appconfig.HeldRender{}, errors.New("the agent died after writing A")
+ }
+ if err := before(hex.EncodeToString(sumB[:])); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ return appconfig.HeldRender{}, errors.New("writing B failed")
+ }
+ h.land(t)
+ h.land(t)
+ if attempt < 2 {
+ t.Fatalf("fixture: %d render attempts", attempt)
+ }
+ pre := journalledPreRender(t, h.landJournal(t))
+ wantA, wantB := "sha256:"+hex.EncodeToString(sumA[:]), "sha256:"+hex.EncodeToString(sumB[:])
+ if len(pre.Intended) != 2 || pre.Intended[0] != wantA || pre.Intended[1] != wantB {
+ t.Fatalf("intended %v; want [A B]", pre.Intended)
+ }
+ if got, err := os.ReadFile(path); err != nil || !bytes.Equal(got, a) {
+ t.Fatalf("fixture: the file is %q %v, want A", got, err)
+ }
+ h.cleanAfterRollback(t)
+ if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("A is still live: %v", err)
+ }
+ if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, a) {
+ t.Fatal("A was not parked")
+ }
+}
+
+// renderAttempt is one fake held render: it journals sum through the
+// installer's before-write hook, writes body to path when write is true,
+// and then dies (no ack, no answer) unless done is true.
+func renderAttempt(path string, body []byte, write, done bool, held appconfig.HeldRender) func(func(string) error) (appconfig.HeldRender, error) {
+ return func(before func(string) error) (appconfig.HeldRender, error) {
+ sum := sha256.Sum256(body)
+ if err := before(hex.EncodeToString(sum[:])); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ if !write {
+ return appconfig.HeldRender{}, errors.New("the write failed")
+ }
+ if err := os.WriteFile(path, body, 0o600); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ if !done {
+ return appconfig.HeldRender{}, errors.New("the agent died after the write")
+ }
+ held.Path, held.SHA256 = path, hex.EncodeToString(sum[:])
+ return held, nil
+ }
+}
+
+// TestRenderCrashAfterThePrunedJournal (fix round 4): render A is written,
+// then B is written; render C prunes the set to what is on disk, B, plus C,
+// journals it, and dies before writing. The file on disk is B, which the set
+// still names, so the cleaning parks it.
+func TestRenderCrashAfterThePrunedJournal(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ path := filepath.Join(h.tdir, p1Host+".config.env")
+ a, b, c := []byte("SECRET=a\n"), []byte("SECRET=b\n"), []byte("SECRET=c\n")
+ for _, step := range []func(func(string) error) (appconfig.HeldRender, error){
+ renderAttempt(path, a, true, false, appconfig.HeldRender{}),
+ renderAttempt(path, b, true, false, appconfig.HeldRender{}),
+ renderAttempt(path, c, false, false, appconfig.HeldRender{}),
+ } {
+ h.renderHook = step
+ h.land(t)
+ }
+ sumB, sumC := sha256.Sum256(b), sha256.Sum256(c)
+ pre := journalledPreRender(t, h.landJournal(t))
+ if want := []string{"sha256:" + hex.EncodeToString(sumB[:]), "sha256:" + hex.EncodeToString(sumC[:])}; len(pre.Intended) != 2 || pre.Intended[0] != want[0] || pre.Intended[1] != want[1] {
+ t.Fatalf("intended %v; want [B C]", pre.Intended)
+ }
+ if pre.Before != "absent" {
+ t.Fatalf("the earlier-state record changed: %q", pre.Before)
+ }
+ h.cleanAfterRollback(t)
+ if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("B is still live: %v", err)
+ }
+ if got, ok := parkedLocally(t, h.tdir)[p1Host+".config.env"]; !ok || !bytes.Equal(got, b) {
+ t.Fatal("B was not parked")
+ }
+}
+
+// TestReturnRenderRetriesStayBounded (fix round 4): a dirty return has no
+// deadline, so its held render may be retried any number of times. Forty
+// failed or changed attempts, half written and half not, keep the intended
+// set at most two hashes, always naming what is on disk; the next render
+// succeeds and the return completes with that content kept as the source's
+// config, still named.
+func TestReturnRenderRetriesStayBounded(t *testing.T) {
+ h := newReturnHarness(t, false)
+ path := filepath.Join(h.sdir, p1Host+".config.env")
+ for i := 0; i < 40; i++ {
+ h.renderHook = renderAttempt(path, []byte(fmt.Sprintf("SECRET=%d\n", i)), i%2 == 0, false, appconfig.HeldRender{})
+ if _, err := h.install(t); err == nil {
+ t.Fatalf("attempt %d: the render answered", i)
+ }
+ pre := journalledPreRender(t, h.returnJournal(t))
+ if len(pre.Intended) > 2 {
+ t.Fatalf("attempt %d: %d intended hashes", i, len(pre.Intended))
+ }
+ if got, err := fileSHA256(path); err == nil && !containsString(pre.Intended, got) {
+ t.Fatalf("attempt %d: the file on disk (%s) is not named by %v", i, got, pre.Intended)
+ }
+ }
+ final := []byte("SECRET=final\n")
+ h.renderHook = renderAttempt(path, final, true, true, h.render)
+ if _, err := h.install(t); err != nil {
+ t.Fatalf("the successful render: %v", err)
+ }
+ if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase != provision.RbSourceRestored {
+ t.Fatalf("the return did not complete: %+v", rb)
+ }
+ got, err := os.ReadFile(path)
+ if err != nil || !bytes.Equal(got, final) {
+ t.Fatalf("the source's config is %q %v, want the final render", got, err)
+ }
+ sum := sha256.Sum256(final)
+ pre := journalledPreRender(t, h.returnJournal(t))
+ if len(pre.Intended) > 2 || !containsString(pre.Intended, "sha256:"+hex.EncodeToString(sum[:])) {
+ t.Fatalf("intended %v after the final render", pre.Intended)
+ }
+}
+
+// TestRenderTempIsRemovedByCleaning (fix round 5): the renderer writes the
+// rendered bytes to <host>.config.env.tmp and renames them over the path
+// (appconfig syncexec.go). A crash between the two leaves the temp holding
+// the rendered secrets, an unfinished write of this op's. The target's
+// cleaning removes it (never parks it: it is not live config), and leaves
+// <host>.config.env as it was before the render: absent, or an earlier file
+// that is not the render's, kept.
+func TestRenderTempIsRemovedByCleaning(t *testing.T) {
+ for _, earlier := range [][]byte{nil, []byte("EARLIER=1\n")} {
+ t.Run(fmt.Sprintf("earlier %v", earlier != nil), func(t *testing.T) {
+ h := &rollbackHarness{landingHarness: newLandingHarness(t)}
+ h.flip(t)
+ path := filepath.Join(h.tdir, p1Host+".config.env")
+ if earlier != nil {
+ if err := os.WriteFile(path, earlier, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ rendered := []byte("SECRET=landing\n")
+ sum := sha256.Sum256(rendered)
+ h.renderHook = func(before func(string) error) (appconfig.HeldRender, error) {
+ if err := before(hex.EncodeToString(sum[:])); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ if err := os.WriteFile(path+".tmp", rendered, 0o600); err != nil {
+ return appconfig.HeldRender{}, err
+ }
+ return appconfig.HeldRender{}, errors.New("the agent died before the rename")
+ }
+ h.land(t)
+ if _, err := os.Stat(path + ".tmp"); err != nil {
+ t.Fatalf("fixture: no temp: %v", err)
+ }
+ h.cleanAfterRollback(t)
+ if _, err := os.Lstat(path + ".tmp"); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the renderer's temp is still there: %v", err)
+ }
+ got, err := os.ReadFile(path)
+ switch {
+ case earlier == nil && !errors.Is(err, os.ErrNotExist):
+ t.Fatalf("a config appeared: %q %v", got, err)
+ case earlier != nil && (err != nil || !bytes.Equal(got, earlier)):
+ t.Fatalf("the earlier config was not kept: %q %v", got, err)
+ }
+ for name, body := range parkedLocally(t, h.tdir) {
+ if bytes.Equal(body, rendered) {
+ t.Fatalf("the rendered secrets were parked as %s", name)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/host/movesfileslive_test.go b/internal/host/movesfileslive_test.go
new file mode 100644
index 000000000..9885ddd8b
--- /dev/null
+++ b/internal/host/movesfileslive_test.go
@@ -0,0 +1,610 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfiles"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// The files live set (J2/F4, Paul's #236 ruling; M10): every moves park
+// excludes the objects a live generation's files manifest names, in its own
+// region, and the source's are protected on the source until the record
+// commits. Plan 5 Task 9a.
+
+// filesManifest writes a hostfiles manifest at <location>/files/g-<gen>/
+// naming objects, each "path=key".
+func filesManifest(t *testing.T, put func(key string, body []byte), location, gen string, objects ...string) {
+ t.Helper()
+ m := hostfiles.Manifest{V: hostfiles.ManifestVersion, Seq: int64(len(objects)), Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}}
+ for i, o := range objects {
+ path, key, _ := strings.Cut(o, "=")
+ m.Entries[path] = hostfiles.Entry{Object: key, SHA256: fmt.Sprintf("%064x", i+1), Size: 4, Mode: 0o644, Seq: int64(i + 1)}
+ }
+ body, err := json.Marshal(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ put(hostfiles.ManifestKey(location+"/files/g-"+gen+"/"), body)
+}
+
+// TestParkKeepsTheFilesLiveSet (M10): in the authoritative region the view
+// pins the exact versions a live generation's files manifest names, also
+// under an older generation's files prefix, so the sweep never makes that
+// generation a candidate and its objects stay. Without a manifest nothing
+// more is pinned and the older generation is parked. Seen from another
+// region nothing is read or pinned.
+func TestParkKeepsTheFilesLiveSet(t *testing.T) {
+ setup := func(t *testing.T, manifest bool) (*evFixture, park.Engine, []string) {
+ f := newEvFixture(t)
+ body, err := json.Marshal(map[string]any{"op": evMoveID, "incarnation": f.inc, "host": evHost, "box": "box", "installed_at": evCreated, "manifest_sha256": "x", "snapshot_txid": 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.bucket.Put("data", f.location+"/generation.json", body)
+ old := []string{f.location + "/files/g-old/objects/0000000000000001-aa", f.genKey("old")}
+ for _, k := range old {
+ f.bucket.Put("data", k, []byte("old"))
+ }
+ f.bucket.Put("data", f.location+"/files/g-"+evMoveID+"/objects/0000000000000002-bb", []byte("cur"))
+ if manifest {
+ filesManifest(t, func(k string, b []byte) { f.bucket.Put("data", k, b) }, f.location, evMoveID,
+ "kept.txt="+old[0], "new.txt="+f.location+"/files/g-"+evMoveID+"/objects/0000000000000002-bb")
+ }
+ seq := 0
+ e := park.Engine{Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, Now: func() time.Time { return evNow },
+ NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) },
+ View: movesParkView(f.provider().Records, f.bucket, "data", "ie"),
+ Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }}
+ return f, e, old
+ }
+ t.Run("a live manifest names an older generation's object", func(t *testing.T) {
+ f, e, old := setup(t, true)
+ v, err := e.View(t.Context(), f.observe(t))
+ if err != nil {
+ t.Fatal(err)
+ }
+ ver := f.bucket.LatestID("data", old[0])
+ if !v.Pinned[park.Ref{Key: old[0], VersionID: ver}] {
+ t.Fatalf("pins %v; want %s@%s", v.Pinned, old[0], ver)
+ }
+ d, err := e.Sweep(t.Context(), evHost, f.inc, evAt, hostbox.RequestLimit)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range d.Candidates {
+ if c.Generation == "old" {
+ t.Fatalf("the generation a live manifest names is a candidate: %+v", d.Candidates)
+ }
+ }
+ for _, k := range old {
+ if _, ok := f.bucket.Current("data", k); !ok {
+ t.Fatalf("%s was parked", k)
+ }
+ }
+ // Seen from a region that is not authoritative, nothing is read.
+ other, err := movesParkView(f.provider().Records, f.bucket, "data", "uk")(t.Context(), f.observe(t))
+ if err != nil || len(other.Pinned) != 0 || other.MarkerKnown {
+ t.Fatalf("the uk view %+v, %v", other, err)
+ }
+ })
+ t.Run("no manifest", func(t *testing.T) {
+ f, e, old := setup(t, false)
+ if _, err := e.Sweep(t.Context(), evHost, f.inc, evAt, hostbox.RequestLimit); err != nil && !errors.Is(err, park.ErrPending) {
+ t.Fatal(err)
+ }
+ for _, k := range old {
+ if _, ok := f.bucket.Current("data", k); ok {
+ t.Fatalf("%s was not parked", k)
+ }
+ }
+ })
+ t.Run("a manifest naming what the location does not hold", func(t *testing.T) {
+ for name, key := range map[string]string{"outside the location": "instances/" + evHost + "/elsewhere/x", "missing": "LOC/files/g-old/objects/0000000000000009-zz"} {
+ f, e, _ := setup(t, false)
+ filesManifest(t, func(k string, b []byte) { f.bucket.Put("data", k, b) }, f.location, evMoveID, "x="+strings.Replace(key, "LOC", f.location, 1))
+ if _, err := e.View(t.Context(), f.observe(t)); !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("%s: %v", name, err)
+ }
+ }
+ })
+}
+
+// TestFilesLiveSetIsProtectedOnTheSourceUntilTheCommit (J2/F4): once the
+// target's probe is admitted the target's region is authoritative, but seen
+// from the source's region the source stays authoritative until the record
+// commits, with its own live manifests pinned; after the commit the source
+// pins nothing (P6-D3: the target's manifests name no source object), and
+// its location park may run.
+func TestFilesLiveSetIsProtectedOnTheSourceUntilTheCommit(t *testing.T) {
+ f := newVwFixture(t)
+ f.marker(t, "cur", f.inc)
+ src := f.location + "/files/g-cur/objects/0000000000000001-aa"
+ ver := f.bucket.Put("data", src, []byte("source file"))
+ filesManifest(t, func(k string, b []byte) { f.bucket.Put("data", k, b) }, f.location, "cur", "a.txt="+src)
+ landing := func(phase string) hostclaim.Observed {
+ return vwObs(&hostclaim.Claim{Holder: f.holder(vwMove, "cur@ie", vwMove+"@uk"), Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: vwMove, Phase: phase, Exec: "uk-box"}})
+ }
+ f.move(provision.MoveLanding)
+ v, err := f.view("ie")(t.Context(), landing(provision.LandProbed))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v.AuthoritativeRegion != "ie" || !v.Pinned[park.Ref{Key: src, VersionID: ver}] {
+ t.Fatalf("before the commit the source's view %+v", v)
+ }
+ if v, err := f.view("uk")(t.Context(), landing(provision.LandProbed)); err != nil || v.AuthoritativeRegion != "uk" {
+ t.Fatalf("before the commit the target's view %+v, %v", v, err)
+ }
+ f.move(provision.MoveCommitted)
+ v, err = f.view("ie")(t.Context(), landing(hostclaim.PhaseCommitted))
+ if err != nil || v.AuthoritativeRegion != "uk" || len(v.Pinned) != 0 {
+ t.Fatalf("after the commit the source's view %+v, %v", v, err)
+ }
+}
+
+// TestLandingResidueParkKeepsTheFilesLiveSet (M10, the landing install's
+// park): after the target's probe makes its generation live, a residue
+// chore's acknowledgement re-lists the location under the real view. An
+// object of the residue generation that the live generation's manifest names
+// refuses the acknowledgement and stays; without that reference the same
+// object is parked.
+func TestLandingResidueParkKeepsTheFilesLiveSet(t *testing.T) {
+ for _, named := range []bool{true, false} {
+ t.Run(fmt.Sprintf("named %v", named), func(t *testing.T) {
+ ctx := context.Background()
+ h := newLandingHarness(t)
+ h.flip(t)
+ h.residue(t)
+ h.land(t)
+ h.installed(t)
+ // The probe's admission: g-<move id> live in uk, its marker
+ // published, a files manifest that names an object under the
+ // earlier stay's generation (written back after the parking).
+ late := cutRoot + "/files/g-old/objects/0000000000000003-cc"
+ if _, err := h.vs.PutIfAbsent(ctx, "data-uk", late, []byte("kept")); err != nil {
+ t.Fatal(err)
+ }
+ if named {
+ filesManifest(t, func(k string, b []byte) {
+ if _, err := h.vs.PutIfAbsent(ctx, "data-uk", k, b); err != nil {
+ t.Fatal(err)
+ }
+ }, cutRoot, p1ID, "kept.txt="+late)
+ }
+ marker := cutRoot + "/generation.json"
+ markerVer := h.f.nextVersion()
+ body, _ := json.Marshal(map[string]any{"op": p1ID, "incarnation": h.inc, "host": p1Host, "box": "uk-box", "installed_at": h.clock.Now(), "manifest_sha256": "x", "snapshot_txid": 1})
+ h.f.store("data-uk", marker, fakeMoveObject{lease.MirrorObject{ETag: "etag-" + markerVer, Size: int64(len(body))}, string(body)})
+ h.f.setVersion("data-uk", marker, markerVer)
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ c.Holder.Generation = p1ID
+ c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "uk", PredecessorRegion: "ie", Session: c.Holder.Session + 1, Box: "uk-box", Generation: p1ID, Predecessor: "cur", Mode: "cold"})
+ })
+ releaseOwnLease(h.lw, p1Host)
+ h.clock.Advance(park.SignatureAge + landingRequestLimit + time.Minute)
+ l := &landingRun{w: h.lw, s: h.s, rec: h.read(t, p1Host), g: provision.MoveGuard{ID: p1ID, Incarnation: h.inc}}
+ eng := park.Engine{Store: h.vs, Claims: h.s, Bucket: "data-uk", Region: "uk", Leases: h.leases, Now: h.clock.Now, NewID: obligationID,
+ View: movesParkView(MovesRecordsFrom(h.s), h.vs, "data-uk", "uk"), Authorize: movesArmAuthorize(allowParkAuthorize, l.authorizeObligation)}
+ var runErr error
+ for _, o := range h.claim(t).Obligations {
+ if o.Kind == "park-generation" && o.Generation == "old" {
+ for i := 0; i < 3; i++ {
+ runErr = eng.Run(ctx, park.Work{Host: p1Host, ID: o.ID, Op: p1ID})
+ }
+ }
+ }
+ _, kept := h.f.buckets["data-uk"][late]
+ if named {
+ if !errors.Is(runErr, park.ErrUnsafe) || !kept {
+ t.Fatalf("a named object: run %v, kept %v", runErr, kept)
+ }
+ return
+ }
+ if kept {
+ t.Fatalf("an unnamed residue object stayed (run %v)", runErr)
+ }
+ })
+ }
+}
+
+// TestReturnResidueParkKeepsTheFilesLiveSet (M10, the rollback return's
+// park): once the source's probe admits the return generation the source is
+// authoritative again, and its live manifest's objects are pinned, also
+// under a pre-move generation; before that admission the target is
+// authoritative and the source pins nothing (the dirty return parks the
+// pre-move generations, D6).
+func TestReturnResidueParkKeepsTheFilesLiveSet(t *testing.T) {
+ f := newVwFixture(t)
+ f.rec.Move = &provision.MoveState{ID: vwMove, Incarnation: f.inc, From: "ie", To: "uk", State: provision.MoveRollingBack,
+ Rollback: &provision.MoveRollback{ID: vwRb, Dirty: true}}
+ body, _ := json.Marshal(map[string]any{"op": vwRb, "incarnation": f.inc, "host": vwHost, "box": "ie-box", "installed_at": vwCreated, "manifest_sha256": "x", "snapshot_txid": 1})
+ f.bucket.Put("data", f.location+"/generation.json", body)
+ pre := f.location + "/files/g-cur/objects/0000000000000001-aa"
+ ver := f.bucket.Put("data", pre, []byte("pre-move"))
+ filesManifest(t, func(k string, b []byte) { f.bucket.Put("data", k, b) }, f.location, vwRb, "a.txt="+pre)
+ restoring := func(gen string, admissions ...string) hostclaim.Observed {
+ return vwObs(&hostclaim.Claim{Holder: f.holder(gen, admissions...), Op: hostclaim.Op{Kind: hostclaim.KindRestoring, ID: vwMove, Phase: hostclaim.PhaseProbing, Exec: "ie-box"}})
+ }
+ v, err := f.view("ie")(t.Context(), restoring(vwRb, "cur@ie", vwMove+"@uk", vwRb+"@ie"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v.AuthoritativeRegion != "ie" || !v.Pinned[park.Ref{Key: pre, VersionID: ver}] {
+ t.Fatalf("after the source's admission %+v", v)
+ }
+ v, err = f.view("ie")(t.Context(), restoring(vwMove, "cur@ie", vwMove+"@uk"))
+ if err != nil || v.AuthoritativeRegion != "uk" || len(v.Pinned) != 0 {
+ t.Fatalf("before the source's admission %+v, %v", v, err)
+ }
+}
+
+// TestFilesLiveSetFollowsTheLiveChain (fix round 1, I2): with the marker
+// lagging, park.protected treats the whole unfinished chain g3 -> g2 -> g1
+// as live, and the view reads every one of those generations' manifests. An
+// object only g2's manifest names, under the older generation g0, is pinned
+// and survives the sweep's park.
+func TestFilesLiveSetFollowsTheLiveChain(t *testing.T) {
+ f := newEvFixture(t)
+ f.setClaim(t, func(c *hostclaim.Claim) {
+ c.Holder.Generation = "g3"
+ c.Holder.Admissions = []hostclaim.AdmissionRecord{
+ {Session: 1, Box: "box", Generation: "g1", Mode: "cold", Region: "ie"},
+ {Session: 2, Box: "box", Generation: "g2", Predecessor: "g1", Mode: "cold", Region: "ie", PredecessorRegion: "ie"},
+ {Session: 3, Box: "box", Generation: "g3", Predecessor: "g2", Mode: "cold", Region: "ie", PredecessorRegion: "ie"},
+ }
+ c.Holder.Stamp, c.Holder.Session = 3, 3
+ })
+ body, err := json.Marshal(map[string]any{"op": "g1", "incarnation": f.inc, "host": evHost, "box": "box", "installed_at": evCreated, "manifest_sha256": "x", "snapshot_txid": 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.bucket.Put("data", f.location+"/generation.json", body)
+ only := f.location + "/files/g-g0/objects/0000000000000001-aa"
+ ver := f.bucket.Put("data", only, []byte("named by g2 alone"))
+ other := f.location + "/files/g-g0/objects/0000000000000002-bb"
+ f.bucket.Put("data", other, []byte("named by nobody"))
+ filesManifest(t, func(k string, b []byte) { f.bucket.Put("data", k, b) }, f.location, "g2", "a.txt="+only)
+ seq := 0
+ e := park.Engine{Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, Now: func() time.Time { return evNow },
+ NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) },
+ View: movesParkView(f.provider().Records, f.bucket, "data", "ie"),
+ Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }}
+ v, err := e.View(t.Context(), f.observe(t))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !v.Pinned[park.Ref{Key: only, VersionID: ver}] {
+ t.Fatalf("pins %v; want g2's object %s", v.Pinned, only)
+ }
+ if _, err := e.Sweep(t.Context(), evHost, f.inc, evAt, hostbox.RequestLimit); err != nil && !errors.Is(err, park.ErrPending) {
+ t.Fatal(err)
+ }
+ if got, ok := f.bucket.Current("data", only); !ok || string(got) != "named by g2 alone" {
+ t.Fatalf("g2's object did not survive the park: %q %v", got, ok)
+ }
+}
+
+// putFilesGeneration writes, in bucket, a files object of gen at location
+// and gen's manifest naming it, returning the object's key.
+func putFilesGeneration(t *testing.T, vs *fakeVersioned, bucket, location, gen string) string {
+ t.Helper()
+ ctx := context.Background()
+ obj := location + "/files/g-" + gen + "/objects/0000000000000001-aa"
+ if _, err := vs.PutIfAbsent(ctx, bucket, obj, []byte("a live file")); err != nil {
+ t.Fatal(err)
+ }
+ filesManifest(t, func(k string, b []byte) {
+ if _, err := vs.PutIfAbsent(ctx, bucket, k, b); err != nil {
+ t.Fatal(err)
+ }
+ }, location, gen, "a.txt="+obj)
+ return obj
+}
+
+// archivedUnder reports whether bucket holds a parked copy of key, a key
+// under location (park's destination: parked/<inc>/<host>/<retirement>/
+// instances/<key below the location>@<version>).
+func archivedUnder(f *fakeMoveMirror, bucket, inc, location, key string) bool {
+ tail := strings.TrimPrefix(key, location+"/")
+ for k := range f.buckets[bucket] {
+ if strings.HasPrefix(k, "parked/"+inc+"/") && strings.Contains(k, "/instances/"+tail+"@") {
+ return true
+ }
+ }
+ return false
+}
+
+// admitTarget is the target's probe admission on the claim (as Plan 3's
+// probe makes it): holder.generation is the move's, admitted in uk.
+func admitTarget(t *testing.T, h *landingHarness) {
+ t.Helper()
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "uk", PredecessorRegion: "ie", Session: c.Holder.Session + 1, Box: "uk-box", Generation: p1ID, Predecessor: c.Holder.Generation, Mode: "cold"})
+ c.Holder.Generation = p1ID
+ })
+}
+
+// TestSourceCleanupKeepsTheFilesLiveSetUntilTheCommit (fix round 1, I3; the
+// source cleanup, J2/F4): the source's location park runs through the real
+// view and engine. Before the record commits, with the target's probe
+// admitted, a park of the source's location in its own region is refused
+// and the source's files stay byte for byte. After the commit the source's
+// finalization runs that park and the files are retired, archived under
+// parked/.
+func TestSourceCleanupKeepsTheFilesLiveSetUntilTheCommit(t *testing.T) {
+ ctx := context.Background()
+ h := newFinalHarness(t)
+ h.advanceToProbed(t)
+ admitTarget(t, h.landingHarness)
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/")
+ obj := putFilesGeneration(t, h.vs, "data-ie", location, "cur")
+ view := movesParkView(MovesRecordsFrom(h.s), h.vs, "data-ie", "ie")
+
+ // Before the commit: the same scope the commit queues, run by the engine.
+ rec := h.read(t, p1Host)
+ scope, err := sourceLocationParkScope(rec, *h.claim(t).Holder)
+ if err != nil {
+ t.Fatal(err)
+ }
+ early := hostclaim.Obligation{ID: obligationID(), Op: p1ID, Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: obligationID(), Park: &scope}
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Obligations = append(c.Obligations, early) })
+ e := park.Engine{Store: h.vs, Claims: h.s, Bucket: "data-ie", Region: "ie", Leases: h.leases, Now: h.clock.Now, NewID: obligationID,
+ View: view, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }}
+ if err := e.Run(ctx, park.Work{Host: p1Host, ID: early.ID, Op: p1ID}); !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("a park of the source before the commit: %v", err)
+ }
+ if body, ok := h.f.buckets["data-ie"][obj]; !ok || body.body != "a live file" {
+ t.Fatalf("the source's file before the commit: %+v %v", body, ok)
+ }
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) {
+ kept := c.Obligations[:0]
+ for _, o := range c.Obligations {
+ if o.ID != early.ID {
+ kept = append(kept, o)
+ }
+ }
+ c.Obligations = kept
+ })
+
+ // The commit, then the source's finalization through the real view.
+ h.commitRecord(t)
+ h.commitClaim(t)
+ h.cfg.ParkView = view
+ h.sourcePass(t)
+ if m := h.read(t, p1Host).Move; m.SourceParkedAt == nil {
+ t.Fatalf("the source did not finalize: %+v", m)
+ }
+ if _, ok := h.f.buckets["data-ie"][obj]; ok {
+ t.Fatal("the source's file was not retired after the commit")
+ }
+ if !archivedUnder(h.f, "data-ie", h.inc, location, obj) {
+ t.Fatal("the source's file was not archived")
+ }
+}
+
+// TestRollbackReturnCleanupKeepsTheFilesLiveSet (fix round 1, I3; the
+// rollback return's cleanup): the target's region location park, queued
+// when the target released, runs through the real view and engine. While
+// the target's admitted generation is still the holder's (before the return
+// names the source's again) the target region is authoritative and the park
+// is refused, the target's files kept; once the source has taken the host
+// back, the target's cleaning runs it and the files are retired.
+func TestRollbackReturnCleanupKeepsTheFilesLiveSet(t *testing.T) {
+ ctx := context.Background()
+ h := newRollbackHarness(t)
+ admitTarget(t, h.landingHarness)
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/")
+ body, err := json.Marshal(map[string]any{"op": p1ID, "incarnation": h.inc, "host": p1Host, "box": "uk-box", "installed_at": h.clock.Now(), "manifest_sha256": "x", "snapshot_txid": 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := h.vs.PutIfAbsent(ctx, "data-uk", location+"/generation.json", body); err != nil {
+ t.Fatal(err)
+ }
+ obj := putFilesGeneration(t, h.vs, "data-uk", location, p1ID)
+ view := movesParkView(MovesRecordsFrom(h.s), h.vs, "data-uk", "uk")
+ h.parkView = view
+ h.rollBack(t)
+ h.rollbackPass(t)
+ var chore *hostclaim.Obligation
+ for _, o := range h.claim(t).Obligations {
+ if o.Op == p1ID && o.Kind == "park-replica" && o.Region == "uk" && o.Park != nil && o.Park.Location == location {
+ chore = &o
+ }
+ }
+ if chore == nil {
+ t.Fatalf("the release queued no target location park: %+v", h.claim(t).Obligations)
+ }
+ e := park.Engine{Store: h.vs, Claims: h.s, Bucket: "data-uk", Region: "uk", Leases: h.leases, Now: h.clock.Now, NewID: obligationID,
+ View: view, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }}
+ if err := e.Run(ctx, park.Work{Host: p1Host, ID: chore.ID, Op: p1ID}); !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("a park of the target while its generation is live: %v", err)
+ }
+ if b, ok := h.f.buckets["data-uk"][obj]; !ok || b.body != "a live file" {
+ t.Fatal("the target's file was taken before the return")
+ }
+
+ // The source takes the host back: the return names its generation again
+ // and it recovers; the target's cleaning then runs the park.
+ rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Holder.Generation = "cur" })
+ h.recovered(t)
+ h.rollbackPass(t)
+ if rb := h.rb(t); rb.TargetPhase != provision.RbTargetCleaned {
+ t.Fatalf("the target did not clean: %+v", rb)
+ }
+ if _, ok := h.f.buckets["data-uk"][obj]; ok {
+ t.Fatal("the target's file was not retired by the cleaning")
+ }
+ if !archivedUnder(h.f, "data-uk", h.inc, location, obj) {
+ t.Fatal("the target's file was not archived")
+ }
+}
+
+// returnPreMove lists the pre-move generation's objects at the source's
+// location, with their bodies.
+func returnPreMove(t *testing.T, h *returnHarness, location string) map[string]string {
+ t.Helper()
+ out := map[string]string{}
+ for k, o := range h.f.buckets["data-ie"] {
+ if strings.HasPrefix(k, location+"/g-cur/") {
+ out[k] = o.body
+ }
+ }
+ if len(out) == 0 {
+ t.Fatal("fixture: no pre-move generation at the source's location")
+ }
+ return out
+}
+
+// TestReturnResidueParkRunsThroughTheInstall (fix round 2, I3; the source's
+// return-residue park, movesinstall.go parking): with the record's evidence
+// that the target admitted this move (its probe's admission of the move's
+// prepared generation), a dirty return's parking runs its residue chores
+// through the installer's own engine with the real view: the pre-move
+// generation is archived under parked/ and retired from the source's
+// location, and the return moves on.
+func TestReturnResidueParkRunsThroughTheInstall(t *testing.T) {
+ h := newReturnHarness(t, false)
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/")
+ pre := returnPreMove(t, h, location)
+ h.rcfg.ParkView = movesParkView(MovesRecordsFrom(h.s), h.vs, "data-ie", "ie")
+ if _, err := h.install(t); err != nil {
+ t.Fatalf("the return with the target's admission: %v", err)
+ }
+ if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase == provision.RbSourceParking {
+ t.Fatalf("the return is still parking: %+v", rb)
+ }
+ for k := range pre {
+ if _, ok := h.f.buckets["data-ie"][k]; ok {
+ t.Fatalf("%s was not retired", k)
+ }
+ if !archivedUnder(h.f, "data-ie", h.inc, location, k) {
+ t.Fatalf("%s was not archived", k)
+ }
+ }
+ if j := h.returnJournal(t); j.Steps[installStepResidueKept] != "" {
+ t.Fatalf("a return with admission evidence journalled the no-op: %q", j.Steps[installStepResidueKept])
+ }
+}
+
+// TestDirtyReturnWithoutAdmissionKeepsThePreMoveGeneration (fix round 3,
+// ruling (a)): the record carries no evidence that the target admitted this
+// move (the landing never reached probing, and the claim holds no admission
+// of the move's generation). The pre-move generation is still the source's
+// live state, so the return's parking is a journalled no-op: through the
+// real view and engine the return completes, the pre-move bytes are intact
+// and no residue park is queued.
+func TestDirtyReturnWithoutAdmissionKeepsThePreMoveGeneration(t *testing.T) {
+ h := newReturnHarnessWith(t, false, false)
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/")
+ pre := returnPreMove(t, h, location)
+ h.rcfg.ParkView = movesParkView(MovesRecordsFrom(h.s), h.vs, "data-ie", "ie")
+ if _, err := h.install(t); err != nil {
+ t.Fatalf("the return without admission evidence: %v", err)
+ }
+ if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase != provision.RbSourceRestored {
+ t.Fatalf("the return did not complete its install: %+v", rb)
+ }
+ for k, body := range pre {
+ if got, ok := h.f.buckets["data-ie"][k]; !ok || got.body != body {
+ t.Fatalf("%s changed", k)
+ }
+ }
+ j := h.returnJournal(t)
+ if got := j.Steps[installStepResidueKept]; !strings.Contains(got, "never admitted") {
+ t.Fatalf("the journal's no-op line: %q", got)
+ }
+ if got := j.Steps[landStepResidue]; got != "[]" {
+ t.Fatalf("the residue list %q, want none", got)
+ }
+ for _, o := range h.claim(t).Obligations {
+ if o.Kind == "park-generation" {
+ t.Fatalf("a residue park was queued: %+v", o)
+ }
+ }
+}
+
+// TestReturnResidueParkWaitsOnATransientRefusal (fix round 3): with the
+// target's admission recorded, a view refusal never turns the parking into
+// the no-op. The return waits at parking with the pre-move bytes intact and
+// no no-op line, and parks once the view answers.
+func TestReturnResidueParkWaitsOnATransientRefusal(t *testing.T) {
+ h := newReturnHarness(t, false)
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/")
+ pre := returnPreMove(t, h, location)
+ real := movesParkView(MovesRecordsFrom(h.s), h.vs, "data-ie", "ie")
+ down := true
+ h.rcfg.ParkView = func(ctx context.Context, obs hostclaim.Observed) (park.View, error) {
+ if down {
+ return park.View{}, fmt.Errorf("%w: the record read timed out", park.ErrUnsafe)
+ }
+ return real(ctx, obs)
+ }
+ if _, err := h.install(t); err == nil {
+ t.Fatal("the return went on through a view refusal")
+ }
+ if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase != provision.RbSourceParking {
+ t.Fatalf("the return left parking: %+v", rb)
+ }
+ if j := h.returnJournal(t); j.Steps[installStepResidueKept] != "" {
+ t.Fatalf("a view refusal became the no-op: %q", j.Steps[installStepResidueKept])
+ }
+ for k, body := range pre {
+ if got, ok := h.f.buckets["data-ie"][k]; !ok || got.body != body {
+ t.Fatalf("%s changed during the refusal", k)
+ }
+ }
+ down = false
+ if _, err := h.install(t); err != nil {
+ t.Fatalf("after the refusal: %v", err)
+ }
+ for k := range pre {
+ if _, ok := h.f.buckets["data-ie"][k]; ok {
+ t.Fatalf("%s was not retired once the view answered", k)
+ }
+ }
+}
+
+// TestDirtyReturnAtProbingWithoutAdmissionKeepsTheResidue (fix round 4): the
+// record enters probing before the probe runs (movesland.go), so a landing
+// that crashed or failed at probing before its admission and was then
+// rolled back dirty is not evidence that the target admitted. With the
+// holder holding no admission of the move's generation, the return keeps
+// the pre-move generation and completes.
+func TestDirtyReturnAtProbingWithoutAdmissionKeepsTheResidue(t *testing.T) {
+ h := newReturnHarnessWith(t, false, false)
+ if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, func(rec *provision.Record) error {
+ rec.Move.LandingPhase = provision.LandProbing
+ return nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/")
+ pre := returnPreMove(t, h, location)
+ h.rcfg.ParkView = movesParkView(MovesRecordsFrom(h.s), h.vs, "data-ie", "ie")
+ if _, err := h.install(t); err != nil {
+ t.Fatalf("the return at probing without an admission: %v", err)
+ }
+ if rb := h.read(t, p1Host).Move.Rollback; rb.SourcePhase != provision.RbSourceRestored {
+ t.Fatalf("the return did not complete its install: %+v", rb)
+ }
+ for k, body := range pre {
+ if got, ok := h.f.buckets["data-ie"][k]; !ok || got.body != body {
+ t.Fatalf("%s changed", k)
+ }
+ }
+ if got := h.returnJournal(t).Steps[installStepResidueKept]; !strings.Contains(got, "never admitted") {
+ t.Fatalf("the journal's no-op line: %q", got)
+ }
+}
diff --git a/internal/host/movesfinal.go b/internal/host/movesfinal.go
index 3c3ed9d32..4c6502015 100644
--- a/internal/host/movesfinal.go
+++ b/internal/host/movesfinal.go
@@ -69,11 +69,11 @@ import (
// still has the row or the op's journal. Plan 4's rollback visits reuse it.
//
// Arming. Every hook that exists on main is wired (the guard, park.Local, the
-// registry's delete, the versioned store), but the park engine's leases and
-// Plan 5's view and authorizer are not (ruling G17, as for the landing), so
-// sourceFinalArmed answers false in production and the pass only logs. The
-// tests arm it with fakes and the stand-ins below. No production record
-// reaches committed before Task 7 and Group D anyway.
+// registry's delete, the versioned store). The claim's park leases, view and
+// authorizer come from its MoveParks (armMovesParks, Plan 5 Task 9a), and
+// park.Local's authorizer is claim Plan 12's (M9); armMovesParks sets them
+// only with claim Plan 12, so sourceFinalArmed answers false in production
+// and the pass only logs until it lands. The tests arm it with fakes.
// The finalization's op journal steps (moves.Journal, the source's own
// journal of this op, which the cut began).
@@ -167,13 +167,13 @@ func (w *movesWorker) participantRound(ctx context.Context, s release.Store, rec
}
// sourceFinalArmed reports whether this box can run the finalization: every
-// hook it calls is wired, the park engine's leases and Plan 5's view and
-// authorizer included. Nothing stands in for a missing owner in production
-// (fix round 1): the tests inject their own. In production none of the
-// three is wired yet (ruling G17).
+// hook it calls is wired, the claim's three park hooks and claim Plan 12's
+// local authorizer (M9) included. Nothing stands in for a missing owner:
+// the tests inject their own. In production armMovesParks sets them only
+// with claim Plan 12, so this answers false there until it lands.
func sourceFinalArmed(cfg moveCfg) bool {
return cfg.Box != "" && cfg.DataDir != "" && cfg.Guard.DataDir != "" && cfg.Local.DataDir != "" &&
- cfg.ParkLeases != nil && cfg.ParkView != nil && cfg.ParkAuthorize != nil &&
+ parkHooksArmed(cfg) && cfg.ParkLocalAuthorize != nil &&
cfg.Versioned != nil && cfg.Route != nil && cfg.DeleteHeld != nil
}
@@ -215,7 +215,7 @@ func (w *movesWorker) finalizeSource(ctx context.Context, s release.Store, rec p
return nil
}
if !sourceFinalArmed(cfg) {
- w.notice(rec.Host+"#final", "moves: %s: the source's finalization of %s waits: this box's finalization is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.ID)
+ w.notice(rec.Host+"#final", "moves: %s: the source's finalization of %s waits: this box's finalization is not armed (the claim's park hooks, set from MoveParks once claim Plan 12 lands, are needed)", rec.Host, m.ID)
return nil
}
f := &sourceFinal{w: w, s: s, rec: rec, g: provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation}}
@@ -376,12 +376,24 @@ func (f *sourceFinal) sourceOf(obs hostclaim.Observed) error {
return nil
}
-// engine is the park engine over this box's own bucket, with the wired
-// leases, view and authorizer (sourceFinalArmed requires all three).
+// engine is the park engine over this box's own bucket, with the claim's
+// leases and view, and the claim's authorizer behind the finalization's own
+// check (sourceFinalArmed requires all three): the claim is still this
+// move's commit with this box its source, and the chore is this op's in
+// this region.
func (f *sourceFinal) engine(vs lease.VersionedStore) park.Engine {
cfg := f.cfg()
+ own := func(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
+ if err := f.sourceOf(obs); err != nil {
+ return err
+ }
+ if o.Op != f.g.ID || o.Region != cfg.Region {
+ return fmt.Errorf("%w: not this finalization's chore", park.ErrUnsafe)
+ }
+ return nil
+ }
return park.Engine{Store: vs, Claims: f.s, Bucket: cfg.OwnBucket, Region: cfg.Region, Leases: cfg.ParkLeases,
- Now: f.now, NewID: obligationID, View: cfg.ParkView, Authorize: cfg.ParkAuthorize, After: cfg.parkFault}
+ Now: f.now, NewID: obligationID, View: cfg.ParkView, Authorize: movesArmAuthorize(cfg.ParkAuthorize, own), After: cfg.parkFault}
}
// remote is step 1. done is true once the source region's location park has
@@ -479,11 +491,19 @@ func (f *sourceFinal) installation(ctx context.Context) (string, error) {
local.Box = cfg.Box
}
if local.Authorize == nil {
- // park.Local asks the same owner the engine does, over the claim
- // it read under the guard.
- authorize := cfg.ParkAuthorize
+ // park.Local asks claim Plan 12's local authorizer (M9), behind
+ // the finalization's own check over the claim it read under the
+ // guard. The claim's engine authorizer refuses a park-local, and
+ // nothing stands in: sourceFinalArmed requires it.
+ authorize := cfg.ParkLocalAuthorize
local.Authorize = func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error {
- return authorize(ctx, hostclaim.Observed{Host: host, Claim: c}, o)
+ if err := f.sourceOf(hostclaim.Observed{Host: host, Claim: c}); err != nil {
+ return err
+ }
+ if o.Op != id || o.Kind != "park-local" || o.Box != cfg.Box {
+ return fmt.Errorf("%w: not this finalization's local chore", park.ErrUnsafe)
+ }
+ return authorize(ctx, c, o)
}
}
if err := local.RetireInstallation(ctx, host, hostguard.Captured{Incarnation: captured.Incarnation, Stamp: stamp}, chore.ID, id); err != nil {
diff --git a/internal/host/movesfinal_test.go b/internal/host/movesfinal_test.go
index 236dc2510..c4cf9d286 100644
--- a/internal/host/movesfinal_test.go
+++ b/internal/host/movesfinal_test.go
@@ -94,6 +94,7 @@ func newFinalHarness(t *testing.T) *finalHarness {
h.cfg.Local = park.Local{DataDir: dd, Box: "ie-box"}
h.cfg.ParkLeases = h.leases
h.cfg.ParkView, h.cfg.ParkAuthorize = sourceTestView, sourceTestAuthorize
+ h.cfg.ParkLocalAuthorize = sourceTestLocalAuthorize
h.cfg.parkFault = h.lcfg.parkFault
h.cfg.finalFault = func(point string) error {
h.finalHits = append(h.finalHits, point)
@@ -751,9 +752,10 @@ func TestSourceParkWaitsUnarmedInProduction(t *testing.T) {
h := newFinalHarness(t)
h.commit(t)
for name, unarm := range map[string]func(*moveCfg){
- "no park leases": func(c *moveCfg) { c.ParkLeases = nil },
- "no park view": func(c *moveCfg) { c.ParkView = nil },
- "no park authorizer": func(c *moveCfg) { c.ParkAuthorize = nil },
+ "no park leases": func(c *moveCfg) { c.ParkLeases = nil },
+ "no park view": func(c *moveCfg) { c.ParkView = nil },
+ "no park authorizer": func(c *moveCfg) { c.ParkAuthorize = nil },
+ "no claim Plan 12 local authorizer": func(c *moveCfg) { c.ParkLocalAuthorize = nil },
} {
saved := h.cfg
unarm(&h.cfg)
@@ -866,6 +868,12 @@ func sourceTestView(_ context.Context, obs hostclaim.Observed) (park.View, error
GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil
}
+// sourceTestLocalAuthorize is claim Plan 12's local authorizer (M9) for the
+// source's park-local: sourceTestAuthorize over the claim park.Local read.
+func sourceTestLocalAuthorize(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error {
+ return sourceTestAuthorize(ctx, hostclaim.Observed{Host: p1Host, Claim: c}, o)
+}
+
func sourceTestAuthorize(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
c := obs.Claim
if obs.Absent || c == nil || c.Op.ID != p1ID || c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted ||
diff --git a/internal/host/movesinstall.go b/internal/host/movesinstall.go
index f90509e38..972297786 100644
--- a/internal/host/movesinstall.go
+++ b/internal/host/movesinstall.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log"
+ "os"
"path/filepath"
"sort"
"strconv"
@@ -53,20 +54,26 @@ import (
// of every pre-move artifact still at the flat paths, inside the host
// guard. placeStaged and restore then find nothing there, and refuse
// anything that appears;
-// - the stand-in park view's authoritative region (J24): the source's for
-// a landing, the target's for a return, each the other side of the move
-// before this side's probe is admitted.
+// - the authoritative region the claim's view names for its parks (J24,
+// §5.1): the source's for a landing, the target's for a return, each the
+// other side of the move before this side's probe is admitted.
//
// The phase-name convention is Plan 2's (G8): restored is written by the CAS
// that ends configured, after the database is installed and its markers
// written. Every record write goes through a provision transition (the
// record_seq rule). The installer is unarmed in production, as the landing
-// is (ruling G17): only the landing's pass and the tests call it.
+// is (Plan 5 Task 9a: until claim Plan 12 lands): only the landing's pass,
+// the recovery and the tests call it.
// installStepPreMove is the moves-owned rename of a return's pre-move
// artifacts (parkLocalArtifacts' step: the retirement id).
const installStepPreMove = "pre_move_parked"
+// installStepResidueKept is a dirty return's no-op parking (Task 9a fix
+// round 3, ruling (a)): the sentence saying the target never admitted the
+// move, so the pre-move generation stays the source's live state.
+const installStepResidueKept = "residue_kept"
+
// installStamp is what a phase step records besides the phase: at the CAS
// that ends configured, the installed database's digest (a landing records
// it in landing.installed_db_sha256) and the inputs a return rendered.
@@ -215,7 +222,22 @@ func newInstallRun(s release.Store, w *movesWorker, rec provision.Record, src in
ir.local.Box = cfg.Box
}
if ir.local.Authorize == nil {
- ir.local.Authorize = ir.authorizeChore
+ // park.Local's authorizer: claim Plan 12's local authorizer
+ // (ParkLocalAuthorize) behind this install's own check; the claim's
+ // engine authorizer refuses a park-local (M9). Until claim Plan 12
+ // lands this install's own rule stands alone: a named exception, the
+ // first of the two. claim Plan 12 F: drop the fallback once
+ // ParkLocalAuthorize is always wired.
+ plan12 := cfg.ParkLocalAuthorize
+ ir.local.Authorize = func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error {
+ if err := ir.authorizeChore(ctx, c, o); err != nil {
+ return err
+ }
+ if plan12 != nil {
+ return plan12(ctx, c, o)
+ }
+ return nil
+ }
}
return ir
}
@@ -347,10 +369,12 @@ func (ir *installRun) versioned(ctx context.Context) (lease.VersionedStore, erro
// @root), scoped to the location with the flip's time as the evidence
// horizon. On the target that is an earlier stay's residue; on the source
// of a dirty return it is the pre-move generations (§7.8, D6). Each chore is
-// run once per pass through the park engine, which takes the location's
-// lease itself (ErrPending is the expected answer: the acknowledgement waits
-// for the horizon, and is C4's runner's); the phase advances once every
-// chore's manifest exists and its deletion set is gone (Engine.Parked).
+// run once per pass through the park engine. The location's lease is the
+// install's own, held from here (a landing's from checking) through
+// published, and each chore runs under it through claim Plan 12's RunHeld
+// (M5). ErrPending is the expected answer: the acknowledgement waits for
+// the horizon, and is C4's runner's. The phase advances once every chore's
+// manifest exists and its deletion set is gone (Engine.Parked).
// Per-generation chores are what let the acknowledgement finish after the
// probe makes the new generation live here.
//
@@ -375,6 +399,24 @@ func (ir *installRun) parking(ctx context.Context) (bool, error) {
if err != nil {
return false, err
}
+ _, listed := j.Steps[landStepResidue]
+ if !listed && ir.src.Role == roleSource && !targetAdmitted(ir.rec.Move, obs.Claim) {
+ // A dirty return whose target never admitted this move (ruling
+ // (a), fix round 3): holder.generation is still the pre-move one,
+ // the source's live state, so there is no residue to park. This
+ // is decided from the claim's recorded admissions only (fix round
+ // 4: not the landing phase), never from a park view's
+ // refusal, which can be transient.
+ why := fmt.Sprintf("the target never admitted move %s: the claim holds no admission of its generation; the pre-move generation stays the source's live state", id)
+ j.Phase = phase
+ j.Mark(landStepResidue, "[]")
+ j.Mark(installStepResidueKept, why)
+ if err := j.Write(cfg.DataDir); err != nil {
+ return false, err
+ }
+ log.Printf("moves: %s: the %s parks nothing: %s", host, ir.src.Name, why)
+ return ir.advance(ctx, phase, next, installStamp{})
+ }
var gens []string
if raw, ok := j.Steps[landStepResidue]; ok {
if err := json.Unmarshal([]byte(raw), &gens); err != nil {
@@ -424,6 +466,15 @@ func (ir *installRun) parking(ctx context.Context) (bool, error) {
if ir.rec.Move.FlippedAt == nil {
return false, fmt.Errorf("move %s: the record has no flip time", id)
}
+ // The parks run under the build's own lease on the location, which the
+ // install holds from here (a landing from checking) through published:
+ // the residue scope's lease set is that lock (M6), so Engine.Run would
+ // wait on the install itself. claim Plan 12's RunHeld runs each chore
+ // under the held lease instead (M5).
+ held, waiting, err := ir.lease(ctx, holder)
+ if waiting || err != nil {
+ return false, err
+ }
scope := moves.ResidueScope(holder, *ir.rec.Move.FlippedAt)
if obs, err = hostclaim.Update(ctx, ir.s, obs, hostclaim.QueueResidueParks(id, cfg.Region, scope, gens, obligationID)); err != nil {
return false, fmt.Errorf("queue the residue parks: %w", err)
@@ -438,7 +489,10 @@ func (ir *installRun) parking(ctx context.Context) (bool, error) {
continue
}
w := park.Work{Host: host, ID: o.ID, Op: id}
- if err := eng.Run(ctx, w); err != nil && !errors.Is(err, park.ErrPending) {
+ if cfg.ParkRunHeld == nil {
+ return false, fmt.Errorf("move %s: %w", id, &ClaimDependencyError{Contracts: movesPlan12Contracts[:1]})
+ }
+ if err := cfg.ParkRunHeld(ctx, eng, w, held); err != nil && !errors.Is(err, park.ErrPending) {
return false, fmt.Errorf("park generation %s: %w", o.Generation, err)
}
parked, err := eng.Parked(ctx, w)
@@ -459,6 +513,24 @@ func (ir *installRun) parking(ctx context.Context) (bool, error) {
return ir.advance(ctx, phase, next, installStamp{})
}
+// targetAdmitted reports whether the claim carries evidence that the
+// move's target admitted a writer: the holder's admissions include the
+// move's prepared generation (an admission appends its record atomically,
+// and ReturnToSource keeps them). The record's landing phase is not
+// evidence: it enters probing before the probe runs (fix round 4), so a
+// landing that stopped at probing before its admission never admitted.
+func targetAdmitted(m *provision.MoveState, c *hostclaim.Claim) bool {
+ if m == nil || c == nil || c.Holder == nil {
+ return false
+ }
+ for _, a := range c.Holder.Admissions {
+ if a.Generation == m.ID {
+ return true
+ }
+ }
+ return false
+}
+
// building (§7.6, §5.1 step 2): the generation built from the seal's pinned
// replica versions under the location's lease (moves.BuildGeneration: the
// intent, the snapshot, the anchor, the completed install manifest, every
@@ -661,7 +733,14 @@ func (ir *installRun) local_(ctx context.Context) (bool, error) {
}
}
if !j.Done(landStepInventory) {
- paths := []string{host + ".db", host + ".db.gen", host, host + ".env"}
+ // Only what this install has placed (carry (b)): the files directory
+ // and <host>.env, which installFiles moved into place when nothing
+ // it did not write was there. The database and its marker are
+ // registered by configured, once the install proves the database at
+ // DBPath is this build (registerDatabase): registered here, before
+ // the install, a rollback's retirement of F would park whatever
+ // database stands at DBPath, a foreign one included.
+ paths := []string{host, host + ".env"}
if err := ir.local.RecordInventory(ctx, host, captured, "flat", paths, ir.src.ClaimCheck(phase)); err != nil {
return false, fmt.Errorf("record the inventory: %w", err)
}
@@ -939,7 +1018,19 @@ func (ir *installRun) configured(ctx context.Context) (bool, error) {
if err != nil {
return false, err
}
- hr, err := cfg.RenderHeld(ctx, ir.s, rt, ir.src.HoldOp)
+ // The render's ownership is journalled before its write (carry (c),
+ // fix round 1, I1): the hook runs once the rendered bytes are known and
+ // before they are placed, and journals their sha256 with the file's
+ // state before the render (pre_render). The rendered sha256 is journalled
+ // again as soon as the render answers, even when its config ack failed.
+ // A rollback's cleaning parks <host>.config.env only when its content is
+ // one this op rendered (rollbackOwnership).
+ hr, err := cfg.RenderHeld(ctx, ir.s, rt, ir.src.HoldOp, ir.notePreRender)
+ if hr.SHA256 != "" {
+ if jerr := ir.noteRendered(hr.SHA256); jerr != nil {
+ return false, errors.Join(err, jerr)
+ }
+ }
if err != nil {
return false, fmt.Errorf("render the held row: %w", err)
}
@@ -985,12 +1076,145 @@ func (ir *installRun) configured(ctx context.Context) (bool, error) {
case restoreMarked:
return true, nil // the record changed: the refusal, not an install
}
+ if err := ir.registerDatabase(ctx, obs, phase); err != nil {
+ return false, err
+ }
if ir.pastDeadline() {
return ir.refuse(ctx, refuseDeadline)
}
return ir.advance(ctx, phase, next, installStamp{InstalledDBSHA256: sha, RecoveryInputs: recovery})
}
+// registerDatabase registers the installed database and its marker under F
+// (carry (b)), once restore has proved the database at DBPath is this
+// install's build: renamed into place inside the guard, or found there
+// with the build's sha256. Journalled, so a replay registers once.
+func (ir *installRun) registerDatabase(ctx context.Context, obs hostclaim.Observed, phase string) error {
+ j, err := ir.journal()
+ if err != nil {
+ return err
+ }
+ if j.Done(landStepInventoryDB) {
+ return nil
+ }
+ captured, err := ir.capture(*obs.Claim.Holder)
+ if err != nil {
+ return err
+ }
+ host := ir.rec.Host
+ if err := ir.local.RecordInventory(ctx, host, captured, "flat", []string{host + ".db", host + ".db.gen"}, ir.src.ClaimCheck(phase)); err != nil {
+ return fmt.Errorf("register the installed database: %w", err)
+ }
+ j.Phase = phase
+ j.Mark(landStepInventoryDB, "")
+ if err := j.Write(ir.cfg().DataDir); err != nil {
+ return err
+ }
+ return ir.fault("after-" + landStepInventoryDB)
+}
+
+// configEnvPath is the held render's file, <DataDir>/<host>.config.env.
+func (ir *installRun) configEnvPath() string {
+ return filepath.Join(ir.cfg().DataDir, ir.rec.Host+".config.env")
+}
+
+// preRender is the landStepPreRender record: <host>.config.env's state
+// before the first held render ("absent", "sha256:<hex>" of a regular file,
+// or "other"), and the intended set: the sha256 of each content a render of
+// this op may have placed, journalled before its write. The set is pruned
+// to the file on disk (fix round 4): before each attempt it keeps the hash
+// of the content now on disk, only when it is already one of the set's, and
+// adds the new content's. So it never holds more than two hashes, and every
+// content this op placed that can still be on disk stays named.
+type preRender struct {
+ Before string `json:"before"`
+ Intended []string `json:"intended"`
+}
+
+// notePreRender is the held render's before-write hook. It hashes the file
+// now on disk (Lstat'ed, never followed) and journals the intended set as
+// {that hash, if the set already names it} plus {the new content's hash},
+// before the write. A file whose content the set does not name (the earlier
+// state, or foreign content) contributes nothing, and the earlier-state
+// record is kept as first seen. The renderer replaces the file by an atomic
+// rename (appconfig's renderExecConfigBefore), so the file on disk is always
+// one whole content: a hash pruned here is never on disk again unless this
+// op renders it again, when it is journalled again first.
+func (ir *installRun) notePreRender(sha string) error {
+ if sha == "" {
+ return errors.New("moves: the held render names no content")
+ }
+ j, err := ir.journal()
+ if err != nil {
+ return err
+ }
+ path := ir.configEnvPath()
+ disk := ""
+ fi, err := os.Lstat(path)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ case err != nil:
+ return err
+ case fi.Mode().IsRegular():
+ if disk, err = fileSHA256(path); err != nil {
+ return err
+ }
+ }
+ var pre preRender
+ if raw, ok := j.Steps[landStepPreRender]; ok {
+ if err := json.Unmarshal([]byte(raw), &pre); err != nil {
+ return fmt.Errorf("the journal's %s step: %w", landStepPreRender, err)
+ }
+ } else {
+ pre.Before = "other"
+ switch {
+ case errors.Is(err, os.ErrNotExist) || fi == nil:
+ pre.Before = "absent"
+ case disk != "":
+ pre.Before = disk
+ }
+ }
+ want := "sha256:" + sha
+ var next []string
+ if disk != "" && containsString(pre.Intended, disk) {
+ next = append(next, disk)
+ }
+ if want != disk || len(next) == 0 {
+ next = append(next, want)
+ }
+ if strings.Join(next, " ") == strings.Join(pre.Intended, " ") {
+ return nil
+ }
+ pre.Intended = next
+ body, err := json.Marshal(pre)
+ if err != nil {
+ return err
+ }
+ j.Mark(landStepPreRender, string(body))
+ if err := j.Write(ir.cfg().DataDir); err != nil {
+ return err
+ }
+ return ir.fault("after-" + landStepPreRender)
+}
+
+// noteRendered journals the sha256 (hex) of the file the held render wrote,
+// as "sha256:<hex>": the latest render's, which is what the file holds.
+func (ir *installRun) noteRendered(sha string) error {
+ j, err := ir.journal()
+ if err != nil {
+ return err
+ }
+ want := "sha256:" + sha
+ if j.Steps[landStepRendered] == want {
+ return nil
+ }
+ j.Mark(landStepRendered, want)
+ if err := j.Write(ir.cfg().DataDir); err != nil {
+ return err
+ }
+ return ir.fault("after-" + landStepRendered)
+}
+
// recoveryInstallSource binds the installer to a dirty return on the source
// (§7.8 source table, the dirty branch; rulings J13, J14, J20): the record
// rolling_back and dirty with the source at an install phase; the claim
diff --git a/internal/host/movesinstall_test.go b/internal/host/movesinstall_test.go
index 631dc7fc1..5f29a618d 100644
--- a/internal/host/movesinstall_test.go
+++ b/internal/host/movesinstall_test.go
@@ -56,10 +56,32 @@ type returnHarness struct {
// flat paths in the claim's inventory at S and queues its park-local, as
// Task 5's claimed does when the inventory lists them (J13).
func newReturnHarness(t *testing.T, registered bool) *returnHarness {
+ t.Helper()
+ return newReturnHarnessWith(t, registered, true)
+}
+
+// returnJournal is the return's own op journal on the source.
+func (h *returnHarness) returnJournal(t *testing.T) *moves.Journal {
+ t.Helper()
+ j, err := moves.ReadJournal(h.sdir, p1Host, p1RB)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return j
+}
+
+// newReturnHarnessWith is newReturnHarness with the target's admission
+// recorded or not (fix round 3): admitted, the claim holds the target's
+// probe admission of the move's prepared generation, the evidence a dirty
+// return needs before it parks the pre-move generations.
+func newReturnHarnessWith(t *testing.T, registered, admitted bool) *returnHarness {
t.Helper()
ctx := context.Background()
h := &returnHarness{landingHarness: newLandingHarness(t)}
h.flip(t)
+ if admitted {
+ admitTarget(t, h.landingHarness)
+ }
h.sdir = h.cfg.DataDir
rec := h.read(t, p1Host)
h.baseline = *rec.Move.Source
@@ -187,8 +209,15 @@ func (h *returnHarness) sourceCfg(t *testing.T) moveCfg {
}
c.Guard = hostguard.Guard{DataDir: h.sdir, LockDir: t.TempDir()}
c.Local = park.Local{DataDir: h.sdir, Box: "ie-box"}
- c.RenderHeld = func(_ context.Context, _ release.Store, rt registry.Route, op string) (appconfig.HeldRender, error) {
+ c.ParkView = recordView(h.s, "ie", roleSource)
+ c.RenderHeld = func(_ context.Context, _ release.Store, rt registry.Route, op string, before func(string) error) (appconfig.HeldRender, error) {
h.renders++
+ if h.renderHook != nil {
+ return h.renderHook(before)
+ }
+ if err := before(h.render.SHA256); err != nil {
+ return appconfig.HeldRender{}, err
+ }
if op != p1RB || rt.HoldOp != p1RB {
t.Errorf("rendered for %q over a row held by %q, want the rollback %s", op, rt.HoldOp, p1RB)
}
diff --git a/internal/host/movesland.go b/internal/host/movesland.go
index 6367404db..207d9654c 100644
--- a/internal/host/movesland.go
+++ b/internal/host/movesland.go
@@ -70,14 +70,15 @@ import (
// paths: the archive is extracted into the op's own staging and moved into
// place only when nothing it did not write is there.
//
-// Arming (ruling G17). armPhase1 wires the hooks that exist on main (the
-// guard, park.Local, the own-location lease, RenderHeld, the registry
-// setters, the restore-verification record, the marker writer), and leaves
-// ParkLeases, ParkView and ParkAuthorize nil: the production park engine
-// needs claim Plan 6b's regional runner for the acknowledgement half and
-// Plan 5's view and authorizer. landingArmed therefore answers false in
-// production and the pass logs a standing notice instead of landing; the
-// tests arm everything with fakes and the stand-ins below. Under
+// Arming (ruling G17; Plan 5 Task 9a). armPhase1 wires the hooks that exist
+// on main (the guard, park.Local, the own-location lease, RenderHeld, the
+// registry setters, the restore-verification record, the marker writer).
+// ParkLeases, ParkView and ParkAuthorize are the claim's (MoveParks), and
+// ParkRunHeld claim Plan 12's (M5), all set by armMovesParks only once claim
+// Plan 12 lands; nothing stands in for them. landingArmed therefore answers
+// false in production and the pass logs a standing notice instead of
+// landing; the tests arm everything with fakes (movesparkfakes_test.go).
+// Under
// CARLOS_HOST_CLAIM the landing refuses at checking with a recorded reason
// (ruling G16). The policy its held writes need, hostbox.Policy.Moves, is
// MovesClaimPolicy (movespolicy.go) and the timers import already selects
@@ -106,13 +107,16 @@ const (
landStepStaged = "staged" // the archive is extracted into <JournalDir>/extract/ (JSON stagedFiles)
landStepFilesPlaced = "files_placed" // the install receipt: the staged tree is at the live paths (JSON stagedFiles)
landStepExtracted = "extracted" // the archive is in the files directory and <host>.env
- landStepInventory = "inventory" // the new paths are registered under F
+ landStepInventory = "inventory" // the placed files and env are registered under F
landStepTimers = "timers" // the timers table is the sealed snapshot
landStepColumns = "columns" // replication and next_due carried over
landStepVerify = "verify" // the restore-verification entry is gone
landStepFilesManifest = "files_manifest" // the prepared files manifest (C8; skipped until it exists)
landStepFilesGen = "files_gen" // .files-gen written
landStepPreInstall = "pre_install" // the database's siblings absent before its install (JSON preInstall)
+ landStepInventoryDB = "inventory_db" // the installed database and its marker are registered under F (carry (b))
+ landStepPreRender = "pre_render" // the held render's ownership before its write (JSON preRender, carry (c))
+ landStepRendered = "rendered" // the held render's file, "sha256:<hex>", journalled even when its ack failed (carry (c))
)
// landingFilesManifestSkipped is what the files_manifest step records while
@@ -128,11 +132,20 @@ type landingRun struct {
guard hostguard.Guard
}
+// parkHooksArmed reports whether the claim's three park hooks are wired:
+// the leases, the view and the authorizer (M17). Every moves arm requires
+// all three; nothing stands in for a missing one.
+func parkHooksArmed(cfg moveCfg) bool {
+ return cfg.ParkLeases != nil && cfg.ParkView != nil && cfg.ParkAuthorize != nil
+}
+
// landingArmed reports whether this box can run the landing: every hook
-// the phases call is wired. In production ParkLeases, ParkView and
-// ParkAuthorize are not (ruling G17), so this answers false there.
+// the phases call is wired, the claim's three park hooks and claim Plan 12's
+// RunHeld (M5: the residue parks run under the landing's own lease) among
+// them. In production armMovesParks sets those only with claim Plan 12, so
+// this answers false there until it lands.
func landingArmed(cfg moveCfg) bool {
- return cfg.Box != "" && cfg.Guard.DataDir != "" && cfg.Local.DataDir != "" && cfg.ParkLeases != nil &&
+ return cfg.Box != "" && cfg.Guard.DataDir != "" && cfg.Local.DataDir != "" && parkHooksArmed(cfg) && cfg.ParkRunHeld != nil &&
cfg.OwnLease != nil && cfg.RenderHeld != nil && cfg.ReplaceTimers != nil && cfg.SetReplication != nil &&
cfg.SetNextDue != nil && cfg.RemoveVerifyEntry != nil && cfg.WriteGen != nil && cfg.Route != nil &&
cfg.FilesDir != nil && cfg.Versioned != nil && cfg.ReleaseReady != nil && cfg.SecretsPriv != nil
@@ -154,7 +167,7 @@ func (w *movesWorker) landingPass(ctx context.Context, s release.Store, rec prov
return err
}
if !landingArmed(cfg) {
- w.notice(rec.Host+"#landing", "moves: %s: the landing of %s waits: this box's landing is not armed (claim Plan 6b's regional runner and Plan 5's park view are needed)", rec.Host, rec.Move.ID)
+ w.notice(rec.Host+"#landing", "moves: %s: the landing of %s waits: this box's landing is not armed (the claim's park hooks, set from MoveParks once claim Plan 12 lands, are needed)", rec.Host, rec.Move.ID)
return nil
}
l := &landingRun{w: w, s: s, rec: rec, g: provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation}}
@@ -590,6 +603,12 @@ func (l *landingRun) checking(ctx context.Context) (bool, error) {
return false, fmt.Errorf("queue the local park: %w", err)
}
}
+ // Spec §7.6 checking: the lease on the location in this box's own
+ // bucket, held through published (M5). The residue parks run under it
+ // (ParkRunHeld), so the landing never waits on its own lock.
+ if _, waiting, err := acquireOwnLease(ctx, l.w, host, holder); waiting || err != nil {
+ return false, err
+ }
return l.advance(ctx, provision.LandChecking, provision.LandParking, nil)
}
diff --git a/internal/host/movesland_test.go b/internal/host/movesland_test.go
index e4c39e4ab..e4c13a06d 100644
--- a/internal/host/movesland_test.go
+++ b/internal/host/movesland_test.go
@@ -46,33 +46,38 @@ import (
//
// fakeParkLeases is one lease table for the landing's own-location lease
// and the park engine's: a second acquisition of a held location is
-// refused, so a lease the landing still held while a residue park ran
-// would fail the test (round-2 finding 9).
+// refused. The landing holds its lease from checking through published
+// (spec §7.6), so its residue parks run under it through claim Plan 12's
+// RunHeld (testRunHeld, M5); an Engine.Run that took the lock again would
+// fail the test.
type landingHarness struct {
*cutHarness
- tdir string // the target's data directory
- leases *fakeParkLeases
- lw *movesWorker
- lcfg moveCfg
- route registry.Route
- rows int // rows the source database holds
- n uint64
- replaced [][]registry.Timer
- repl []string
- nextDue []int64
- renders int
- render func() (appconfig.HeldRender, error)
- release moves.Check
- landAt string
- landHits []string
- landDone bool
- parkAt string
- parkHits []string
- parkDone bool
- parkView func(context.Context, hostclaim.Observed) (park.View, error)
- landFault func(point string) error
- store lease.VersionedStore // when set, the target's versioned store
+ tdir string // the target's data directory
+ leases *fakeParkLeases
+ lw *movesWorker
+ lcfg moveCfg
+ route registry.Route
+ rows int // rows the source database holds
+ n uint64
+ replaced [][]registry.Timer
+ repl []string
+ nextDue []int64
+ renders int
+ render func() (appconfig.HeldRender, error)
+ // renderHook, when set, is the whole held render: it gets the
+ // before-write callback the installer passes (fix round 1, I1).
+ renderHook func(before func(string) error) (appconfig.HeldRender, error)
+ release moves.Check
+ landAt string
+ landHits []string
+ landDone bool
+ parkAt string
+ parkHits []string
+ parkDone bool
+ parkView func(context.Context, hostclaim.Observed) (park.View, error)
+ landFault func(point string) error
+ store lease.VersionedStore // when set, the target's versioned store
// probe, when set, is the target's probe wake (Plan 3 Task 6).
probe func(context.Context, activator.ProbeConfig) (activator.ProbeResult, error)
// probeLive, when set, is the activator's live probe entry.
@@ -92,8 +97,13 @@ type landingHarness struct {
releaseErr error
// ownAuthorize drops the park authorizer the target would inherit from
// the source's configuration (the finalization harness sets one), so
- // the target's parks use their own stand-ins (Plan 5 Task 3).
+ // the target's parks use the allowing fake behind their own checks
+ // (Plan 5 Task 3).
ownAuthorize bool
+ // parkLeases and runHeld, when set, replace the park fakes' leases and
+ // claim Plan 12's RunHeld (Plan 5 Task 9a, M5).
+ parkLeases park.Leases
+ runHeld func(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error
}
// fakeParkLeases refuses a second acquisition of a held location.
@@ -340,26 +350,36 @@ func (h *landingHarness) landingCfg(t *testing.T) moveCfg {
}
c.Guard = hostguard.Guard{DataDir: h.tdir, LockDir: t.TempDir()}
c.Local = park.Local{DataDir: h.tdir, Box: "uk-box"}
- c.ParkLeases = h.leases
- c.ParkView = func(ctx context.Context, obs hostclaim.Observed) (park.View, error) {
- if h.parkView == nil {
- return park.View{}, errors.New("no view override")
- }
- return h.parkView(ctx, obs)
+ inherited := c.ParkAuthorize
+ armParkFakes(&c, h.s, h.leases, c.Region, roleTarget)
+ if h.parkView != nil {
+ c.ParkView = func(ctx context.Context, obs hostclaim.Observed) (park.View, error) { return h.parkView(ctx, obs) }
+ }
+ if inherited != nil && !h.ownAuthorize {
+ c.ParkAuthorize = inherited
}
- if h.parkView == nil {
- c.ParkView = nil
+ if h.parkLeases != nil {
+ c.ParkLeases = h.parkLeases
}
- if h.ownAuthorize {
- c.ParkAuthorize = nil
+ if h.runHeld != nil {
+ c.ParkRunHeld = h.runHeld
}
c.OwnLease = h.leases.Acquire
if h.store != nil {
c.Versioned = func(context.Context, string, string) (lease.VersionedStore, error) { return h.store, nil }
}
- c.RenderHeld = func(context.Context, release.Store, registry.Route, string) (appconfig.HeldRender, error) {
+ c.RenderHeld = func(_ context.Context, _ release.Store, _ registry.Route, _ string, before func(string) error) (appconfig.HeldRender, error) {
h.renders++
- return h.render()
+ if h.renderHook != nil {
+ return h.renderHook(before)
+ }
+ hr, err := h.render()
+ if hr.SHA256 != "" {
+ if berr := before(hr.SHA256); berr != nil {
+ return appconfig.HeldRender{}, berr
+ }
+ }
+ return hr, err
}
c.ReleaseReady = func(context.Context, release.Store, provision.Record, string) moves.Check { return h.release }
c.ReplaceTimers = func(_ context.Context, host string, rows []registry.Timer) error {
@@ -719,8 +739,14 @@ func TestLandingCheckingRequiresClaimEvidenceAndTouch(t *testing.T) {
if err != nil || captured.Stamp != c.Holder.Stamp || captured.Incarnation != h.inc {
t.Fatalf("active_stamp %+v %v, want F %d", captured, err, c.Holder.Stamp)
}
- if len(h.leases.acquired) != 0 {
- t.Fatalf("checking took a lease: %v", h.leases.acquired)
+ // Spec §7.6 checking (Plan 5 Task 9a, M5): the location's lease in
+ // the target's own bucket, taken once and held into parking.
+ want := "data-uk/" + strings.TrimSuffix(c.Holder.Location, "/")
+ if len(h.leases.acquired) != 1 || h.leases.acquired[0] != want {
+ t.Fatalf("checking took %v, want the location's lease %s", h.leases.acquired, want)
+ }
+ if held := h.leases.holding(); len(held) != 1 || held[0] != want {
+ t.Fatalf("after checking the landing holds %v", held)
}
})
t.Run("a nested row is refused for a rollback", func(t *testing.T) {
diff --git a/internal/host/moveslandcommit.go b/internal/host/moveslandcommit.go
index 721073acf..597d19885 100644
--- a/internal/host/moveslandcommit.go
+++ b/internal/host/moveslandcommit.go
@@ -42,7 +42,8 @@ import (
// rest and only clears the record.
//
// Unarmed in production with the rest of the landing: landingArmed fails
-// there (ruling G17), and the open needs ReleaseHeld.
+// there until claim Plan 12 lands (Plan 5 Task 9a), and the open needs
+// ReleaseHeld.
// probed commits the landing: the record's CAS, then the committed arm.
func (l *landingRun) probed(ctx context.Context) (bool, error) {
diff --git a/internal/host/moveslandpark.go b/internal/host/moveslandpark.go
index 2c08a40b0..76244f0b9 100644
--- a/internal/host/moveslandpark.go
+++ b/internal/host/moveslandpark.go
@@ -15,17 +15,39 @@ import (
"github.com/carlosframework/platform/internal/provision"
)
-// The installer's park and lease adapters (moves spec §6.4, §7.6; ruling
-// G17), shared by the landing and the dirty return: the park engine over
-// this box's own bucket for the residue parks, the stand-in view and
-// authorizer the tests use until Plan 5's providers, the generation
-// classifier they share, and the own-location lease the build holds. The
-// phases are in movesinstall.go, the landing's pass in movesland.go.
+// The installer's park and lease adapters (moves spec §6.4, §7.6), shared by
+// the landing and the dirty return: the park engine over this box's own
+// bucket for the residue parks, with the claim's view and authorizer
+// (MoveParks, armMovesParks), the generation classifier the view shares, and
+// the own-location lease the build holds from checking through published.
+// The phases are in movesinstall.go, the landing's pass in movesland.go.
+//
+// Nothing stands in for the claim's view or authorizer (Plan 5 Task 9a): an
+// arm without ParkView and ParkAuthorize is not armed, and the tests inject
+// fakes. Each arm keeps its own operation check around the claim's
+// authorizer (M17, movesArmAuthorize). The one exception is park.Local's
+// authorizer for the install's own local chores (M9): until claim Plan 12's
+// local authorizer is wired as ParkLocalAuthorize, the install keeps its own
+// rule (installRun.authorizeChore).
-// authorizeMoveChore is park.Local.Authorize's and park.Engine.Authorize's
-// stand-in rule until Plan 5: a chore is run only when it is the op's own
-// and the claim still names the op, at the install's kind, with this box
-// executing.
+// movesArmAuthorize is an arm's park authorizer (M17): the arm's own
+// operation check, then the claim's authorizer, ParkAuthorize from the
+// claim's MoveParks. A missing claim authorizer refuses; nothing stands in.
+func movesArmAuthorize(claim, own func(context.Context, hostclaim.Observed, hostclaim.Obligation) error) func(context.Context, hostclaim.Observed, hostclaim.Obligation) error {
+ return func(ctx context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
+ if claim == nil || own == nil {
+ return fmt.Errorf("%w: no park authorizer", park.ErrUnsafe)
+ }
+ if err := own(ctx, obs, o); err != nil {
+ return err
+ }
+ return claim(ctx, obs, o)
+ }
+}
+
+// authorizeMoveChore is an install's own operation check: a chore is run
+// only when it is the op's own and the claim still names the op, at the
+// install's kind, with this box executing.
func authorizeMoveChore(c *hostclaim.Claim, o hostclaim.Obligation, opID, kind, box string) error {
if c == nil || o.Op != opID || c.Op.ID != opID || c.Op.Kind != kind || c.Op.Exec != box {
return fmt.Errorf("%w: not this install's chore", park.ErrUnsafe)
@@ -33,13 +55,17 @@ func authorizeMoveChore(c *hostclaim.Claim, o hostclaim.Obligation, opID, kind,
return nil
}
-// authorizeChore is park.Local.Authorize's stand-in for this install.
+// authorizeChore is park.Local.Authorize for this install's own local
+// chores while claim Plan 12's local authorizer is missing: a named
+// exception (M9), the only park stand-in left in production.
+// claim Plan 12 F: replace it with ParkLocalAuthorize once the MoveParks
+// capability carries the local authorizer.
func (ir *installRun) authorizeChore(_ context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error {
return authorizeMoveChore(c, o, ir.g.ID, ir.src.kind(), ir.cfg().Box)
}
-// authorizeObligation is park.Engine.Authorize's stand-in (ruling G17): the
-// same rule over an observation.
+// authorizeObligation is the install's own operation check over an
+// observation, which the engine's authorizer runs before the claim's.
func (ir *installRun) authorizeObligation(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
if obs.Absent {
return park.ErrUnsafe
@@ -47,8 +73,8 @@ func (ir *installRun) authorizeObligation(_ context.Context, obs hostclaim.Obser
return authorizeMoveChore(obs.Claim, o, ir.g.ID, ir.src.kind(), ir.cfg().Box)
}
-// authorizeObligation is the landing's engine stand-in, the installer's rule
-// for a landing.
+// authorizeObligation is the landing's own operation check, the installer's
+// rule for a landing.
func (l *landingRun) authorizeObligation(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
if obs.Absent {
return park.ErrUnsafe
@@ -56,34 +82,6 @@ func (l *landingRun) authorizeObligation(_ context.Context, obs hostclaim.Observ
return authorizeMoveChore(obs.Claim, o, l.g.ID, hostclaim.KindLanding, l.cfg().Box)
}
-// standInView is park.Engine.View's stand-in (ruling G17): the location's
-// objects classified by key, seen from region, and the other side of the
-// move authoritative, which §5.1's table says it is before this side's
-// probe is admitted (every install phase). role names this side (ruling
-// J24): a landing (roleTarget) works while the source's region, From, is
-// authoritative; a dirty return (roleSource) while the target's, To, is.
-// Plan 5's provider replaces it; a test may set cfg.ParkView to the view a
-// later state has.
-func standInView(region, role string, m *provision.MoveState) func(context.Context, hostclaim.Observed) (park.View, error) {
- return func(_ context.Context, obs hostclaim.Observed) (park.View, error) {
- if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || m == nil {
- return park.View{}, park.ErrUnsafe
- }
- var authoritative string
- switch role {
- case roleTarget:
- authoritative = m.From
- case roleSource:
- authoritative = m.To
- default:
- return park.View{}, park.ErrUnsafe
- }
- location := strings.TrimSuffix(obs.Claim.Holder.Location, "/")
- return park.View{ClaimETag: obs.ETag, Region: region, AuthoritativeRegion: authoritative,
- GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil
- }
-}
-
// landingMarkerGeneration is what landingGenerationOf answers for the
// location's generation.json. The marker is the one key every generation
// reuses (§5.1): the probe's admission supersedes an earlier stay's marker
@@ -132,35 +130,37 @@ func isHexString(s string) bool {
return s != ""
}
-// engine is the park engine over this box's own bucket: the wired leases,
-// and the view and authorizer Plan 5 supplies, or the stand-ins.
+// engine is the park engine over this box's own bucket with the claim's
+// leases and view, and the claim's authorizer behind this install's own
+// check (the arming requires all three).
func (ir *installRun) engine(vs lease.VersionedStore) park.Engine {
cfg := ir.cfg()
- view, auth := cfg.ParkView, cfg.ParkAuthorize
- if view == nil {
- view = standInView(cfg.Region, ir.src.Role, ir.rec.Move)
- }
- if auth == nil {
- auth = ir.authorizeObligation
- }
return park.Engine{Store: vs, Claims: ir.s, Bucket: cfg.OwnBucket, Region: cfg.Region, Leases: cfg.ParkLeases,
- Now: cfg.Now, NewID: obligationID, View: view, Authorize: auth, After: cfg.parkFault}
+ Now: cfg.Now, NewID: obligationID, View: cfg.ParkView, Authorize: movesArmAuthorize(cfg.ParkAuthorize, ir.authorizeObligation), After: cfg.parkFault}
}
// lease is the own-location lease the build holds (§7.6 checking: "hold it
// through published"), kept in the worker's per-host map across passes and
// re-acquired after a restart. waiting is true when another holder has it.
func (ir *installRun) lease(ctx context.Context, holder hostclaim.Holder) (held park.HeldLease, waiting bool, err error) {
- cfg := ir.cfg()
- host := ir.rec.Host
- ir.w.mu.Lock()
- held = ir.w.landLeases[host]
- ir.w.mu.Unlock()
+ return acquireOwnLease(ctx, ir.w, ir.rec.Host, holder)
+}
+
+// acquireOwnLease takes host's own-location lease in this box's own bucket,
+// or answers the one w already holds while it still checks. A landing takes
+// it at checking (spec §7.6) and keeps it through published; its residue
+// parks run under it (ParkRunHeld, M5). waiting is true when another holder
+// has it.
+func acquireOwnLease(ctx context.Context, w *movesWorker, host string, holder hostclaim.Holder) (held park.HeldLease, waiting bool, err error) {
+ cfg := w.cfg
+ w.mu.Lock()
+ held = w.landLeases[host]
+ w.mu.Unlock()
if held != nil {
if err := held.Check(ctx); err == nil {
return held, false, nil
}
- ir.releaseLease()
+ releaseOwnLease(w, host)
}
held, err = cfg.OwnLease(ctx, cfg.OwnBucket, strings.TrimSuffix(holder.Location, "/"))
if err != nil {
@@ -171,12 +171,12 @@ func (ir *installRun) lease(ctx context.Context, holder hostclaim.Holder) (held
}
return nil, false, fmt.Errorf("the location's lease: %w", err)
}
- ir.w.mu.Lock()
- if ir.w.landLeases == nil {
- ir.w.landLeases = map[string]park.HeldLease{}
+ w.mu.Lock()
+ if w.landLeases == nil {
+ w.landLeases = map[string]park.HeldLease{}
}
- ir.w.landLeases[host] = held
- ir.w.mu.Unlock()
+ w.landLeases[host] = held
+ w.mu.Unlock()
return held, false, nil
}
diff --git a/internal/host/movesparkfakes_test.go b/internal/host/movesparkfakes_test.go
new file mode 100644
index 000000000..fbaca8928
--- /dev/null
+++ b/internal/host/movesparkfakes_test.go
@@ -0,0 +1,76 @@
+package host
+
+import (
+ "context"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The park fakes the moves tests inject (Plan 5 Task 9a). Production has no
+// stand-in for the claim's view or authorizer; these are the rules the
+// stand-ins had, kept for the tests that do not exercise the real view
+// (movesParkView, movessources_test.go).
+
+// standInView is the tests' park view: the location's objects classified by
+// key, seen from region, and the other side of the move authoritative, which
+// §5.1's table says it is before this side's probe is admitted (every install
+// phase). role names this side (ruling J24): a landing (roleTarget) works
+// while the source's region, From, is authoritative; a dirty return
+// (roleSource) while the target's, To, is.
+func standInView(region, role string, m *provision.MoveState) func(context.Context, hostclaim.Observed) (park.View, error) {
+ return func(_ context.Context, obs hostclaim.Observed) (park.View, error) {
+ if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || m == nil {
+ return park.View{}, park.ErrUnsafe
+ }
+ var authoritative string
+ switch role {
+ case roleTarget:
+ authoritative = m.From
+ case roleSource:
+ authoritative = m.To
+ default:
+ return park.View{}, park.ErrUnsafe
+ }
+ location := strings.TrimSuffix(obs.Claim.Holder.Location, "/")
+ return park.View{ClaimETag: obs.ETag, Region: region, AuthoritativeRegion: authoritative,
+ GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil
+ }
+}
+
+// recordView is standInView over the record's move as the store holds it
+// when the engine asks.
+func recordView(s release.Store, region, role string) func(context.Context, hostclaim.Observed) (park.View, error) {
+ return func(ctx context.Context, obs hostclaim.Observed) (park.View, error) {
+ if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil {
+ return park.View{}, park.ErrUnsafe
+ }
+ h := obs.Claim.Holder
+ rec, err := provision.GetRecord(ctx, s, h.Account, h.App, obs.Host)
+ if err != nil {
+ return park.View{}, err
+ }
+ return standInView(region, role, rec.Move)(ctx, obs)
+ }
+}
+
+// allowParkAuthorize is the tests' claim authorizer: it allows every chore,
+// so each arm's own operation check (movesArmAuthorize) decides alone.
+func allowParkAuthorize(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }
+
+// allowLocalAuthorize is the tests' claim Plan 12 local authorizer (M9).
+func allowLocalAuthorize(context.Context, *hostclaim.Claim, hostclaim.Obligation) error { return nil }
+
+// armParkFakes arms c's park hooks with the tests' fakes: leases, the
+// record-backed stand-in view for region and role, the allowing claim
+// authorizer and claim Plan 12's pieces (testRunHeld, allowLocalAuthorize).
+func armParkFakes(c *moveCfg, s release.Store, leases park.Leases, region, role string) {
+ c.ParkLeases = leases
+ c.ParkView = recordView(s, region, role)
+ c.ParkAuthorize = allowParkAuthorize
+ c.ParkRunHeld = testRunHeld
+ c.ParkLocalAuthorize = allowLocalAuthorize
+}
diff --git a/internal/host/movesparks.go b/internal/host/movesparks.go
index 9ec7aa2d6..8abaf8631 100644
--- a/internal/host/movesparks.go
+++ b/internal/host/movesparks.go
@@ -46,8 +46,8 @@ import (
// Production (K4). The staging park and the close are production-live only
// after Plan 5 Task 9 (park arming) and claim Plans 10 and 11. The committed
// arm runs only where the landing is armed (landingArmed), and production
-// wires neither the park engine's leases nor Plan 5's view and authorizer
-// until Task 9; the commit and the open themselves need the probe wake of
+// sets the claim's park leases, view and authorizer (armMovesParks, Plan 5
+// Task 9a) only once claim Plan 12 lands; the commit and the open themselves need the probe wake of
// claim Plan 11's launcher; and the source's location park is acknowledged
// by C4's regional runner, whose moves evidence is claim Plan 10's. Until
// then the tests arm everything with fakes.
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index 4edda2f80..4f24f3df5 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -184,8 +184,11 @@ func (w *movesWorker) mirror(ctx context.Context, dstRegion string) (lease.Mirro
// service that answers whether a child is down, and the exec backend (the
// Phase 1 launch environment and S2). Called once, before svc.Start: with
// the Phase 1 switch on it sets eb.Strict, which the spawn path reads; with
-// it off eb.Strict stays nil, so every spawn is main's.
-func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config) {
+// it off eb.Strict stays nil, so every spawn is main's. providers is the
+// claim wiring's (newClaimWiring builds it before this): with both switches
+// on, the park arms take their engine from its MoveParks (armMovesParks,
+// Plan 5 Task 9a), behind the claim Plan 12 guard.
+func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry, svc *activator.Service, eb *activator.ExecBackend, writes hostwrite.Config, providers ClaimProviders) {
readyDelta, ok := moveReadyDeltaFromEnv(os.Getenv)
if !ok {
log.Printf("agent: %s=%q ignored: want a whole number of bytes, 0 or more — keeping the built-in default of %d", moveReadyDeltaEnv, os.Getenv(moveReadyDeltaEnv), defaultMoveReadyDelta)
@@ -239,6 +242,13 @@ func armMovesPass(hib activator.Config, dataDir string, reg *registry.Registry,
armPhase1(&cfg, w, hib, dataDir, reg, svc, eb, writes, os.Getenv)
w.holds = newMovesHoldView(cfg.Box, cfg.Region, hostclaim.Read, holdStageRequester(dataDir))
cfg.TimersDrained = w.holds.drainTimers
+ if cfg.claimRecords {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ if err := armMovesParks(ctx, &cfg, providers, movesPlan12Parks); err != nil {
+ log.Printf("agent: moves: the park arms stay unarmed: %v", err)
+ }
+ cancel()
+ }
}
w.cfg = cfg
armedMovesWorker.Store(w)
@@ -293,11 +303,12 @@ func armPhase1(cfg *moveCfg, w *movesWorker, hib activator.Config, dataDir strin
cfg.HoldMoving = holdMoving(reg, holdStageRequester(dataDir))
cfg.Timers = reg.TimersFor
}
- // The landing (movesland.go). Everything that exists on main is wired;
- // ParkLeases, ParkView and ParkAuthorize are not (ruling G17: the
- // residue parks need claim Plan 6b's regional runner for their
- // acknowledgement and Plan 5's view and authorizer), so landingArmed is
- // false in production and the landing pass only logs. The held timers
+ // The landing (movesland.go). Everything that exists on main is wired
+ // here; ParkLeases, ParkView and ParkAuthorize come from the claim's
+ // MoveParks (armMovesParks, Plan 5 Task 9a, in armMovesPass), with claim
+ // Plan 12's RunHeld and local authorizer, and are set only once claim
+ // Plan 12 lands, so landingArmed is false in production and the landing
+ // pass only logs. The held timers
// write goes through schedule.SelectedTimers with a held-timers Work
// selected from writes (ruling G16, heldTimersReplacer): with
// CARLOS_HOST_CLAIM off that is the registry's own ReplaceTimers, as
@@ -308,8 +319,8 @@ func armPhase1(cfg *moveCfg, w *movesWorker, hib activator.Config, dataDir strin
cfg.Guard = hostguard.Guard{DataDir: dataDir}
cfg.Local = park.Local{DataDir: dataDir, Box: cfg.Box}
cfg.OwnLease = hostbox.RenewingLeases{Region: hib.Region}.Acquire
- cfg.RenderHeld = func(ctx context.Context, s release.Store, rt registry.Route, op string) (appconfig.HeldRender, error) {
- return appconfig.RenderHeld(ctx, s, cfg.SecretsPriv, dataDir, rt, op, cfg.Box)
+ cfg.RenderHeld = func(ctx context.Context, s release.Store, rt registry.Route, op string, before func(string) error) (appconfig.HeldRender, error) {
+ return appconfig.RenderHeldBefore(ctx, s, cfg.SecretsPriv, dataDir, rt, op, cfg.Box, before)
}
cfg.RemoveVerifyEntry = dbverify.RemoveHost
cfg.WriteGen = hostbox.WriteGenerationMarker
@@ -550,31 +561,41 @@ type moveCfg struct {
// over this box's data directory (its Claims are set per pass from the
// pass's store when nil); Local is park.Local over the same directory
// (its Guard, Claims and, when nil, Authorize are set per pass).
- // ParkLeases, ParkView and ParkAuthorize are the park engine's over the
- // own bucket for the residue parks: nil in production until claim Plan
- // 6b's regional runner and Plan 5's providers (ruling G17), which is
- // what leaves the landing unarmed there (landingArmed); a nil View or
- // Authorize with ParkLeases set gets the landing's stand-ins. OwnLease
+ // ParkLeases, ParkView and ParkAuthorize are the claim's park engine's
+ // (MoveParks, armMovesParks): its leases, view and authorizer over the
+ // own bucket. Every arm requires all three (parkHooksArmed) and runs its
+ // own check before the authorizer (movesArmAuthorize); nothing stands
+ // in. In production they are set only once claim Plan 12 lands, which
+ // leaves every arm unarmed there. OwnLease
// takes the location's lease in the own bucket for the build
// (hostbox.RenewingLeases over lease.Acquire). RenderHeld is
- // appconfig.RenderHeld for the held row; ReplaceTimers is the
+ // appconfig.RenderHeldBefore for the held row (before journals the
+ // render's ownership ahead of its write, carry (c)); ReplaceTimers is the
// registry's through schedule.SelectedTimers with a held-timers Work
// (heldTimersReplacer); SetReplication and SetNextDue are the
// registry's; RemoveVerifyEntry is
// dbverify.RemoveHost; WriteGen is hostbox.WriteGenerationMarker, the
// one <dbPath>.gen writer (ruling G2).
- Guard hostguard.Guard
- Local park.Local
- ParkLeases park.Leases
- ParkView func(context.Context, hostclaim.Observed) (park.View, error)
- ParkAuthorize func(context.Context, hostclaim.Observed, hostclaim.Obligation) error
- OwnLease func(ctx context.Context, bucket, location string) (park.HeldLease, error)
- RenderHeld func(ctx context.Context, s release.Store, rt registry.Route, op string) (appconfig.HeldRender, error)
- ReplaceTimers func(ctx context.Context, host string, rows []registry.Timer) error
- SetReplication func(host, class string) error
- SetNextDue func(host string, due int64) error
- RemoveVerifyEntry func(ctx context.Context, s release.Store, host string) error
- WriteGen func(path, generation string) error
+ Guard hostguard.Guard
+ Local park.Local
+ ParkLeases park.Leases
+ ParkView func(context.Context, hostclaim.Observed) (park.View, error)
+ ParkAuthorize func(context.Context, hostclaim.Observed, hostclaim.Obligation) error
+ // ParkRunHeld and ParkLocalAuthorize are claim Plan 12's pieces of the
+ // MoveParks capability (movesClaimPlan12, movessources.go): RunHeld runs
+ // a chore under a lease the caller holds (M5: the landing's residue park
+ // under its own-location lease), AuthorizeLocal is the owner authorizer
+ // for this box's park.Local chores (M9). Nil in production until claim
+ // Plan 12 lands; armMovesParks sets all five or none.
+ ParkRunHeld func(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error
+ ParkLocalAuthorize func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error
+ OwnLease func(ctx context.Context, bucket, location string) (park.HeldLease, error)
+ RenderHeld func(ctx context.Context, s release.Store, rt registry.Route, op string, before func(sha256hex string) error) (appconfig.HeldRender, error)
+ ReplaceTimers func(ctx context.Context, host string, rows []registry.Timer) error
+ SetReplication func(host, class string) error
+ SetNextDue func(host string, due int64) error
+ RemoveVerifyEntry func(ctx context.Context, s release.Store, host string) error
+ WriteGen func(path, generation string) error
// Probe is the landing's probe wake (Plan 3 Task 6): Task 5's
// activator.Service.ProbeWake. Nil in production, with a standing
// notice, until claim Plan 11's launcher (R10) and Plan 5 Task 9's
diff --git a/internal/host/movespass_test.go b/internal/host/movespass_test.go
index 7a4161b49..e8b6ab3ac 100644
--- a/internal/host/movespass_test.go
+++ b/internal/host/movespass_test.go
@@ -926,7 +926,7 @@ func TestArmMovesPassReadsTheReadyDelta(t *testing.T) {
{"lots", defaultMoveReadyDelta},
} {
t.Setenv("CARLOS_MOVE_READY_DELTA_BYTES", tc.raw)
- armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{})
+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{}, ClaimProviders{})
if got := armedMovesWorker.Load().cfg.ReadyDelta; got != tc.want {
t.Errorf("CARLOS_MOVE_READY_DELTA_BYTES=%q armed a bar of %d, want %d", tc.raw, got, tc.want)
}
@@ -945,7 +945,7 @@ func TestArmMovesPassReadsTheSwitch(t *testing.T) {
{"", false}, {"on", true}, {"1", true}, {"true", true}, {"off", false}, {"yes", false},
} {
t.Setenv(moves.SwitchEnv, tc.raw)
- armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{})
+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, nil, hostwrite.Config{}, ClaimProviders{})
if got := armedMovesWorker.Load().cfg.Enabled; got != tc.want {
t.Errorf("%s=%q armed Enabled=%v, want %v", moves.SwitchEnv, tc.raw, got, tc.want)
}
diff --git a/internal/host/movesphase1_test.go b/internal/host/movesphase1_test.go
index a6efc9a4c..c1b301a55 100644
--- a/internal/host/movesphase1_test.go
+++ b/internal/host/movesphase1_test.go
@@ -144,7 +144,7 @@ func TestPhase1IsInertWithTheSwitchOff(t *testing.T) {
}
t.Setenv(moves.SwitchEnv, v)
eb := &activator.ExecBackend{}
- armMovesPass(activator.Config{}, t.TempDir(), nil, nil, eb, hostwrite.Config{})
+ armMovesPass(activator.Config{}, t.TempDir(), nil, nil, eb, hostwrite.Config{}, ClaimProviders{})
if (eb.Strict != nil) != on {
t.Fatalf("switch %v: eb.Strict set = %v", on, eb.Strict != nil)
}
diff --git a/internal/host/movesrecover.go b/internal/host/movesrecover.go
index a1fd78401..af5e5757c 100644
--- a/internal/host/movesrecover.go
+++ b/internal/host/movesrecover.go
@@ -91,11 +91,12 @@ import (
// Every record write goes through a provision transition (the record_seq
// rule); every claim write through a hostclaim transition.
//
-// Arming (rulings J18, G17). Like the landing and the target's half of the
-// rollback, the source's recovery is unarmed in production until Plan 5:
-// it needs the park engine's leases (ParkLeases, for the dirty return's
-// residue parks), which production does not wire, so recoverySourceArmed
-// answers false there and the pass logs a standing notice. The probe hook is
+// Arming (rulings J18, G17; Plan 5 Task 9a). Like the landing and the
+// target's half of the rollback, the source's recovery needs the claim's
+// park hooks and claim Plan 12's RunHeld (for the dirty return's residue
+// parks), which armMovesParks sets only once claim Plan 12 lands, so
+// recoverySourceArmed answers false in production and the pass logs a
+// standing notice. The probe hook is
// the recovery probe adapter (armRecoveryProbe) with CARLOS_HOST_CLAIM on,
// bound to the activator's Group D entry. Without the claim switch the
// legacy nil hook still waits, recording nothing. LogEvent
@@ -154,8 +155,9 @@ type recoveryProbeRequest struct {
// recoverySourceArmed reports whether this box can run the source's
// recovery: every hook a dirty return's install and the open call is wired,
-// the park engine's leases included. In production ParkLeases is not
-// (ruling G17), so this answers false there (J18).
+// the claim's park hooks and claim Plan 12's RunHeld included (landingArmed).
+// In production armMovesParks sets those only with claim Plan 12, so this
+// answers false there until it lands (J18).
func recoverySourceArmed(cfg moveCfg) bool {
return landingArmed(cfg) && cfg.DataDir != "" && cfg.HandHold != nil && cfg.ReleaseHeld != nil && cfg.Timers != nil
}
@@ -192,7 +194,7 @@ func (w *movesWorker) recoverSource(ctx context.Context, s release.Store, rec pr
return nil
}
if !recoverySourceArmed(cfg) {
- w.notice(rec.Host+"#recover", "moves: %s: the source's recovery of rollback %s waits: this box's recovery is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.Rollback.ID)
+ w.notice(rec.Host+"#recover", "moves: %s: the source's recovery of rollback %s waits: this box's recovery is not armed (the claim's park hooks, set from MoveParks once claim Plan 12 lands, are needed)", rec.Host, m.Rollback.ID)
return nil
}
r := &recoverRun{w: w, s: s, rec: rec, g: provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation}, rb: m.Rollback.ID}
diff --git a/internal/host/movesrecover_test.go b/internal/host/movesrecover_test.go
index f0174bb47..e83220e4a 100644
--- a/internal/host/movesrecover_test.go
+++ b/internal/host/movesrecover_test.go
@@ -116,8 +116,12 @@ func (h *recoverHarness) sourceCfg(t *testing.T) moveCfg {
}
c.Guard = hostguard.Guard{DataDir: h.sdir, LockDir: h.lockDir}
c.Local = park.Local{DataDir: h.sdir, Box: "ie-box"}
- c.RenderHeld = func(_ context.Context, _ release.Store, rt registry.Route, op string) (appconfig.HeldRender, error) {
+ c.ParkView = recordView(h.s, "ie", roleSource)
+ c.RenderHeld = func(_ context.Context, _ release.Store, rt registry.Route, op string, before func(string) error) (appconfig.HeldRender, error) {
h.srcRends++
+ if err := before(h.srcRender.SHA256); err != nil {
+ return appconfig.HeldRender{}, err
+ }
if rb := h.read(t, p1Host).Move.Rollback; rb == nil || op != rb.ID || rt.HoldOp != rb.ID {
t.Errorf("rendered for %q over a row held by %q, want the rollback", op, rt.HoldOp)
}
diff --git a/internal/host/movesrefusals.go b/internal/host/movesrefusals.go
new file mode 100644
index 000000000..ba327236d
--- /dev/null
+++ b/internal/host/movesrefusals.go
@@ -0,0 +1,225 @@
+package host
+
+// Quiet refusals (moves Plan 5 Task 9a; the Task 5 rulings). The claim's
+// regional worker asks Task 5's provider about every host at rest, every
+// minute, and an ordinary state refuses: a live move, residue the move's
+// close does not vouch for, a free host whose lanes are claim Plan 12's. The
+// worker reports each refusal, so the agent's log would repeat them every
+// minute. movesRefusals keeps them quiet:
+//
+// - report is the worker's report: a provider refusal (MovesSweepRefusal,
+// NotMovesEvidence) or a named dependency (ClaimDependencyError) is
+// logged at most once an hour per host and reason; anything else is
+// logged every time, as before;
+// - evidence wraps the provider: a refusal marked Anomaly (the history
+// contradicts itself, or something wrote where no move could) pages once
+// per host and reason through the box pager, as a warning: marked sent
+// only once delivered, never twice at once, a failure retried with
+// backoff a bounded number of times. The worker gets every error
+// unchanged.
+//
+// Both are wired only with CARLOS_MOVES_PHASE1 and CARLOS_HOST_CLAIM on;
+// otherwise the worker's report is the plain log line it always was.
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/alerting"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/park"
+)
+
+// movesRefusalLogEvery is how often one expected refusal is logged.
+const movesRefusalLogEvery = time.Hour
+
+// movesPageAttempts bounds the tries one anomaly page gets.
+const movesPageAttempts = 5
+
+// movesRefusals is the agent's refusal log and anomaly pager for the moves
+// evidence. Its memory is the process's: a restarted agent logs and pages
+// each standing condition once more.
+type movesRefusals struct {
+ box string
+ page func(ctx context.Context, level, subject, body string) error
+ logf func(format string, args ...any)
+ now func() time.Time
+
+ mu sync.Mutex
+ logged map[string]time.Time
+ pages map[string]*movesPageState
+}
+
+// movesPageState is one anomaly's page: delivered once a send succeeds;
+// while a send is in flight no other starts; a failed send is retried by a
+// later observation after a doubling backoff, at most movesPageAttempts
+// times in all.
+type movesPageState struct {
+ delivered, inflight bool
+ failures int
+ next time.Time
+}
+
+// movesPageBackoff is the wait after the first failed page; it doubles with
+// each further failure, up to movesPageBackoffMax.
+const (
+ movesPageBackoff = time.Minute
+ movesPageBackoffMax = 30 * time.Minute
+)
+
+func newMovesRefusals(box string, page func(context.Context, string, string, string) error, logf func(string, ...any), now func() time.Time) *movesRefusals {
+ return &movesRefusals{box: box, page: page, logf: logf, now: now, logged: map[string]time.Time{}, pages: map[string]*movesPageState{}}
+}
+
+// movesQuietEvidence is Task 5's provider with its anomalies paged. It is
+// the regional evidence and the sweep verifier the claim registers.
+type movesQuietEvidence struct {
+ movesRetirementEvidence
+ r *movesRefusals
+}
+
+var (
+ _ hostbox.RegionalEvidence = movesQuietEvidence{}
+ _ park.MoveSweepVerifier = movesQuietEvidence{}
+)
+
+func (r *movesRefusals) evidence(e movesRetirementEvidence) movesQuietEvidence {
+ return movesQuietEvidence{movesRetirementEvidence: e, r: r}
+}
+
+func (e movesQuietEvidence) VerifyRetirement(ctx context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (string, time.Time, error) {
+ inc, at, err := e.movesRetirementEvidence.VerifyRetirement(ctx, o, versions)
+ if err != nil {
+ e.r.observe(ctx, err)
+ }
+ return inc, at, err
+}
+
+func (e movesQuietEvidence) VerifyMovesSweep(ctx context.Context, o hostclaim.Observed, versions []lease.ObjectVersion) (park.MoveSweepProof, error) {
+ proof, err := e.movesRetirementEvidence.VerifyMovesSweep(ctx, o, versions)
+ if err != nil {
+ e.r.observe(ctx, err)
+ }
+ return proof, err
+}
+
+// observe pages an anomaly once per host and reason (fix round 1, I4): it
+// is marked delivered only after the pager succeeds; a concurrent
+// observation while a page is in flight sends nothing; a failed page is
+// retried by a later observation after its backoff, at most
+// movesPageAttempts times.
+func (r *movesRefusals) observe(ctx context.Context, err error) {
+ var refusal *MovesSweepRefusal
+ if r == nil || r.page == nil || !errors.As(err, &refusal) || !refusal.Anomaly {
+ return
+ }
+ key := refusal.Host + "\x00" + refusal.Reason
+ now := r.now()
+ r.mu.Lock()
+ st := r.pages[key]
+ if st == nil {
+ st = &movesPageState{}
+ r.pages[key] = st
+ }
+ if st.delivered || st.inflight || st.failures >= movesPageAttempts || now.Before(st.next) {
+ r.mu.Unlock()
+ return
+ }
+ st.inflight = true
+ r.mu.Unlock()
+ subject, body := movesAnomalyPage(refusal.Host, r.box)
+ perr := r.page(ctx, alerting.LevelWarning, subject, body)
+ r.mu.Lock()
+ st.inflight = false
+ if perr == nil {
+ st.delivered = true
+ r.mu.Unlock()
+ return
+ }
+ st.failures++
+ wait := movesPageBackoff << (st.failures - 1)
+ if wait > movesPageBackoffMax || wait <= 0 {
+ wait = movesPageBackoffMax
+ }
+ st.next = now.Add(wait)
+ failures := st.failures
+ r.mu.Unlock()
+ if r.logf != nil {
+ if failures >= movesPageAttempts {
+ r.logf("moves: %s: the page for the refused cleanup failed %d times; no more are sent while this agent runs: %v", refusal.Host, failures, perr)
+ } else {
+ r.logf("moves: %s: the page for the refused cleanup failed (try %d of %d, again after %s): %v", refusal.Host, failures, movesPageAttempts, wait, perr)
+ }
+ }
+}
+
+// movesAnomalyPage is the anomaly's page: the subject (MV-N336) and the
+// body (MV-N337). The reason stays in the agent's log, never in the page.
+func movesAnomalyPage(host, box string) (subject, body string) {
+ subject = fmt.Sprintf("carlos move cleanup refused: %s", host)
+ body = fmt.Sprintf("The cleanup of %s on %s stopped: its move history does not add up, or something wrote where no move could. Nothing of %s is cleaned up in this region until an operator checks the agent's log on %s.",
+ host, box, host, box)
+ return subject, body
+}
+
+// report is the regional worker's report: each error of the pass, the
+// expected refusals at most once an hour per host and reason.
+func (r *movesRefusals) report(err error) {
+ for _, leaf := range joinedErrors(err) {
+ key, expected := movesExpectedRefusal(leaf)
+ if !expected {
+ r.logf("agent: regional claims: %v", leaf)
+ continue
+ }
+ now := r.now()
+ r.mu.Lock()
+ last, seen := r.logged[key]
+ if !seen || !now.Before(last.Add(movesRefusalLogEvery)) {
+ r.logged[key] = now
+ seen = false
+ }
+ r.mu.Unlock()
+ if !seen {
+ r.logf("agent: regional claims: %v (logged at most once an hour while it stands)", leaf)
+ }
+ }
+}
+
+// joinedErrors flattens errors.Join's tree into its leaves.
+func joinedErrors(err error) []error {
+ if err == nil {
+ return nil
+ }
+ j, ok := err.(interface{ Unwrap() []error })
+ if !ok {
+ return []error{err}
+ }
+ var out []error
+ for _, e := range j.Unwrap() {
+ out = append(out, joinedErrors(e)...)
+ }
+ return out
+}
+
+// movesExpectedRefusal names a refusal that stands until its condition
+// changes: the provider's typed refusals and a named dependency.
+func movesExpectedRefusal(err error) (string, bool) {
+ var refusal *MovesSweepRefusal
+ var notMoves *NotMovesEvidence
+ var dep *ClaimDependencyError
+ switch {
+ case errors.As(err, &refusal):
+ return "sweep\x00" + refusal.Host + "\x00" + refusal.Reason, true
+ case errors.As(err, &notMoves):
+ return "not-moves\x00" + notMoves.Host + "\x00" + notMoves.Generation + "\x00" + notMoves.Reason, true
+ case errors.As(err, &dep):
+ return "dependency\x00" + strings.Join(dep.Contracts, "\x00"), true
+ }
+ return "", false
+}
diff --git a/internal/host/movesrefusals_test.go b/internal/host/movesrefusals_test.go
new file mode 100644
index 000000000..14ff1b521
--- /dev/null
+++ b/internal/host/movesrefusals_test.go
@@ -0,0 +1,226 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/alerting"
+ "github.com/carlosframework/platform/internal/park"
+)
+
+// Quiet refusals (Plan 5 Task 9a): the Task 5 provider's expected refusals
+// are rate-limited in the regional worker's log, and an anomaly pages once
+// through the box pager.
+
+type refusalLog struct {
+ mu sync.Mutex
+ lines []string
+ now time.Time
+}
+
+func (l *refusalLog) logf(format string, args ...any) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.lines = append(l.lines, fmt.Sprintf(format, args...))
+}
+
+func (l *refusalLog) clock() time.Time {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.now
+}
+
+func (l *refusalLog) take() []string {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ out := l.lines
+ l.lines = nil
+ return out
+}
+
+// TestExpectedMovesRefusalsAreRateLimited: a live move, residue the close
+// does not vouch for and a free host (the provider's dependency refusal) are
+// each logged once, then not again within the hour; an unexpected error is
+// logged every time.
+func TestExpectedMovesRefusalsAreRateLimited(t *testing.T) {
+ l := &refusalLog{now: evNow}
+ var pages pagesSeen
+ r := newMovesRefusals("ie-box", pages.hook(), l.logf, l.clock)
+ boom := errors.New("listing failed")
+ pass := errors.Join(
+ &MovesSweepRefusal{Host: "a.example", Reason: "the record carries the live move mv_x"},
+ &NotMovesEvidence{Host: "b.example", Generation: "g", Reason: "written after the close"},
+ &ClaimDependencyError{Contracts: []string{movesLegacyCensus}},
+ boom,
+ )
+ r.report(pass)
+ if got := l.take(); len(got) != 4 {
+ t.Fatalf("the first pass logged %q", got)
+ }
+ l.now = evNow.Add(30 * time.Minute)
+ r.report(pass)
+ if got := l.take(); len(got) != 1 || !strings.Contains(got[0], boom.Error()) {
+ t.Fatalf("within the hour the pass logged %q; want only the unexpected error", got)
+ }
+ l.now = evNow.Add(61 * time.Minute)
+ r.report(pass)
+ if got := l.take(); len(got) != 4 {
+ t.Fatalf("after the hour the pass logged %q", got)
+ }
+ if len(pages) != 0 {
+ t.Fatalf("an expected refusal paged: %+v", pages)
+ }
+}
+
+// TestMovesAnomalyPagesOnce: an anomaly from either method pages once per
+// host and reason, as a warning naming the host and the box and carrying no
+// path; an expected refusal pages nothing, and the error reaches the worker
+// unchanged.
+func TestMovesAnomalyPagesOnce(t *testing.T) {
+ l := &refusalLog{now: evNow}
+ var pages pagesSeen
+ r := newMovesRefusals("ie-box", pages.hook(), l.logf, l.clock)
+ f := newEvFixture(t)
+ f.rec.CreatedAt = evCreated.Add(time.Hour) // a record of another incarnation: an anomaly
+ e := r.evidence(f.provider())
+ obs, census := f.observe(t), f.census(t)
+ for i := 0; i < 3; i++ {
+ _, _, err := e.VerifyRetirement(t.Context(), obs, census)
+ var refusal *MovesSweepRefusal
+ if !errors.As(err, &refusal) || !refusal.Anomaly {
+ t.Fatalf("the wrapper changed the refusal: %v", err)
+ }
+ if _, err := e.VerifyMovesSweep(t.Context(), obs, census); !errors.As(err, &refusal) {
+ t.Fatalf("the wrapper changed the sweep refusal: %v", err)
+ }
+ }
+ if len(pages) != 1 {
+ t.Fatalf("pages %+v, want one", pages)
+ }
+ subject, body := movesAnomalyPage(evHost, "ie-box")
+ if p := pages[0]; p.level != alerting.LevelWarning || p.subject != subject || p.body != body || strings.Contains(p.subject+p.body, "/") {
+ t.Fatalf("page %+v", p)
+ }
+ // An expected refusal of another host pages nothing.
+ g := newEvFixture(t)
+ g.rec.LastMove = nil
+ if _, _, err := r.evidence(g.provider()).VerifyRetirement(t.Context(), g.observe(t), g.census(t)); err == nil {
+ t.Fatal("no refusal")
+ }
+ if len(pages) != 1 {
+ t.Fatalf("an expected refusal paged: %+v", pages)
+ }
+}
+
+// TestRegionalWorkerRefusalsAreQuiet: through the claim's real regional
+// worker, an anomalous host refuses the sweep every pass, pages once and is
+// logged once within the hour.
+func TestRegionalWorkerRefusalsAreQuiet(t *testing.T) {
+ l := &refusalLog{now: evNow}
+ var pages pagesSeen
+ r := newMovesRefusals("ie-box", pages.hook(), l.logf, l.clock)
+ f := newEvFixture(t)
+ f.rec.CreatedAt = evCreated.Add(time.Hour)
+ f.lane(evMoveID, "move.json") // the census finds the host by its objects
+ w := f.worker(t, evNow)
+ e := r.evidence(f.provider())
+ w.Evidence = e
+ var verifier park.MoveSweepVerifier = e
+ w.Regional.Engine.VerifyMovesSweep = verifier.VerifyMovesSweep
+ for i := 0; i < 3; i++ {
+ err := w.Run(t.Context())
+ if err == nil {
+ t.Fatal("the sweep ran on an anomaly")
+ }
+ r.report(err)
+ }
+ if lines := l.take(); len(pages) != 1 || len(lines) != 1 {
+ t.Fatalf("pages %+v, log lines %q over three passes", pages, lines)
+ }
+}
+
+// anomalyErr is one anomalous refusal of evHost.
+var anomalyErr = &MovesSweepRefusal{Host: evHost, Reason: "the record is not the holder's incarnation", Anomaly: true}
+
+// TestMovesAnomalyPageRetriesUntilDelivered (fix round 1, I4): a page that
+// fails is not marked delivered; a later observation, past the backoff,
+// pages again; once one succeeds nothing more is sent. A pager that always
+// fails is tried a bounded number of times.
+func TestMovesAnomalyPageRetriesUntilDelivered(t *testing.T) {
+ l := &refusalLog{now: evNow}
+ calls, fail := 0, true
+ r := newMovesRefusals("ie-box", func(context.Context, string, string, string) error {
+ calls++
+ if fail {
+ return errors.New("the webhook answered 503")
+ }
+ return nil
+ }, l.logf, l.clock)
+ r.observe(t.Context(), anomalyErr)
+ if calls != 1 {
+ t.Fatalf("first observation: %d calls", calls)
+ }
+ l.now = evNow.Add(30 * time.Second)
+ r.observe(t.Context(), anomalyErr)
+ if calls != 1 {
+ t.Fatalf("inside the backoff: %d calls", calls)
+ }
+ fail = false
+ l.now = evNow.Add(2 * time.Minute)
+ r.observe(t.Context(), anomalyErr)
+ if calls != 2 {
+ t.Fatalf("after the backoff: %d calls", calls)
+ }
+ for i := 0; i < 5; i++ {
+ l.now = l.now.Add(time.Hour)
+ r.observe(t.Context(), anomalyErr)
+ }
+ if calls != 2 {
+ t.Fatalf("after a delivered page: %d calls", calls)
+ }
+
+ l2 := &refusalLog{now: evNow}
+ always := 0
+ r2 := newMovesRefusals("ie-box", func(context.Context, string, string, string) error { always++; return errors.New("down") }, l2.logf, l2.clock)
+ for i := 0; i < 50; i++ {
+ r2.observe(t.Context(), anomalyErr)
+ l2.now = l2.now.Add(2 * time.Hour)
+ }
+ if always != movesPageAttempts {
+ t.Fatalf("a pager that always fails was called %d times, want %d", always, movesPageAttempts)
+ }
+}
+
+// TestMovesAnomalyPageIsNotDoubled (fix round 1, I4): while one page is in
+// flight, a concurrent observation of the same anomaly does not page again.
+func TestMovesAnomalyPageIsNotDoubled(t *testing.T) {
+ l := &refusalLog{now: evNow}
+ var mu sync.Mutex
+ calls := 0
+ entered, release := make(chan struct{}), make(chan struct{})
+ r := newMovesRefusals("ie-box", func(context.Context, string, string, string) error {
+ mu.Lock()
+ calls++
+ mu.Unlock()
+ close(entered)
+ <-release
+ return nil
+ }, l.logf, l.clock)
+ done := make(chan struct{})
+ go func() { r.observe(context.Background(), anomalyErr); close(done) }()
+ <-entered
+ r.observe(t.Context(), anomalyErr) // in flight: skipped, does not block
+ close(release)
+ <-done
+ r.observe(t.Context(), anomalyErr)
+ mu.Lock()
+ defer mu.Unlock()
+ if calls != 1 {
+ t.Fatalf("%d pages, want one", calls)
+ }
+}
diff --git a/internal/host/movesrevert.go b/internal/host/movesrevert.go
index dfa54e91d..6960da54d 100644
--- a/internal/host/movesrevert.go
+++ b/internal/host/movesrevert.go
@@ -94,8 +94,8 @@ import (
// unarmed in production until Plan 5: the recovery probe is nil there until
// Task 9 (claim Plan 10's recovery admission), so revertSourceArmed answers
// false and the source only writes the claim half, as Plan 2's arm did; and
-// the target's lane park needs the park engine's leases (ParkLeases), which
-// production does not wire (revertLaneArmed). LogEvent is nil in production,
+// the target's lane park needs the claim's park leases, view and authorizer,
+// which armMovesParks sets only once claim Plan 12 lands (revertLaneArmed). LogEvent is nil in production,
// where an event is a log line.
// revertStepsPerPass bounds the steps one pass runs: evidence to the close is
@@ -158,10 +158,11 @@ func revertSourceArmed(cfg moveCfg) bool {
}
// revertLaneArmed reports whether this box can run the target region's lane
-// park: the park engine's leases and this box's own bucket. In production
-// ParkLeases is not wired (ruling G17), so this answers false there.
+// park: the claim's park leases, view and authorizer, and this box's own
+// bucket. In production armMovesParks sets the three only with claim Plan
+// 12, so this answers false there until it lands.
func revertLaneArmed(cfg moveCfg) bool {
- return cfg.Box != "" && cfg.Region != "" && cfg.OwnBucket != "" && cfg.ParkLeases != nil && cfg.Versioned != nil
+ return cfg.Box != "" && cfg.Region != "" && cfg.OwnBucket != "" && parkHooksArmed(cfg) && cfg.Versioned != nil
}
// restatusHold is moveCfg.RestatusHold in production: an op's own hold
@@ -860,11 +861,10 @@ func (w *movesWorker) sweepRevertJournal(dataDir, host, op string, j *moves.Jour
// hostclaim.Abort queued, Plan 5 Task 1) and a committed landing
// (landing/committed; the park hostclaim.Commit queued, Plan 5 Task 3, §7.9
// step 4, called from the committed arm). No phases means any phase of
-// kind. The view and the authorizer are Plan 5's when wired, else the
-// stand-ins: the source's region authoritative (see the view below), and
-// only this move's lane park under kind and its phases, with this box not
-// executing the host unless the landing is committed (the target serves it
-// then).
+// kind. The view and the authorizer are the claim's (MoveParks), the
+// authorizer behind this lane's own check: only this move's lane park under
+// kind and its phases, with this box not executing the host unless the
+// landing is committed (the target serves it then).
func (w *movesWorker) laneParkTarget(ctx context.Context, s release.Store, rec provision.Record, kind string, phases ...string) error {
cfg := w.cfg
m := rec.Move
@@ -876,7 +876,7 @@ func (w *movesWorker) laneParkTarget(ctx context.Context, s release.Store, rec p
notice = rec.Host + "#rv-lane"
}
if !revertLaneArmed(cfg) {
- w.notice(notice, "moves: %s: the park of move %s's lane waits: this box's park engine is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.ID)
+ w.notice(notice, "moves: %s: the park of move %s's lane waits: this box's park engine is not armed (the claim's park hooks, set from MoveParks once claim Plan 12 lands, are needed)", rec.Host, m.ID)
return nil
}
host, id := rec.Host, m.ID
@@ -908,28 +908,19 @@ func (w *movesWorker) laneParkTarget(ctx context.Context, s release.Store, rec p
return err
}
committed := kind == hostclaim.KindLanding
- view, auth := cfg.ParkView, cfg.ParkAuthorize
- if view == nil {
- // The stand-in names From authoritative, after a commit too. For a
- // lane, which is never the holder's location, the engine reads the
- // authoritative region only to decide whether it must protect the
- // location's live generations (park.protected), and a stand-in has
- // no marker to protect them with. Plan 5's view (Task 9) answers
- // the target's region after the commit, with its marker.
- view = standInView(cfg.Region, roleTarget, m)
- }
- if auth == nil {
- auth = func(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
- c := obs.Claim
- if obs.Absent || c == nil || !inOp(c.Op) || (!committed && c.Op.Exec == cfg.Box) ||
- o.Op != id || o.Kind != "park-replica" || o.Region != cfg.Region || o.Park == nil || o.Park.Location != lane {
- return fmt.Errorf("%w: not this move's lane park", park.ErrUnsafe)
- }
- return nil
+ // The claim's view (movesParkView) names the target's region
+ // authoritative after the commit, with its marker, and the source's
+ // before it; the claim's authorizer runs behind this lane's own check.
+ own := func(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
+ c := obs.Claim
+ if obs.Absent || c == nil || !inOp(c.Op) || (!committed && c.Op.Exec == cfg.Box) ||
+ o.Op != id || o.Kind != "park-replica" || o.Region != cfg.Region || o.Park == nil || o.Park.Location != lane {
+ return fmt.Errorf("%w: not this move's lane park", park.ErrUnsafe)
}
+ return nil
}
e := park.Engine{Store: vs, Claims: s, Bucket: cfg.OwnBucket, Region: cfg.Region, Leases: cfg.ParkLeases,
- Now: w.now, NewID: obligationID, View: view, Authorize: auth, After: cfg.parkFault}
+ Now: w.now, NewID: obligationID, View: cfg.ParkView, Authorize: movesArmAuthorize(cfg.ParkAuthorize, own), After: cfg.parkFault}
for _, o := range chores {
err := e.Run(ctx, park.Work{Host: host, ID: o.ID, Op: id})
switch {
diff --git a/internal/host/movesrevert_test.go b/internal/host/movesrevert_test.go
index dd8dfd456..a35434fdb 100644
--- a/internal/host/movesrevert_test.go
+++ b/internal/host/movesrevert_test.go
@@ -162,7 +162,7 @@ func revertRendered() provision.MoveInputs {
func (h *revertHarness) laneCfg() moveCfg {
c := h.target()
c.DataDir = h.tdir
- c.ParkLeases = h.tleases
+ armParkFakes(&c, h.s, h.tleases, c.Region, roleTarget)
return c
}
diff --git a/internal/host/movesrollback.go b/internal/host/movesrollback.go
index d52afdedc..a3e2a4242 100644
--- a/internal/host/movesrollback.go
+++ b/internal/host/movesrollback.go
@@ -16,10 +16,12 @@ import (
"github.com/carlosframework/platform/internal/alerting"
"github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/hostbox"
"github.com/carlosframework/platform/internal/hostclaim"
"github.com/carlosframework/platform/internal/hostguard"
"github.com/carlosframework/platform/internal/lease"
"github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/moves/archive"
"github.com/carlosframework/platform/internal/park"
"github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/release"
@@ -88,10 +90,10 @@ import (
// transaction under the host's hold lock (J12); the record says cleaned
// with target_parked_at, and the op's journal is parked last.
//
-// Arming (ruling J18). Like the landing, the rollback's target half is
-// unarmed in production until Plan 5: it needs the park engine's leases
-// (ParkLeases), which production does not wire, so rollbackTargetArmed
-// answers false there and the pass logs a standing notice. The decision is
+// Arming (ruling J18; Plan 5 Task 9a). Like the landing, the rollback's
+// target half needs the claim's park leases, view and authorizer, which
+// armMovesParks sets only once claim Plan 12 lands, so rollbackTargetArmed
+// answers false in production and the pass logs a standing notice. The decision is
// not gated (J10). The move's event log (moves.LogEvent, Plan 6) is the
// LogEvent hook: nil in production, where the event is a log line. The
// stuck mark pages an operator (pageStuck, the Page hook); the release
@@ -244,11 +246,11 @@ func (w *movesWorker) rollbackClaim(ctx context.Context, s release.Store, rec pr
}
// rollbackTargetArmed reports whether this box can run the target's half of
-// a rollback: every hook it calls is wired, the park engine's leases
-// included. In production ParkLeases is not (ruling G17), so this answers
-// false there (J18).
+// a rollback: every hook it calls is wired, the claim's three park hooks
+// included. In production armMovesParks sets them only once claim Plan 12
+// lands, so this answers false there (J18).
func rollbackTargetArmed(cfg moveCfg) bool {
- return cfg.Box != "" && cfg.DataDir != "" && cfg.Guard.DataDir != "" && cfg.Local.DataDir != "" && cfg.ParkLeases != nil &&
+ return cfg.Box != "" && cfg.DataDir != "" && cfg.Guard.DataDir != "" && cfg.Local.DataDir != "" && parkHooksArmed(cfg) &&
cfg.HandHold != nil && cfg.Fence != nil && cfg.TimersDrained != nil && cfg.Evidence != nil && cfg.FinalCheckpoint != nil &&
cfg.LocalPosition != nil && cfg.Timers != nil && cfg.FilesDir != nil && cfg.Route != nil && cfg.Versioned != nil &&
cfg.StagingLease != nil && cfg.Staging != nil && cfg.DeleteHeld != nil && cfg.SecretsPub != nil
@@ -278,7 +280,7 @@ func (w *movesWorker) rollbackTarget(ctx context.Context, s release.Store, rec p
// nothing held is nothing to release.
releaseOwnLease(w, rec.Host)
if !rollbackTargetArmed(cfg) {
- w.notice(rec.Host+"#rollback", "moves: %s: the target's half of rollback %s waits: this box's rollback is not armed (Plan 5's park view and the park engine's leases are needed)", rec.Host, m.Rollback.ID)
+ w.notice(rec.Host+"#rollback", "moves: %s: the target's half of rollback %s waits: this box's rollback is not armed (the claim's park hooks, set from MoveParks once claim Plan 12 lands, are needed)", rec.Host, m.Rollback.ID)
return nil
}
r := &rollbackRun{w: w, s: s, rec: rec, g: provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation}, rb: m.Rollback.ID}
@@ -512,6 +514,13 @@ func (r *rollbackRun) releasing(ctx context.Context) (bool, error) {
dir, envPath := cfg.FilesDir(host)
plain, _, _, err := archivePlainDigest(dir, envPath)
if err != nil {
+ if reached && archiveRefused(err) {
+ // Files the archive can never carry: no retry by itself can
+ // help, so the release stops for an operator (Plan 5 Task 9a),
+ // keeping its phase and everything the target holds.
+ return r.stuck(ctx, &returnRefusal{reason: provision.StuckReleaseTargetFiles,
+ err: fmt.Errorf("move %s: the target's stopped files cannot be archived: %w", id, err)})
+ }
if reached {
return false, fmt.Errorf("digest the stopped files: %w", err)
}
@@ -870,6 +879,15 @@ func (e *returnRefusal) Unwrap() error { return e.err }
func refuseReturn(reason string, err error) error { return &returnRefusal{reason: reason, err: err} }
+// archiveRefused reports whether err is one the archive gives for files it
+// can never carry (a special file, over the cap, too many entries, a path
+// too long), as opposed to a change under the walk or the disk's error,
+// which a retry may get past.
+func archiveRefused(err error) bool {
+ return errors.Is(err, archive.ErrSpecialFile) || errors.Is(err, archive.ErrOverCap) ||
+ errors.Is(err, archive.ErrTooManyEntries) || errors.Is(err, archive.ErrBadPath)
+}
+
// databaseGone reports whether err says the database's data is not there to
// read (absent, short, or not a database), as opposed to a read that failed:
// an *os.PathError other than "does not exist" is the disk's, and transient.
@@ -882,11 +900,11 @@ func databaseGone(err error) bool {
}
// stuck stops the record for an operator (MoveMarkStuck) at the target's
-// returning: the phase, the row and everything the target holds are kept,
-// and move --retry runs the return again.
+// releasing or returning: the phase, the row and everything the target
+// holds are kept, and move --retry runs that phase again.
func (r *rollbackRun) stuck(ctx context.Context, refused *returnRefusal) (bool, error) {
host := r.rec.Host
- log.Printf("moves: %s: the return of %s cannot be built (%v); it stops for an operator: %s", host, r.rb, refused.err, refused.reason)
+ log.Printf("moves: %s: the target's half of %s cannot go on (%v); it stops for an operator: %s", host, r.rb, refused.err, refused.reason)
if _, err := provision.SetRecordMove(ctx, r.s, r.rec.Account, r.rec.App, host, provision.MoveMarkStuck(r.g, refused.reason)); err != nil {
if errors.Is(err, provision.ErrMoveConflict) {
log.Printf("moves: %s: the stuck mark of %s refused (%v); the record moved on", host, r.rb, err)
@@ -1163,30 +1181,26 @@ func (r *rollbackRun) remote(ctx context.Context, obs hostclaim.Observed, j *mov
return true, nil
}
-// engine is the park engine over this box's own bucket: the wired leases, and
-// the view and authorizer Plan 5 supplies, or the stand-ins (the view with
-// the source's region authoritative, which it is again after the rollback).
+// engine is the park engine over this box's own bucket with the claim's
+// leases and view, and the claim's authorizer behind this rollback's own
+// check (rollbackTargetArmed requires all three; nothing stands in).
func (r *rollbackRun) engine(vs lease.VersionedStore) park.Engine {
cfg := r.cfg()
- view, auth := cfg.ParkView, cfg.ParkAuthorize
- if view == nil {
- view = standInView(cfg.Region, roleTarget, r.rec.Move)
- }
- if auth == nil {
- auth = func(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
- if obs.Absent {
- return park.ErrUnsafe
- }
- return r.authorizeChore(obs.Claim, o)
+ own := func(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error {
+ if obs.Absent {
+ return park.ErrUnsafe
}
+ return r.authorizeChore(obs.Claim, o)
}
return park.Engine{Store: vs, Claims: r.s, Bucket: cfg.OwnBucket, Region: cfg.Region, Leases: cfg.ParkLeases,
- Now: r.now, NewID: obligationID, View: view, Authorize: auth, After: cfg.parkFault}
+ Now: r.now, NewID: obligationID, View: cfg.ParkView, Authorize: movesArmAuthorize(cfg.ParkAuthorize, own), After: cfg.parkFault}
}
-// authorizeChore is the stand-in rule until Plan 5 (ruling G17): a chore is
+// authorizeChore is this rollback's own operation check (M17): a chore is
// run only when it is this op's, the claim is still this rollback's with this
-// box no longer executing, and the chore is this region's or this box's.
+// box no longer executing, and the chore is this region's or this box's. The
+// engine runs it before the claim's authorizer; park.Local runs it alone
+// while claim Plan 12's local authorizer is missing (installation).
func (r *rollbackRun) authorizeChore(c *hostclaim.Claim, o hostclaim.Obligation) error {
cfg := r.cfg()
if r.ours(c) != nil || o.Op != r.g.ID || c.Op.Exec == cfg.Box {
@@ -1263,11 +1277,22 @@ func (r *rollbackRun) installation(ctx context.Context, ours bool) (string, erro
local.Box = cfg.Box
}
if local.Authorize == nil {
+ // park.Local's authorizer: claim Plan 12's local authorizer
+ // (ParkLocalAuthorize) behind this rollback's own check. The
+ // claim's engine authorizer refuses a park-local (M9), so it is
+ // never asked here. Until claim Plan 12 lands this rollback's own
+ // rule stands alone: a named exception, the second of the two.
+ // claim Plan 12 F: drop the fallback once ParkLocalAuthorize is
+ // always wired.
+ plan12 := cfg.ParkLocalAuthorize
local.Authorize = func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error {
- if auth := cfg.ParkAuthorize; auth != nil {
- return auth(ctx, hostclaim.Observed{Host: host, Claim: c}, o)
+ if err := r.authorizeChore(c, o); err != nil {
+ return err
+ }
+ if plan12 != nil {
+ return plan12(ctx, c, o)
}
- return r.authorizeChore(c, o)
+ return nil
}
}
if err := r.fault("before-rb-local-retire"); err != nil {
@@ -1346,6 +1371,11 @@ func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark
if incarnation != r.g.Incarnation {
return fmt.Errorf("move %s: the claim names another incarnation", id)
}
+ if _, rendered := j.Steps[landStepPreRender]; rendered {
+ if err := removeRenderTemp(cfg.DataDir, host); err != nil {
+ return err
+ }
+ }
_, err := parkLocalArtifacts(ctx, j, mark, localArtifactPark{DataDir: cfg.DataDir, Incarnation: incarnation, Host: host,
Op: id, Step: rbStepLocal, Paths: paths, NewID: obligationID, Fault: r.fault, Owns: owns})
return err
@@ -1368,6 +1398,35 @@ func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark
return g.RetireInstallationWith(ctx, host, hostguard.Captured{Incarnation: r.g.Incarnation, Stamp: f}, check, work)
}
+// removeRenderTemp removes the held renderer's temp sibling of
+// <host>.config.env (Task 9a fix round 5). The flat renderer writes the
+// rendered bytes to <path>.tmp and renames them over the path (appconfig
+// renderExecConfigBefore), so a crash between the two leaves the temp
+// holding the rendered secrets: never live config, only an unfinished write
+// of this op's, whose render the journal records (pre_render). It is
+// removed, never parked. Only a regular file is removed (Lstat'ed, never
+// followed); anything else at the name is logged and left. A moved row is
+// always flat (the landing refuses a nested row at checking), so the nested
+// renderer's temp (instancefs.WriteAtomic) never holds a held render.
+func removeRenderTemp(dataDir, host string) error {
+ tmp := filepath.Join(dataDir, host+".config.env.tmp")
+ fi, err := os.Lstat(tmp)
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ return nil
+ case err != nil:
+ return err
+ case !fi.Mode().IsRegular():
+ log.Printf("moves: %s: %s is not a regular file (%s); it is not the renderer's temp, and is left", host, tmp, fi.Mode())
+ return nil
+ }
+ if err := os.Remove(tmp); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("remove the held render's temp: %w", err)
+ }
+ log.Printf("moves: %s: the held render's unfinished write %s is removed", host, tmp)
+ return syncPath(dataDir)
+}
+
// rollbackOwnership is the record that this op created a path the target's
// cleaning may park (the final review's finding 1). The registered
// installation, the paths the landing listed under F, is park.Local's and is
@@ -1381,21 +1440,30 @@ func (r *rollbackRun) parkLeftovers(ctx context.Context, j *moves.Journal, mark
// its database, as the record says (landing.installed_db_sha256, stamped
// by the CAS that ends configured); the install refused a WAL or shared
// memory beside an absent database, so what sits beside it since is the
-// op's instance's;
+// op's instance's. Before that stamp, the database and its marker are
+// the op's only when they are the build itself: the database's sha256
+// is the journalled install manifest's, and the marker names the op's
+// generation (a crash between the install and its stamp; carry (b));
// - the quarantine and the litestream shadow, which the install does not
// refuse: the database installed as above, and the install's journal
// records the path absent just before it put the database in place
// (pre_install, notePreInstall). Present then, or no record, is never
// the op's (the final re-review's residual);
-// - the rendered config, <host>.config.env: the held render's ack on this
-// box names this op, and the file's sha256 is the one the ack recorded;
+// - the rendered config, <host>.config.env: the file's sha256 is one this
+// op rendered, as the held render's ack on this box records it (the ack
+// names this op), or as the landing's journal records it: pre_render's
+// intended hash, journalled before the render wrote, or rendered,
+// journalled as soon as the render answered (carry (c); fix round 1,
+// I1). Any other content is not the op's;
// - the sidecar's env: never, since a moved host has no sidecar.
type rollbackOwnership struct {
host, dataDir string
placed stagedFiles // the landing's receipt, or its staged identities
installed bool
+ builtSHA256 string // the journalled install manifest's database, "sha256:<hex>", or ""
+ generation string // the generation the install names (the op id)
absentBefore map[string]bool // the siblings pre_install recorded absent
- configSHA256 string // the held render's ack, "sha256:<hex>", or ""
+ configSHA256 []string // what this op rendered, "sha256:<hex>": the ack's and the journal's
}
// ownership reads the evidence: the landing's journal, the record and the
@@ -1418,6 +1486,31 @@ func (r *rollbackRun) ownership(ctx context.Context, j *moves.Journal) (rollback
if l := r.rec.Move.Landing; l != nil && l.InstalledDBSHA256 != "" {
ev.installed = true
}
+ ev.generation = id
+ if raw, ok := j.Steps[landStepInstall]; ok {
+ var m moves.InstallManifest
+ if err := json.Unmarshal([]byte(raw), &m); err == nil && m.Op == id && m.Host == host && m.DBSHA256 != "" {
+ ev.builtSHA256 = "sha256:" + m.DBSHA256
+ }
+ }
+ if rendered := j.Steps[landStepRendered]; strings.HasPrefix(rendered, "sha256:") {
+ ev.configSHA256 = append(ev.configSHA256, rendered)
+ }
+ if raw, ok := j.Steps[landStepPreRender]; ok {
+ // The hash the render journalled before it wrote (fix round 1, I1):
+ // a crash after the write and before either the ack or the
+ // rendered record still leaves the file the op's by content.
+ var pre preRender
+ if err := json.Unmarshal([]byte(raw), &pre); err != nil {
+ log.Printf("moves: %s: the journal's %s step is unreadable (%v); it vouches for no rendered config", host, landStepPreRender, err)
+ } else {
+ for _, h := range pre.Intended {
+ if strings.HasPrefix(h, "sha256:") {
+ ev.configSHA256 = append(ev.configSHA256, h)
+ }
+ }
+ }
+ }
if raw, ok := j.Steps[landStepPreInstall]; ok {
var pre preInstall
if err := json.Unmarshal([]byte(raw), &pre); err != nil {
@@ -1439,7 +1532,7 @@ func (r *rollbackRun) ownership(ctx context.Context, j *moves.Journal) (rollback
if err := json.Unmarshal(body, &a); err != nil {
log.Printf("moves: %s: the config ack is unreadable (%v); the rendered config is not counted as %s's", host, err, id)
} else if a.MoveOp == id && a.Box == cfg.Box && a.Host == host && a.FileSHA256 != "" {
- ev.configSHA256 = "sha256:" + a.FileSHA256
+ ev.configSHA256 = append(ev.configSHA256, "sha256:"+a.FileSHA256)
}
}
return ev, nil
@@ -1454,7 +1547,7 @@ func (ev rollbackOwnership) owns(path, identity string) (bool, error) {
case ev.host + ".env":
return ev.placed.Env != "" && identity == ev.placed.Env, nil
case ev.host + ".config.env":
- if ev.configSHA256 == "" {
+ if len(ev.configSHA256) == 0 {
return false, nil
}
full := filepath.Join(ev.dataDir, path)
@@ -1465,8 +1558,13 @@ func (ev rollbackOwnership) owns(path, identity string) (bool, error) {
if err != nil {
return false, err
}
- return got == ev.configSHA256, nil
- case db, db + ".gen", db + "-wal", db + "-shm":
+ return containsString(ev.configSHA256, got), nil
+ case db, db + ".gen":
+ if ev.installed {
+ return true, nil
+ }
+ return ev.isBuild(path)
+ case db + "-wal", db + "-shm":
return ev.installed, nil
case db + ".corrupt", ".activator-" + db + "-litestream":
return ev.installed && ev.absentBefore[path], nil
@@ -1474,6 +1572,32 @@ func (ev rollbackOwnership) owns(path, identity string) (bool, error) {
return false, nil
}
+// isBuild reports whether path, the database or its marker at the flat
+// paths, is this op's build before the record stamped it installed: the
+// database's sha256 is the journalled build's, and the marker names the
+// op's generation. A link or anything else is not.
+func (ev rollbackOwnership) isBuild(path string) (bool, error) {
+ if ev.builtSHA256 == "" {
+ return false, nil
+ }
+ full := filepath.Join(ev.dataDir, path)
+ if fi, err := os.Lstat(full); err != nil || !fi.Mode().IsRegular() {
+ return false, nil
+ }
+ if strings.HasSuffix(path, ".gen") {
+ gen, err := hostbox.ReadGenerationMarker(full)
+ if err != nil {
+ return false, nil
+ }
+ return gen == ev.generation, nil
+ }
+ got, err := fileSHA256(full)
+ if err != nil {
+ return false, err
+ }
+ return got == ev.builtSHA256, nil
+}
+
// deleteRow deletes the row and its timers rows in one registry transaction
// (registry.DeleteIfHoldWithTimers, J12), only while the rollback holds the
// row, under the host's hold lock with the record re-read there; the journal
diff --git a/internal/host/movesrollback_finalfix_test.go b/internal/host/movesrollback_finalfix_test.go
index d227e650d..db865eb91 100644
--- a/internal/host/movesrollback_finalfix_test.go
+++ b/internal/host/movesrollback_finalfix_test.go
@@ -354,8 +354,8 @@ func (p *policyProbe) wrap(c *moveCfg) {
p.check(movesActionHeldTimers, hostwrite.Identity{})
return timers(ctx, host, rows)
}
- c.RenderHeld = func(ctx context.Context, s release.Store, rt registry.Route, op string) (appconfig.HeldRender, error) {
- hr, err := render(ctx, s, rt, op)
+ c.RenderHeld = func(ctx context.Context, s release.Store, rt registry.Route, op string, before func(string) error) (appconfig.HeldRender, error) {
+ hr, err := render(ctx, s, rt, op, before)
if err == nil {
p.check(movesActionHeldConfig, HeldConfigIdentity(hr.BundleETags, hr.OverlayETag))
}
diff --git a/internal/host/movessources.go b/internal/host/movessources.go
new file mode 100644
index 000000000..e3caad3ed
--- /dev/null
+++ b/internal/host/movessources.go
@@ -0,0 +1,648 @@
+package host
+
+// The moves park providers in production (moves Plan 5 Task 9a; the delta's
+// findings M5, M6, M9, M10 and M17, and the Task 9a rulings). Three things
+// are built here and registered with the claim at agent start, before
+// newClaimWiring:
+//
+// - the park view (movesParkView): the spec's authoritative-region table
+// (§5.1), the exact-version generation marker, and the pins;
+// - the lease set (movesLeaseSet): the scope's own lock, for moves/<host>
+// scopes and for instances/<host>[/<inc>] locations alike (M6);
+// - Task 5's retirement evidence (movesRetirementEvidence), which also
+// answers VerifyMovesSweep.
+//
+// The moves arms then take the claim's own regional engine through
+// ClaimProviders.MoveParks (armMovesParks): its leases, view and authorizer,
+// after checking that its bucket and region are this box's own (M17).
+//
+// Production stays unarmed. Registration and arming happen only with both
+// CARLOS_MOVES_PHASE1 and CARLOS_HOST_CLAIM on. Even then, three pieces are
+// claim Plan 12's and are not on main: Engine.RunHeld (M5), the lease-set
+// contract for moves scopes (M6) and the MoveParks local authorizer (M9).
+// Each sits behind movesClaimPlan12, and the production guard
+// (movesPlan12Parks) refuses by name while it is missing. Until it lands:
+//
+// - the registered view and lease set refuse every call with a
+// *ClaimDependencyError naming claim Plan 12, so the claim's regional
+// worker queues and runs nothing through them;
+// - armMovesParks leaves ParkLeases, ParkView and ParkAuthorize unset, so
+// every moves arm stays unarmed and logs its standing notice.
+//
+// The tests inject fakes for the Plan 12 pieces.
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/genselect"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfiles"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// movesNoLiveRegion is the authoritative region the view names when the
+// spec's table says none: a claim with no holder, or a creating, deleting,
+// parking or purging op. No region code is "none", so the engine protects
+// nothing in any region, which is what "nothing is live" means.
+const movesNoLiveRegion = "none"
+
+// movesClaimPlan12 is what claim Plan 12 adds to the MoveParks capability.
+// Nothing on main implements it.
+//
+// - RunHeld (M5) runs one chore under a lease the caller already holds,
+// the landing's own-location lease, which the landing keeps from
+// checking through published (spec §7.6). Engine.Run would acquire the
+// same lock again and wait on itself.
+// - AuthorizeLocal (M9) is the owner authorizer for this box's own
+// park.Local chores. The MoveParks engine's authorizer sends every chore
+// without a bound Park to VerifyBound, which refuses a park-local.
+//
+// The lease-set contract for moves scopes (M6) is agreed with platform-1e and
+// arrives with the same plan; it has no method of its own.
+type movesClaimPlan12 interface {
+ RunHeld(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error
+ AuthorizeLocal(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error
+}
+
+// movesPlan12Contracts names the claim Plan 12 pieces the guard waits on.
+var movesPlan12Contracts = []string{
+ "claim Plan 12 Engine.RunHeld for the landing's own-lease residue park (M5)",
+ "claim Plan 12 lease-set contract for moves scopes (M6)",
+ "claim Plan 12 MoveParks local authorizer (M9)",
+}
+
+// productionMovesPlan12 is claim Plan 12's implementation once it lands.
+var productionMovesPlan12 movesClaimPlan12
+
+// movesPlan12Parks is the production guard: claim Plan 12's pieces, or a
+// refusal that names each one.
+func movesPlan12Parks() (movesClaimPlan12, error) {
+ if productionMovesPlan12 == nil {
+ return nil, &ClaimDependencyError{Contracts: movesPlan12Contracts}
+ }
+ return productionMovesPlan12, nil
+}
+
+// movesParkView is the park view the claim's engine and the moves arms read
+// (park.View): bound to the observation's ETag, seen from region (in
+// production the claim's own region, movesParkViewAt), naming
+// the authoritative region by the spec's table (§5.1, movesAuthoritative).
+// In the authoritative region it reads the location's generation.json by
+// exact version (MarkerKnown, MarkerGeneration, Marker; absent is the origin)
+// and pins the snapshot version the marker names, the installed root.
+// GenerationOf is the landing's classifier over the holder's location.
+// Anything it cannot read or place is an error, never a smaller live set.
+func movesParkView(records MovesRecordReader, data genselect.Reader, bucket, region string) func(context.Context, hostclaim.Observed) (park.View, error) {
+ return movesParkViewAt(records, data, bucket, fixedRegion(region))
+}
+
+// fixedRegion is a region resolver that always answers region.
+func fixedRegion(region string) func(context.Context) (string, error) {
+ return func(context.Context) (string, error) { return region, nil }
+}
+
+// cachedRegion resolves once and keeps the first success; a failure is
+// returned and asked again next time. A box's roster region does not change
+// under a running agent; if it did, the engine's own check (the view's
+// region against its own) refuses.
+func cachedRegion(resolve func(context.Context) (string, error)) func(context.Context) (string, error) {
+ var mu sync.Mutex
+ var cached *string
+ return func(ctx context.Context) (string, error) {
+ mu.Lock()
+ defer mu.Unlock()
+ if cached != nil {
+ return *cached, nil
+ }
+ r, err := resolve(ctx)
+ if err != nil {
+ return "", err
+ }
+ cached = &r
+ return r, nil
+ }
+}
+
+// movesParkViewAt is movesParkView seen from the region regionOf answers.
+// Production's is the claim's verified roster row's region
+// (productionRosterRegion), the region the claim's engine runs in, so the
+// view's region is the claim's by construction (M17; fix round 1). The home
+// region, "", is a region like any other.
+func movesParkViewAt(records MovesRecordReader, data genselect.Reader, bucket string, regionOf func(context.Context) (string, error)) func(context.Context, hostclaim.Observed) (park.View, error) {
+ return func(ctx context.Context, obs hostclaim.Observed) (park.View, error) {
+ if ctx == nil || records == nil || data == nil || bucket == "" || regionOf == nil {
+ return park.View{}, hostwrite.ErrUnavailable
+ }
+ if obs.Absent || obs.Claim == nil || obs.ETag == "" || obs.Host == "" || obs.Claim.Host != obs.Host {
+ return park.View{}, park.ErrUnsafe
+ }
+ region, err := regionOf(ctx)
+ if err != nil {
+ return park.View{}, fmt.Errorf("this box's region: %w", err)
+ }
+ auth, err := movesAuthoritative(ctx, records, obs, region)
+ if err != nil {
+ return park.View{}, err
+ }
+ v := park.View{ClaimETag: obs.ETag, Region: region, AuthoritativeRegion: auth}
+ h := obs.Claim.Holder
+ if h == nil {
+ return v, nil
+ }
+ location := strings.TrimSuffix(h.Location, "/")
+ v.GenerationOf = func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }
+ if auth != region {
+ return v, nil
+ }
+ listing, err := data.ListVersions(ctx, bucket, location+"/")
+ if err != nil {
+ return park.View{}, fmt.Errorf("list %s: %w", location, err)
+ }
+ if err := readMarker(ctx, data, bucket, obs.Host, location, *h, listing, &v); err != nil {
+ return park.View{}, err
+ }
+ if err := pinFilesLiveSet(ctx, data, bucket, location, listing, &v, liveGenerations(obs.Claim, v)...); err != nil {
+ return park.View{}, err
+ }
+ return v, nil
+ }
+}
+
+// liveGenerations are the generations whose files manifests are live
+// (fix round 1, I2): the holder's and the marker's, and every generation
+// park.LiveGenerations, the engine's own protected set, names for the
+// holder's location, the unfinished admission chain included. When the
+// engine refuses that walk it refuses every park of the location too, so
+// the holder's and the marker's are enough then. Sorted, for a stable read
+// order.
+func liveGenerations(c *hostclaim.Claim, v park.View) []string {
+ seen := map[string]bool{c.Holder.Generation: true, v.MarkerGeneration: true}
+ if live, err := park.LiveGenerations(c, v, c.Holder.Location); err == nil {
+ for g := range live {
+ seen[g] = true
+ }
+ }
+ out := make([]string, 0, len(seen))
+ for g := range seen {
+ out = append(out, g)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// pinFilesLiveSet pins the files live set (J2/F4, Paul's #236 ruling; M10):
+// every exact version an object a live generation's files manifest names,
+// read through hostfiles.Load by the version the listing names. The live
+// generations are liveGenerations'. The rule is M10's: spec
+// §6.6 P6-D3 says a target's manifest never references a source-region
+// object, so a manifest in this region's bucket names objects in this
+// region's bucket, and a region pins only what its own live manifests name.
+// With no manifest, nothing more is pinned. A manifest that names an object
+// outside the location, or one the listing does not hold, refuses.
+func pinFilesLiveSet(ctx context.Context, data genselect.Reader, bucket, location string, listing []lease.ObjectVersion, v *park.View, generations ...string) error {
+ seen := map[string]bool{}
+ for _, gen := range generations {
+ if gen == "" || gen == hostclaim.RootGeneration || gen == landingMarkerGeneration || seen[gen] {
+ continue
+ }
+ seen[gen] = true
+ prefix := location + "/files/g-" + gen + "/"
+ key := hostfiles.ManifestKey(prefix)
+ var current *lease.ObjectVersion
+ for i := range listing {
+ if listing[i].Key == key && listing[i].IsLatest {
+ current = &listing[i]
+ }
+ }
+ if current == nil {
+ continue
+ }
+ if current.DeleteMarker || current.VersionID == "" || current.VersionID == "null" {
+ return fmt.Errorf("%w: %s is not a readable version", park.ErrUnsafe, key)
+ }
+ m, found, err := hostfiles.Load(ctx, versionStore{data: data, bucket: bucket, key: key, version: current.VersionID}, prefix)
+ if err != nil || !found {
+ return fmt.Errorf("%w: the files manifest %s: %v", park.ErrUnsafe, key, err)
+ }
+ for _, entries := range []map[string]hostfiles.Entry{m.Entries, m.Platform} {
+ for path, e := range entries {
+ if e.Object == "" || e.Deleted {
+ continue
+ }
+ if !strings.HasPrefix(e.Object, location+"/") {
+ return fmt.Errorf("%w: %s names %s for %s, outside %s", park.ErrUnsafe, key, e.Object, path, location)
+ }
+ if _, ok := v.GenerationOf(park.Ref{Key: e.Object}); !ok {
+ return fmt.Errorf("%w: %s names %s, which belongs to no generation", park.ErrUnsafe, key, e.Object)
+ }
+ pinned := false
+ for _, x := range listing {
+ if x.Key == e.Object && !x.DeleteMarker && x.VersionID != "" {
+ if v.Pinned == nil {
+ v.Pinned = map[park.Ref]bool{}
+ }
+ v.Pinned[park.Ref{Key: x.Key, VersionID: x.VersionID}] = true
+ pinned = true
+ }
+ }
+ if !pinned {
+ return fmt.Errorf("%w: %s names %s, which the bucket does not hold", park.ErrUnsafe, key, e.Object)
+ }
+ }
+ }
+ }
+ return nil
+}
+
+// versionStore is a read-only release.Store over one object read by the
+// exact version a listing named: hostfiles.Load's reader for a manifest.
+type versionStore struct {
+ data genselect.Reader
+ bucket, key, version string
+}
+
+func (s versionStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ if key != s.key {
+ return nil, "", os.ErrNotExist
+ }
+ body, err := s.data.GetVersion(ctx, s.bucket, key, s.version)
+ return body, s.version, err
+}
+func (versionStore) Put(context.Context, string, []byte, string) (string, error) {
+ return "", errors.New("host: the park view reads only")
+}
+func (versionStore) List(context.Context, string) ([]string, error) {
+ return nil, errors.New("host: the park view reads only")
+}
+func (versionStore) ListDirs(context.Context, string) ([]string, error) {
+ return nil, errors.New("host: the park view reads only")
+}
+func (versionStore) Delete(context.Context, string) error {
+ return errors.New("host: the park view reads only")
+}
+
+// readMarker fills the marker and the installed root's pin from the
+// location's generation.json, read by the exact version the listing names.
+func readMarker(ctx context.Context, data genselect.Reader, bucket, host, location string, h hostclaim.Holder, listing []lease.ObjectVersion, v *park.View) error {
+ key := location + "/generation.json"
+ var latest *lease.ObjectVersion
+ for i := range listing {
+ if listing[i].Key == key && listing[i].IsLatest {
+ if latest != nil {
+ return fmt.Errorf("%w: two current versions of %s", park.ErrUnsafe, key)
+ }
+ latest = &listing[i]
+ }
+ }
+ if latest == nil {
+ // Absent everywhere means origin (§5.1).
+ v.MarkerKnown = true
+ return nil
+ }
+ if latest.DeleteMarker || latest.VersionID == "" || latest.VersionID == "null" {
+ return fmt.Errorf("%w: %s is not a readable version", park.ErrUnsafe, key)
+ }
+ raw, err := data.GetVersion(ctx, bucket, key, latest.VersionID)
+ if err != nil {
+ return fmt.Errorf("read %s: %w", key, err)
+ }
+ var m genselect.Marker
+ if err := json.Unmarshal(raw, &m); err != nil {
+ return fmt.Errorf("%w: %s: %v", park.ErrUnsafe, key, err)
+ }
+ if m.Op == "" || m.Host != host || m.Incarnation != h.Incarnation {
+ return fmt.Errorf("%w: %s names %s/%s generation %q, not the holder's", park.ErrUnsafe, key, m.Host, m.Incarnation, m.Op)
+ }
+ v.MarkerKnown, v.MarkerGeneration, v.Marker = true, m.Op, park.Ref{Key: key, VersionID: latest.VersionID}
+ if m.SnapshotKey != "" || m.SnapshotVersion != "" {
+ root := park.Ref{Key: m.SnapshotKey, VersionID: m.SnapshotVersion}
+ if root.Key == "" || root.VersionID == "" {
+ return fmt.Errorf("%w: %s pins %q at %q", park.ErrUnsafe, key, root.Key, root.VersionID)
+ }
+ if _, ok := v.GenerationOf(root); !ok {
+ return fmt.Errorf("%w: %s pins %s outside %s", park.ErrUnsafe, key, root.Key, location)
+ }
+ v.Pinned = map[park.Ref]bool{root: true}
+ }
+ return nil
+}
+
+// movesAuthoritative is the spec's authoritative-region table (§5.1, claim
+// rev 6.5 §6.2a, R4-3): the region of the holder's current installation.
+//
+// - no holder, creating, deleting, parking, purging: none (nothing live);
+// - serving, pushing: op.exec's region, which is the region of the newest
+// admission into holder.generation, or, for a holder with no such
+// admission (a legacy installation), the record's region;
+// - moving (before the flip), reverting: the source's, move.from;
+// - landing: the source's until the target's probe is admitted into
+// holder.generation in move.to, the target's from then on, committed or
+// not; a committed landing with no such admission refuses. Seen from
+// the source's own region, the source stays authoritative until the
+// record commits (J2/F4: its files live set is protected on the source
+// until the commit);
+// - restoring: the region of the newest admission into holder.generation,
+// the source's or the target's (a clean return's CAS names the source's
+// generation again, a dirty return keeps the target's until the source's
+// probe is admitted); no admission is the source's pre-move generation.
+//
+// For a move kind the record must carry the op's move, or have closed it
+// (the record-first close, M4), when the newest admission decides alone. A
+// region the claim's history contradicts refuses. So does any other kind
+// (placing, relocating): it has no row in the table.
+func movesAuthoritative(ctx context.Context, records MovesRecordReader, obs hostclaim.Observed, region string) (string, error) {
+ c := obs.Claim
+ h := c.Holder
+ if h == nil {
+ return movesNoLiveRegion, nil
+ }
+ switch c.Op.Kind {
+ case hostclaim.KindCreating, hostclaim.KindDeleting, hostclaim.KindParking, hostclaim.KindPurging:
+ return movesNoLiveRegion, nil
+ }
+ admitted, known := admittedRegion(*h)
+ switch c.Op.Kind {
+ case hostclaim.KindServing, hostclaim.KindPushing:
+ if known {
+ return admitted, nil
+ }
+ rec, err := holderRecord(ctx, records, obs)
+ if err != nil {
+ return "", err
+ }
+ if rec.Region == "" {
+ return "", fmt.Errorf("%w: %s's record names no region and the claim no admission", park.ErrUnsafe, obs.Host)
+ }
+ return rec.Region, nil
+ case hostclaim.KindMoving, hostclaim.KindReverting, hostclaim.KindLanding, hostclaim.KindRestoring:
+ default:
+ return "", fmt.Errorf("%w: the authoritative-region table has no row for a %s op", park.ErrUnsafe, c.Op.Kind)
+ }
+ rec, err := holderRecord(ctx, records, obs)
+ if err != nil {
+ return "", err
+ }
+ m := rec.Move
+ if m == nil || m.ID != c.Op.ID || m.Legacy() || m.From == m.To {
+ if l := rec.LastMove; m == nil && l != nil && l.ID == c.Op.ID && known {
+ return admitted, nil // the record closed first (M4); the claim's history decides
+ }
+ return "", fmt.Errorf("%w: the record does not carry move %s", park.ErrUnsafe, c.Op.ID)
+ }
+ contradicts := func(want ...string) error {
+ if !known {
+ return nil
+ }
+ for _, w := range want {
+ if admitted == w {
+ return nil
+ }
+ }
+ return fmt.Errorf("%w: holder.generation was admitted in %s, which a %s op of %s cannot have", park.ErrUnsafe, admitted, c.Op.Kind, m.ID)
+ }
+ switch c.Op.Kind {
+ case hostclaim.KindMoving, hostclaim.KindReverting:
+ if err := contradicts(m.From); err != nil {
+ return "", err
+ }
+ return m.From, nil
+ case hostclaim.KindLanding:
+ if err := contradicts(m.From, m.To); err != nil {
+ return "", err
+ }
+ if known && admitted == m.To {
+ if region == m.From && c.Op.Phase != hostclaim.PhaseCommitted && m.State != provision.MoveCommitted {
+ // J2/F4 (Paul's #236 ruling): the source's live set, its
+ // files live set among it, stays protected on the source
+ // until the record commits. Seen from the source's region the
+ // source stays authoritative, so the engine protects there
+ // too (and refuses a park of the location, whose current
+ // generation the source's marker does not name).
+ return m.From, nil
+ }
+ return m.To, nil
+ }
+ if c.Op.Phase == hostclaim.PhaseCommitted || m.State == provision.MoveCommitted {
+ return "", fmt.Errorf("%w: move %s committed with no admission in %s", park.ErrUnsafe, m.ID, m.To)
+ }
+ return m.From, nil
+ default: // restoring
+ if err := contradicts(m.From, m.To); err != nil {
+ return "", err
+ }
+ if known {
+ return admitted, nil
+ }
+ return m.From, nil
+ }
+}
+
+// admittedRegion is the region of the newest admission into
+// holder.generation, the origin included.
+func admittedRegion(h hostclaim.Holder) (string, bool) {
+ want := h.Generation
+ if want == "" {
+ want = hostclaim.RootGeneration
+ }
+ for i := len(h.Admissions) - 1; i >= 0; i-- {
+ g := h.Admissions[i].Generation
+ if g == "" {
+ g = hostclaim.RootGeneration
+ }
+ if g == want {
+ r := h.Admissions[i].Region
+ return r, r != ""
+ }
+ }
+ return "", false
+}
+
+// holderRecord reads the holder's own record: an instance holder's key, the
+// same account, app and host.
+func holderRecord(ctx context.Context, records MovesRecordReader, obs hostclaim.Observed) (*provision.Record, error) {
+ h := obs.Claim.Holder
+ if h.Family != "instance" || h.Key != provision.RecordKey(h.Account, h.App, obs.Host) {
+ return nil, fmt.Errorf("%w: the holder of %s is not an instance record", park.ErrUnsafe, obs.Host)
+ }
+ rec, err := records(ctx, h.Account, h.App, obs.Host)
+ if err != nil {
+ return nil, fmt.Errorf("read %s's record: %w", obs.Host, err)
+ }
+ if rec == nil || rec.Account != h.Account || rec.App != h.App || rec.Host != obs.Host {
+ return nil, fmt.Errorf("%w: %s's record is not the holder's", park.ErrUnsafe, obs.Host)
+ }
+ return rec, nil
+}
+
+// movesLeaseSet is the lease set for the scopes moves parks (M6, platform-1e's
+// answer): the scope's own lock, and nothing else, for each of
+//
+// - moves/<host> and moves/<host>/<op>: the staging writers' bound is the
+// scope's evidence_at (seal, abort, revert or return seal bound), which
+// every staging signature already respects (moves/lease.go), so the
+// staging lease in the deployment bucket is not named;
+// - instances/<host> and instances/<host>/<incarnation>: the location's
+// replica lease is the lock at the location itself, the writer's lease
+// (lease.Acquire on the holder location), so the scope's own lock is it.
+//
+// Only this box's own data bucket is answered; any other scope refuses.
+func movesLeaseSet(bucket string) hostbox.LeaseSet {
+ return func(ctx context.Context, b, scope string) ([]string, error) {
+ if ctx == nil || bucket == "" || b != bucket {
+ return nil, hostbox.ErrChanged
+ }
+ if !movesLeaseScope(scope) {
+ return nil, fmt.Errorf("%w: %q is not a scope moves names a lease set for", park.ErrUnsafe, scope)
+ }
+ return []string{scope}, nil
+ }
+}
+
+// movesLeaseScope reports whether scope is one movesLeaseSet answers.
+func movesLeaseScope(scope string) bool {
+ parts := strings.Split(scope, "/")
+ if len(parts) < 2 || len(parts) > 3 {
+ return false
+ }
+ if _, err := hostclaim.Key(parts[1]); err != nil {
+ return false
+ }
+ switch parts[0] {
+ case "instances":
+ if len(parts) == 2 {
+ return true
+ }
+ return len(parts[2]) == 64 && isHexString(parts[2]) && strings.ToLower(parts[2]) == parts[2]
+ case "moves":
+ return len(parts) == 2 || moves.ValidOpID(parts[2])
+ }
+ return false
+}
+
+// movesParkSources is what registerMovesParks registers: the view and the
+// lease set behind the claim Plan 12 guard, and Task 5's evidence.
+type movesParkSources struct {
+ View func(context.Context, hostclaim.Observed) (park.View, error)
+ Evidence hostbox.RegionalEvidence
+ Leases hostbox.LeaseSet
+}
+
+// newMovesParkSources builds the three providers. plan12 is the guard
+// (movesPlan12Parks in production): while it refuses, the view and the
+// lease set refuse with its error, so nothing is queued or run through them.
+func newMovesParkSources(records MovesRecordReader, data genselect.Reader, bucket string, regionOf func(context.Context) (string, error), evidence hostbox.RegionalEvidence, plan12 func() (movesClaimPlan12, error)) (movesParkSources, error) {
+ if records == nil || data == nil || bucket == "" || regionOf == nil || evidence == nil || plan12 == nil {
+ return movesParkSources{}, hostwrite.ErrUnavailable
+ }
+ view, leases := movesParkViewAt(records, data, bucket, regionOf), movesLeaseSet(bucket)
+ return movesParkSources{
+ View: func(ctx context.Context, obs hostclaim.Observed) (park.View, error) {
+ if _, err := plan12(); err != nil {
+ return park.View{}, err
+ }
+ return view(ctx, obs)
+ },
+ Evidence: evidence,
+ Leases: func(ctx context.Context, b, scope string) ([]string, error) {
+ if _, err := plan12(); err != nil {
+ return nil, err
+ }
+ return leases(ctx, b, scope)
+ },
+ }, nil
+}
+
+// registerMovesParks registers the moves park providers with the claim
+// (RegisterProductionClaimMoveParks), at agent start before newClaimWiring.
+func registerMovesParks(src movesParkSources) error {
+ return RegisterProductionClaimMoveParks(src.View, src.Evidence, src.Leases)
+}
+
+// armMovesParks sets the moves arms' park hooks from the claim's MoveParks
+// capability (M17): its engine's leases, view and authorizer, only when the
+// engine's bucket and region are this box's own bucket and placement
+// region, and claim Plan 12's RunHeld and local authorizer (M5, M9). Any
+// refusal leaves every hook unset, so every arm stays unarmed.
+func armMovesParks(ctx context.Context, cfg *moveCfg, providers ClaimProviders, plan12 func() (movesClaimPlan12, error)) error {
+ if ctx == nil || cfg == nil || plan12 == nil {
+ return hostwrite.ErrUnavailable
+ }
+ engine, _, err := providers.MoveParks(ctx)
+ if err != nil {
+ return fmt.Errorf("the claim's MoveParks: %w", err)
+ }
+ if engine.Bucket != cfg.OwnBucket || engine.Region != cfg.Region {
+ return fmt.Errorf("%w: the claim's park engine is %s in %q, this box's moves use %s in %q", hostbox.ErrChanged, engine.Bucket, engine.Region, cfg.OwnBucket, cfg.Region)
+ }
+ if engine.Leases == nil || engine.View == nil || engine.Authorize == nil {
+ return errors.Join(hostwrite.ErrUnavailable, errors.New("the claim's park engine lacks its leases, view or authorizer"))
+ }
+ p12, err := plan12()
+ if err != nil {
+ return err
+ }
+ if p12 == nil {
+ return &ClaimDependencyError{Contracts: movesPlan12Contracts}
+ }
+ cfg.ParkLeases, cfg.ParkView, cfg.ParkAuthorize = engine.Leases, engine.View, engine.Authorize
+ cfg.ParkRunHeld, cfg.ParkLocalAuthorize = p12.RunHeld, p12.AuthorizeLocal
+ return nil
+}
+
+// registerProductionMovesParks is the agent's registration, before
+// newClaimWiring: only with both CARLOS_HOST_CLAIM and CARLOS_MOVES_PHASE1
+// on (ruling H1), over the deployment store's records, this box's data
+// bucket read by exact version (lease.NewVersionedS3, through rootaws), and
+// the region regionOf answers, behind the claim Plan 12 guard. Production's
+// regionOf is productionRosterRegion: the claim's verified roster row, the
+// home region "" included (fix round 1). Off, it registers nothing and the
+// claim keeps its named refusals. Task 5's evidence goes through refusals,
+// which pages its anomalies once (movesrefusals.go).
+func registerProductionMovesParks(ctx context.Context, getenv func(string) string, store release.Store, hib activator.Config, regionOf func(context.Context) (string, error), refusals *movesRefusals) error {
+ if !hostwrite.Enabled(getenv) || !moves.Enabled(getenv) {
+ return nil
+ }
+ if ctx == nil || store == nil || hib.Bucket == "" || hib.Region == "" || regionOf == nil || refusals == nil {
+ return errors.Join(hostwrite.ErrUnavailable, errors.New("the moves park view needs the deployment store, the data bucket and this box's roster region"))
+ }
+ data, err := lease.NewVersionedS3(ctx, map[string]string{hib.Bucket: hib.Region})
+ if err != nil {
+ return err
+ }
+ records := MovesRecordsFrom(store)
+ evidence := refusals.evidence(movesRetirementEvidence{Records: records, Now: time.Now})
+ src, err := newMovesParkSources(records, data, hib.Bucket, cachedRegion(regionOf), evidence, movesPlan12Parks)
+ if err != nil {
+ return err
+ }
+ return registerMovesParks(src)
+}
+
+// productionRosterRegion is this box's region as the claim knows it: the
+// region of its verified roster row (readProductionRosterRow), the same row
+// OpenClaimOwners builds the claim's identity, and so its engine's region,
+// from.
+func productionRosterRegion(store release.Store, getenv func(string) string) func(context.Context) (string, error) {
+ return func(ctx context.Context) (string, error) {
+ rr, err := readProductionRosterRow(ctx, store, getenv)
+ if err != nil {
+ return "", err
+ }
+ return rr.row.Region, nil
+ }
+}
diff --git a/internal/host/movessources_test.go b/internal/host/movessources_test.go
new file mode 100644
index 000000000..dcef3995a
--- /dev/null
+++ b/internal/host/movessources_test.go
@@ -0,0 +1,712 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/lease/leasetest"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/park"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The moves park providers (Plan 5 Task 9a): the view, the lease set, the
+// claim Plan 12 guard, the registration and the arming.
+
+const (
+ vwHost = "vw.example"
+ vwAcct = "acct"
+ vwApp = "app"
+ vwMove = "mv_vwvwvwvwvwvwvwvwvwvwvwvwvw"
+ vwRb = "rb_vwvwvwvwvwvwvwvwvwvwvwvwvw"
+)
+
+var vwCreated = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+
+// vwFixture is one host's claim observation, its record and its data
+// bucket, for the view.
+type vwFixture struct {
+ inc, location string
+ rec *provision.Record
+ bucket *leasetest.VersionedBucket
+ reads, lists int
+}
+
+func newVwFixture(t *testing.T) *vwFixture {
+ t.Helper()
+ inc, err := hostclaim.Incarnation(vwAcct, vwApp, vwHost, vwCreated)
+ if err != nil {
+ t.Fatal(err)
+ }
+ f := &vwFixture{inc: inc, location: "instances/" + vwHost + "/" + inc, bucket: leasetest.NewVersionedBucket()}
+ f.bucket.AddBucket("data")
+ f.rec = &provision.Record{Schema: 1, Host: vwHost, App: vwApp, Account: vwAcct, CreatedAt: vwCreated, Region: "ie"}
+ return f
+}
+
+func (f *vwFixture) records() MovesRecordReader {
+ return func(_ context.Context, acct, app, host string) (*provision.Record, error) {
+ f.reads++
+ if acct != vwAcct || app != vwApp || host != vwHost || f.rec == nil {
+ return nil, errors.New("no such record")
+ }
+ cp := *f.rec
+ return &cp, nil
+ }
+}
+
+// countingReader counts the view's listings, to show a non-authoritative
+// region reads nothing.
+type countingReader struct {
+ *leasetest.VersionedBucket
+ f *vwFixture
+}
+
+func (r countingReader) ListVersions(ctx context.Context, bucket, prefix string) ([]lease.ObjectVersion, error) {
+ r.f.lists++
+ return r.VersionedBucket.ListVersions(ctx, bucket, prefix)
+}
+
+func (f *vwFixture) view(region string) func(context.Context, hostclaim.Observed) (park.View, error) {
+ return movesParkView(f.records(), countingReader{f.bucket, f}, "data", region)
+}
+
+// holder is the fixture's holder at generation gen with admissions, each
+// "generation@region".
+func (f *vwFixture) holder(gen string, admissions ...string) *hostclaim.Holder {
+ h := &hostclaim.Holder{Family: "instance", Key: provision.RecordKey(vwAcct, vwApp, vwHost), Account: vwAcct, App: vwApp,
+ Incarnation: f.inc, CreatedAt: vwCreated, Location: f.location, Generation: gen, Stamp: 3, Session: 3}
+ for i, a := range admissions {
+ g, r, _ := strings.Cut(a, "@")
+ h.Admissions = append(h.Admissions, hostclaim.AdmissionRecord{Session: uint64(i + 1), Box: r + "-box", Generation: g, Mode: "cold", Region: r})
+ }
+ return h
+}
+
+func vwObs(c *hostclaim.Claim) hostclaim.Observed {
+ c.Host = vwHost
+ return hostclaim.Observed{Host: vwHost, ETag: "etag-1", Claim: c}
+}
+
+func (f *vwFixture) move(state string) {
+ f.rec.Move = &provision.MoveState{ID: vwMove, Incarnation: f.inc, From: "ie", To: "uk", State: state}
+}
+
+// marker writes generation.json at the location naming gen, pinning a
+// snapshot of it, and returns the marker's version and the snapshot's.
+func (f *vwFixture) marker(t *testing.T, gen, inc string) (markerVersion, snapshot string) {
+ t.Helper()
+ snapKey := f.location + "/g-" + gen + "/0009/0000000000000001-0000000000000005.ltx"
+ snapshot = f.bucket.Put("data", snapKey, []byte("snapshot"))
+ body, err := json.Marshal(map[string]any{"op": gen, "incarnation": inc, "host": vwHost, "box": "uk-box", "installed_at": vwCreated,
+ "manifest_sha256": "x", "snapshot_key": snapKey, "snapshot_version": snapshot, "snapshot_txid": 5, "snapshot_crc64": 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return f.bucket.Put("data", f.location+"/generation.json", body), snapshot
+}
+
+// TestMoveParkViewFollowsTheAuthoritativeRegionTable: one row per claim
+// state of the spec's table (§5.1), seen from a region that is neither, so
+// only the authoritative region is asked of the view, plus the refusals.
+func TestMoveParkViewFollowsTheAuthoritativeRegionTable(t *testing.T) {
+ type row struct {
+ name string
+ claim func(f *vwFixture) *hostclaim.Claim
+ rec func(f *vwFixture)
+ want string // "" wants a refusal
+ }
+ op := func(kind, phase string) hostclaim.Op {
+ return hostclaim.Op{Kind: kind, ID: vwMove, Phase: phase, Exec: "box"}
+ }
+ rows := []row{
+ {"free", func(f *vwFixture) *hostclaim.Claim { return &hostclaim.Claim{} }, nil, movesNoLiveRegion},
+ {"creating", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(""), Op: hostclaim.Op{Kind: hostclaim.KindCreating, ID: "op"}}
+ }, nil, movesNoLiveRegion},
+ {"deleting", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindDeleting, ID: "op"}}
+ }, nil, movesNoLiveRegion},
+ {"parking", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindParking, ID: "op"}}
+ }, nil, movesNoLiveRegion},
+ {"purging", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindPurging, ID: "op"}}
+ }, nil, movesNoLiveRegion},
+ {"serving: the admission's region", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "old@ie", "cur@uk"), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "uk-box"}}
+ }, nil, "uk"},
+ {"serving: a legacy holder takes the record's region", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(""), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}}
+ }, nil, "ie"},
+ {"serving: no admission and no record region", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(""), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}}
+ }, func(f *vwFixture) { f.rec.Region = "" }, ""},
+ {"pushing", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@uk"), Op: hostclaim.Op{Kind: hostclaim.KindPushing, ID: "push"}}
+ }, nil, "uk"},
+ {"moving before the flip", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: op(hostclaim.KindMoving, hostclaim.PhaseMoving)}
+ }, func(f *vwFixture) { f.move(provision.MoveCutting) }, "ie"},
+ {"reverting", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: op(hostclaim.KindReverting, "")}
+ }, func(f *vwFixture) { f.move(provision.MoveReverting) }, "ie"},
+ {"landing before the target's probe is admitted", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: op(hostclaim.KindLanding, provision.LandBuilding)}
+ }, func(f *vwFixture) { f.move(provision.MoveLanding) }, "ie"},
+ {"landing from the target's probe's admission", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(vwMove, "cur@ie", vwMove+"@uk"), Op: op(hostclaim.KindLanding, provision.LandProbing)}
+ }, func(f *vwFixture) { f.move(provision.MoveLanding) }, "uk"},
+ {"landing committed", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("fresh2", "cur@ie", vwMove+"@uk", "fresh2@uk"), Op: op(hostclaim.KindLanding, hostclaim.PhaseCommitted)}
+ }, func(f *vwFixture) { f.move(provision.MoveCommitted) }, "uk"},
+ {"landing committed with no admission in the target", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: op(hostclaim.KindLanding, hostclaim.PhaseCommitted)}
+ }, func(f *vwFixture) { f.move(provision.MoveCommitted) }, ""},
+ {"landing closed record first (M4)", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(vwMove, "cur@ie", vwMove+"@uk"), Op: op(hostclaim.KindLanding, hostclaim.PhaseCommitted)}
+ }, func(f *vwFixture) {
+ f.rec.LastMove = &provision.MoveOutcome{ID: vwMove, Outcome: "done", At: vwCreated}
+ }, "uk"},
+ {"restoring, clean, from the return CAS", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie", vwMove+"@uk"), Op: op(hostclaim.KindRestoring, hostclaim.PhaseClaimed)}
+ }, func(f *vwFixture) { f.move(provision.MoveRollingBack) }, "ie"},
+ {"restoring, dirty, before the source's probe", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(vwMove, "cur@ie", vwMove+"@uk"), Op: op(hostclaim.KindRestoring, hostclaim.PhaseClaimed)}
+ }, func(f *vwFixture) { f.move(provision.MoveRollingBack) }, "uk"},
+ {"restoring, dirty, from the source's probe's admission", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(vwRb, "cur@ie", vwMove+"@uk", vwRb+"@ie"), Op: op(hostclaim.KindRestoring, hostclaim.PhaseProbing)}
+ }, func(f *vwFixture) { f.move(provision.MoveRollingBack) }, "ie"},
+ {"restoring before any admission of the generation", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder(""), Op: op(hostclaim.KindRestoring, hostclaim.PhaseReleasing)}
+ }, func(f *vwFixture) { f.move(provision.MoveRollingBack) }, "ie"},
+ {"moving with the generation admitted in the target", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@uk"), Op: op(hostclaim.KindMoving, hostclaim.PhaseMoving)}
+ }, func(f *vwFixture) { f.move(provision.MoveCutting) }, ""},
+ {"a move op the record does not carry", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: op(hostclaim.KindLanding, provision.LandBuilding)}
+ }, nil, ""},
+ {"placing", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindPlacing, ID: "op"}}
+ }, nil, ""},
+ {"relocating", func(f *vwFixture) *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: "relocating", ID: "op"}}
+ }, nil, ""},
+ }
+ for _, r := range rows {
+ t.Run(r.name, func(t *testing.T) {
+ f := newVwFixture(t)
+ if r.rec != nil {
+ r.rec(f)
+ }
+ obs := vwObs(r.claim(f))
+ v, err := f.view("us")(t.Context(), obs)
+ if r.want == "" {
+ if err == nil || !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("view %+v, %v; want a refusal", v, err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v.AuthoritativeRegion != r.want || v.Region != "us" || v.ClaimETag != obs.ETag {
+ t.Fatalf("view %+v; want %s authoritative, seen from us", v, r.want)
+ }
+ if v.MarkerKnown || f.lists != 0 {
+ t.Fatalf("a non-authoritative region read the marker (%d listings)", f.lists)
+ }
+ if obs.Claim.Holder != nil && v.GenerationOf == nil {
+ t.Fatal("no classifier for the holder's location")
+ }
+ })
+ }
+}
+
+// TestMoveParkViewReadsTheMarkerByVersion: in the authoritative region the
+// view reads generation.json by the version the listing names and pins the
+// snapshot it names (the installed root); absent is the origin; a marker of
+// another incarnation refuses.
+func TestMoveParkViewReadsTheMarkerByVersion(t *testing.T) {
+ serving := func(f *vwFixture) hostclaim.Observed {
+ return vwObs(&hostclaim.Claim{Holder: f.holder("cur", "cur@uk"), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "uk-box"}})
+ }
+ t.Run("present", func(t *testing.T) {
+ f := newVwFixture(t)
+ mv, snap := f.marker(t, "cur", f.inc)
+ v, err := f.view("uk")(t.Context(), serving(f))
+ if err != nil {
+ t.Fatal(err)
+ }
+ root := park.Ref{Key: f.location + "/g-cur/0009/0000000000000001-0000000000000005.ltx", VersionID: snap}
+ if !v.MarkerKnown || v.MarkerGeneration != "cur" || v.Marker != (park.Ref{Key: f.location + "/generation.json", VersionID: mv}) || !v.Pinned[root] || len(v.Pinned) != 1 {
+ t.Fatalf("view %+v", v)
+ }
+ if g, ok := v.GenerationOf(root); !ok || g != "cur" {
+ t.Fatalf("the root classifies as %q %v", g, ok)
+ }
+ })
+ t.Run("absent is the origin", func(t *testing.T) {
+ f := newVwFixture(t)
+ v, err := f.view("uk")(t.Context(), serving(f))
+ if err != nil || !v.MarkerKnown || v.MarkerGeneration != "" || v.Marker != (park.Ref{}) || len(v.Pinned) != 0 {
+ t.Fatalf("view %+v, %v", v, err)
+ }
+ })
+ t.Run("another incarnation's marker", func(t *testing.T) {
+ f := newVwFixture(t)
+ f.marker(t, "cur", strings.Repeat("e", 64))
+ if v, err := f.view("uk")(t.Context(), serving(f)); !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("view %+v, %v", v, err)
+ }
+ })
+ t.Run("a deleted marker", func(t *testing.T) {
+ f := newVwFixture(t)
+ f.marker(t, "cur", f.inc)
+ f.bucket.BareDelete("data", f.location+"/generation.json")
+ if v, err := f.view("uk")(t.Context(), serving(f)); !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("view %+v, %v", v, err)
+ }
+ })
+}
+
+// TestMovesLeaseSetCoversTheScope (M6): the scope's own lock, for the four
+// scope shapes moves parks, in this box's own bucket only; acquired through
+// the real AggregateLeases it takes exactly that lock.
+func TestMovesLeaseSetCoversTheScope(t *testing.T) {
+ inc := strings.Repeat("a", 64)
+ set := movesLeaseSet("data-ie")
+ for _, scope := range []string{"instances/" + vwHost, "instances/" + vwHost + "/" + inc, "moves/" + vwHost, "moves/" + vwHost + "/" + vwMove} {
+ names, err := set(t.Context(), "data-ie", scope)
+ if err != nil || len(names) != 1 || names[0] != scope {
+ t.Fatalf("%s: %v %v", scope, names, err)
+ }
+ leases := &fakeParkLeases{}
+ held, err := hostbox.AggregateLeases{Leases: leases, Resolve: set}.Acquire(t.Context(), "data-ie", scope)
+ if err != nil {
+ t.Fatalf("%s: %v", scope, err)
+ }
+ if got := leases.holding(); len(got) != 1 || got[0] != "data-ie/"+scope {
+ t.Fatalf("%s holds %v", scope, got)
+ }
+ held.Close()
+ }
+ if _, err := set(t.Context(), "data-uk", "instances/"+vwHost); !errors.Is(err, hostbox.ErrChanged) {
+ t.Fatalf("another bucket: %v", err)
+ }
+ for _, scope := range []string{"", "instances", "instances/" + vwHost + "/g-cur", "instances/" + vwHost + "/" + strings.ToUpper(inc), "moves/" + vwHost + "/x/y",
+ "moves/" + vwHost + "/not an op", "parked/" + vwHost, "instances/../" + vwHost} {
+ if names, err := set(t.Context(), "data-ie", scope); !errors.Is(err, park.ErrUnsafe) {
+ t.Fatalf("%q answered %v %v", scope, names, err)
+ }
+ }
+}
+
+// fakePlan12 stands in for claim Plan 12's pieces in the arming tests.
+type fakePlan12 struct{ runs, locals int }
+
+func (p *fakePlan12) RunHeld(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error {
+ p.runs++
+ return testRunHeld(ctx, e, w, held)
+}
+func (p *fakePlan12) AuthorizeLocal(context.Context, *hostclaim.Claim, hostclaim.Obligation) error {
+ p.locals++
+ return nil
+}
+
+// TestArmMovesPassRefusesAMismatchedBucketOrRegion (M17): the arms take the
+// claim's engine only when its bucket and region are this box's own, and
+// only with claim Plan 12's pieces; any refusal sets none of the hooks.
+func TestArmMovesPassRefusesAMismatchedBucketOrRegion(t *testing.T) {
+ view := func(context.Context, hostclaim.Observed) (park.View, error) { return park.View{}, nil }
+ auth := func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }
+ providers := func(bucket, region string) ClaimProviders {
+ return ClaimProviders{moveParks: &claimMoveParks{
+ engine: park.Engine{Bucket: bucket, Region: region, Leases: &fakeParkLeases{}, View: view, Authorize: auth},
+ leases: movesLeaseSet(bucket), check: func(context.Context) error { return nil }}}
+ }
+ plan12 := &fakePlan12{}
+ ok := func() (movesClaimPlan12, error) { return plan12, nil }
+ unarmed := func(t *testing.T, c moveCfg) {
+ t.Helper()
+ if c.ParkLeases != nil || c.ParkView != nil || c.ParkAuthorize != nil || c.ParkRunHeld != nil || c.ParkLocalAuthorize != nil {
+ t.Fatal("a refused arming set a hook")
+ }
+ }
+ for name, p := range map[string]ClaimProviders{"another bucket": providers("data-uk", "ie"), "another region": providers("data-ie", "uk")} {
+ t.Run(name, func(t *testing.T) {
+ c := moveCfg{OwnBucket: "data-ie", Region: "ie"}
+ if err := armMovesParks(t.Context(), &c, p, ok); !errors.Is(err, hostbox.ErrChanged) {
+ t.Fatalf("armed: %v", err)
+ }
+ unarmed(t, c)
+ })
+ }
+ t.Run("no MoveParks", func(t *testing.T) {
+ c := moveCfg{OwnBucket: "data-ie", Region: "ie"}
+ if err := armMovesParks(t.Context(), &c, ClaimProviders{}, ok); !errors.Is(err, hostwrite.ErrUnavailable) {
+ t.Fatalf("armed: %v", err)
+ }
+ unarmed(t, c)
+ })
+ t.Run("without claim Plan 12", func(t *testing.T) {
+ c := moveCfg{OwnBucket: "data-ie", Region: "ie"}
+ err := armMovesParks(t.Context(), &c, providers("data-ie", "ie"), movesPlan12Parks)
+ var dep *ClaimDependencyError
+ if !errors.As(err, &dep) {
+ t.Fatalf("armed: %v", err)
+ }
+ for _, piece := range []string{"RunHeld", "lease-set contract", "local authorizer"} {
+ if !strings.Contains(err.Error(), piece) || !strings.Contains(err.Error(), "claim Plan 12") {
+ t.Fatalf("the refusal does not name %s: %v", piece, err)
+ }
+ }
+ unarmed(t, c)
+ })
+ t.Run("this box's own engine", func(t *testing.T) {
+ c := moveCfg{OwnBucket: "data-ie", Region: "ie"}
+ if err := armMovesParks(t.Context(), &c, providers("data-ie", "ie"), ok); err != nil {
+ t.Fatal(err)
+ }
+ if c.ParkLeases == nil || c.ParkView == nil || c.ParkAuthorize == nil || c.ParkRunHeld == nil || c.ParkLocalAuthorize == nil {
+ t.Fatalf("hooks unset: %+v", c)
+ }
+ if err := c.ParkLocalAuthorize(t.Context(), nil, hostclaim.Obligation{}); err != nil || plan12.locals != 1 {
+ t.Fatal("the local authorizer is not claim Plan 12's")
+ }
+ })
+}
+
+// TestMovesParksRefuseUntilClaimPlan12: the registered view and lease set
+// refuse by name while claim Plan 12 is missing, so the claim's regional
+// worker queues and runs nothing through them; Task 5's evidence is
+// registered as it is, reachable as the sweep verifier.
+func TestMovesParksRefuseUntilClaimPlan12(t *testing.T) {
+ f := newVwFixture(t)
+ evidence := movesRetirementEvidence{Records: f.records(), Now: time.Now}
+ src, err := newMovesParkSources(f.records(), f.bucket, "data", fixedRegion("ie"), evidence, movesPlan12Parks)
+ if err != nil {
+ t.Fatal(err)
+ }
+ obs := vwObs(&hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}})
+ var dep *ClaimDependencyError
+ if _, err := src.View(t.Context(), obs); !errors.As(err, &dep) || !strings.Contains(err.Error(), "claim Plan 12") {
+ t.Fatalf("the view answered without claim Plan 12: %v", err)
+ }
+ if _, err := src.Leases(t.Context(), "data", "instances/"+vwHost); !errors.As(err, &dep) || !strings.Contains(err.Error(), "claim Plan 12") {
+ t.Fatalf("the lease set answered without claim Plan 12: %v", err)
+ }
+ if f.reads != 0 || f.lists != 0 {
+ t.Fatal("a refused view read the record or the bucket")
+ }
+ if _, ok := src.Evidence.(park.MoveSweepVerifier); !ok {
+ t.Fatal("the registered evidence is not the sweep verifier")
+ }
+ // With claim Plan 12 the same sources answer.
+ src, err = newMovesParkSources(f.records(), f.bucket, "data", fixedRegion("ie"), evidence, func() (movesClaimPlan12, error) { return &fakePlan12{}, nil })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v, err := src.View(t.Context(), obs); err != nil || v.AuthoritativeRegion != "ie" {
+ t.Fatalf("view %+v, %v", v, err)
+ }
+ if names, err := src.Leases(t.Context(), "data", "instances/"+vwHost); err != nil || len(names) != 1 {
+ t.Fatalf("leases %v, %v", names, err)
+ }
+ // Registration is the claim's one-shot slot.
+ claimMoveSources.Lock()
+ oldSource, oldParks := claimMoveSources.Source, claimMoveSources.Parks
+ claimMoveSources.Source, claimMoveSources.Parks = nil, nil
+ claimMoveSources.Unlock()
+ t.Cleanup(func() {
+ claimMoveSources.Lock()
+ claimMoveSources.Source, claimMoveSources.Parks = oldSource, oldParks
+ claimMoveSources.Unlock()
+ })
+ if err := registerMovesParks(src); err != nil {
+ t.Fatal(err)
+ }
+ if p := claimMoveSources.Parks; p == nil || p.View == nil || p.Leases == nil || p.Evidence == nil {
+ t.Fatalf("registered %+v", p)
+ }
+ if err := registerMovesParks(src); !errors.Is(err, hostwrite.ErrUnavailable) {
+ t.Fatalf("a second registration: %v", err)
+ }
+}
+
+// testRunHeld is the agreed shape of claim Plan 12's Engine.RunHeld (M5):
+// the chore runs under the lease the caller holds, which must be the
+// scope's own lock (the lease set, M6); it is neither acquired again, nor
+// released, nor closed by the run.
+func testRunHeld(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error {
+ if held == nil {
+ return park.ErrUnsafe
+ }
+ e.Leases = borrowedLeases{held}
+ return e.Run(ctx, w)
+}
+
+type borrowedLeases struct{ held park.HeldLease }
+
+func (b borrowedLeases) Acquire(_ context.Context, bucket, scope string) (park.HeldLease, error) {
+ if f, ok := b.held.(*fakeHeldLease); ok && f.key != bucket+"/"+scope {
+ return nil, errors.New("the held lease is " + f.key + ", not " + bucket + "/" + scope)
+ }
+ return borrowedLease{b.held}, nil
+}
+
+type borrowedLease struct{ park.HeldLease }
+
+func (borrowedLease) Release(context.Context) error { return nil }
+func (borrowedLease) Close() {}
+
+// TestCommittedLaneViewNamesTheTargetWithItsMarker (the Task 3 carry): after
+// the commit the target's lane park runs under the real view, which names
+// the target's region authoritative and supplies the marker the target's
+// probe published, read by its exact version; the lane is parked.
+func TestCommittedLaneViewNamesTheTargetWithItsMarker(t *testing.T) {
+ h := newCloseHarness(t)
+ ctx := context.Background()
+ location := strings.TrimSuffix(h.claim(t).Holder.Location, "/") // the cut's holder, at the bare location
+ body, err := json.Marshal(map[string]any{"op": p1ID, "incarnation": h.inc, "host": p1Host, "box": "uk-box", "installed_at": h.clock.Now(),
+ "manifest_sha256": "x", "snapshot_txid": 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ markerVersion, err := h.vs.PutIfAbsent(ctx, "data-uk", location+"/generation.json", body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ real := movesParkView(MovesRecordsFrom(h.s), h.vs, "data-uk", "uk")
+ var committed []park.View
+ h.parkView = func(ctx context.Context, obs hostclaim.Observed) (park.View, error) {
+ v, err := real(ctx, obs)
+ if err == nil && obs.Claim.Op.Kind == hostclaim.KindLanding && obs.Claim.Op.Phase == hostclaim.PhaseCommitted {
+ committed = append(committed, v)
+ }
+ return v, err
+ }
+ h.land(t)
+ h.committedAndOpen(t)
+ if len(committed) == 0 {
+ t.Fatal("the committed lane park never asked the view")
+ }
+ for _, v := range committed {
+ if v.Region != "uk" || v.AuthoritativeRegion != "uk" || !v.MarkerKnown || v.MarkerGeneration != p1ID ||
+ v.Marker != (park.Ref{Key: location + "/generation.json", VersionID: markerVersion}) {
+ t.Fatalf("the committed view %+v; want uk authoritative with the marker at %s", v, markerVersion)
+ }
+ }
+ lane := moves.ScopeLocation(p1Host, p1ID) + "/"
+ for k := range h.f.buckets["data-uk"] {
+ if strings.HasPrefix(k, lane) {
+ t.Fatalf("the lane still holds %s after the committed park", k)
+ }
+ }
+}
+
+// TestRegisterProductionMovesParksNeedsBothSwitches (ruling H1): with either
+// switch off the agent registers nothing. With both on it registers without
+// CARLOS_REGION: the view's region is the claim's verified roster row's
+// (fix round 1, region source), resolved when the view is first asked.
+func TestRegisterProductionMovesParksNeedsBothSwitches(t *testing.T) {
+ claimMoveSources.Lock()
+ oldSource, oldParks := claimMoveSources.Source, claimMoveSources.Parks
+ claimMoveSources.Source, claimMoveSources.Parks = nil, nil
+ claimMoveSources.Unlock()
+ t.Cleanup(func() {
+ claimMoveSources.Lock()
+ claimMoveSources.Source, claimMoveSources.Parks = oldSource, oldParks
+ claimMoveSources.Unlock()
+ })
+ env := func(vals map[string]string) func(string) string { return func(k string) string { return vals[k] } }
+ hib := activator.Config{Bucket: "data-ie", Region: "eu-west-1"}
+ store := release.NewDirStore(t.TempDir())
+ asked := 0
+ roster := func(context.Context) (string, error) { asked++; return "ie", nil }
+ for name, vals := range map[string]map[string]string{
+ "claims off": {moves.SwitchEnv: "on"},
+ "moves off": {"CARLOS_HOST_CLAIM": "1", moves.SwitchEnv: "off"},
+ } {
+ if err := registerProductionMovesParks(t.Context(), env(vals), store, hib, roster, newMovesRefusals("ie-box", nil, t.Logf, time.Now)); err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ if claimMoveSources.Parks != nil {
+ t.Fatalf("%s: registered", name)
+ }
+ }
+ if err := registerProductionMovesParks(t.Context(), env(map[string]string{"CARLOS_HOST_CLAIM": "1", moves.SwitchEnv: "on"}), store, hib, roster, newMovesRefusals("ie-box", nil, t.Logf, time.Now)); err != nil {
+ t.Fatalf("both switches on, no CARLOS_REGION: %v", err)
+ }
+ if claimMoveSources.Parks == nil || asked != 0 {
+ t.Fatalf("registered %v, roster asked %d times at registration", claimMoveSources.Parks != nil, asked)
+ }
+}
+
+// TestHomeBoxArms (fix round 1, region source): the home region, "", is a
+// region like any other. A home box's view is seen from "" and its arms take
+// an engine whose region is "".
+func TestHomeBoxArms(t *testing.T) {
+ f := newVwFixture(t)
+ f.rec.Region = ""
+ view := movesParkViewAt(f.records(), f.bucket, "data", fixedRegion(""))
+ obs := vwObs(&hostclaim.Claim{Holder: f.holder(""), Op: hostclaim.Op{Kind: hostclaim.KindMoving, ID: vwMove, Phase: hostclaim.PhaseMoving, Exec: "box"}})
+ f.rec.Move = &provision.MoveState{ID: vwMove, Incarnation: f.inc, From: "", To: "uk", State: provision.MoveCutting}
+ v, err := view(t.Context(), obs)
+ if err != nil || v.Region != "" || v.AuthoritativeRegion != "" || !v.MarkerKnown {
+ t.Fatalf("the home view %+v, %v", v, err)
+ }
+ auth := func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }
+ providers := ClaimProviders{moveParks: &claimMoveParks{
+ engine: park.Engine{Bucket: "data-home", Region: "", Leases: &fakeParkLeases{}, View: view, Authorize: auth},
+ leases: movesLeaseSet("data-home"), check: func(context.Context) error { return nil }}}
+ c := moveCfg{OwnBucket: "data-home", Region: ""}
+ if err := armMovesParks(t.Context(), &c, providers, func() (movesClaimPlan12, error) { return &fakePlan12{}, nil }); err != nil {
+ t.Fatalf("a home box did not arm: %v", err)
+ }
+ if !parkHooksArmed(c) {
+ t.Fatal("hooks unset")
+ }
+}
+
+// TestParkViewRegionIsTheRosters (fix round 1, region source): the view
+// asks the roster resolver once, caches a success, and a failure refuses
+// that call and asks again on the next.
+func TestParkViewRegionIsTheRosters(t *testing.T) {
+ f := newVwFixture(t)
+ calls, fail := 0, true
+ resolve := cachedRegion(func(context.Context) (string, error) {
+ calls++
+ if fail {
+ return "", hostbox.ErrChanged
+ }
+ return "uk", nil
+ })
+ view := movesParkViewAt(f.records(), f.bucket, "data", resolve)
+ obs := vwObs(&hostclaim.Claim{Holder: f.holder("cur", "cur@ie"), Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box"}})
+ if _, err := view(t.Context(), obs); !errors.Is(err, hostbox.ErrChanged) {
+ t.Fatalf("a roster failure: %v", err)
+ }
+ fail = false
+ for i := 0; i < 3; i++ {
+ if v, err := view(t.Context(), obs); err != nil || v.Region != "uk" {
+ t.Fatalf("view %+v, %v", v, err)
+ }
+ }
+ if calls != 2 {
+ t.Fatalf("the roster was asked %d times, want 2", calls)
+ }
+}
+
+// TestLandingResidueParkUnderItsOwnLease (M5): the landing takes its lease on
+// the location at checking and holds it through published (spec §7.6). Over
+// the real AggregateLeases and the moves lease set, the residue park's
+// scope resolves to that same lock, so Engine.Run would wait on the landing
+// itself; claim Plan 12's RunHeld runs the chores under the held lease and
+// the landing goes on to restored.
+func TestLandingResidueParkUnderItsOwnLease(t *testing.T) {
+ location := "data-uk/" + cutRoot
+ t.Run("under RunHeld", func(t *testing.T) {
+ h := newLandingHarness(t)
+ h.flip(t)
+ keys := h.residue(t)
+ h.parkLeases = hostbox.AggregateLeases{Leases: h.leases, Resolve: movesLeaseSet("data-uk")}
+ runs := 0
+ h.runHeld = func(ctx context.Context, e park.Engine, w park.Work, held park.HeldLease) error {
+ runs++
+ if got := h.leases.holding(); len(got) != 1 || got[0] != location {
+ t.Errorf("a residue park ran while the landing held %v, want %s", got, location)
+ }
+ return testRunHeld(ctx, e, w, held)
+ }
+ h.land(t)
+ h.installed(t)
+ if runs == 0 {
+ t.Fatal("no residue park ran")
+ }
+ for _, k := range keys {
+ if _, ok := h.f.buckets["data-uk"][k]; ok && !strings.HasSuffix(k, "/generation.json") {
+ t.Fatalf("%s is still at the location", k)
+ }
+ }
+ })
+ t.Run("Engine.Run waits on the landing's own lease", func(t *testing.T) {
+ h := newLandingHarness(t)
+ h.flip(t)
+ keys := h.residue(t)
+ h.parkLeases = hostbox.AggregateLeases{Leases: h.leases, Resolve: movesLeaseSet("data-uk")}
+ h.runHeld = func(ctx context.Context, e park.Engine, w park.Work, _ park.HeldLease) error { return e.Run(ctx, w) }
+ for i := 0; i < 3; i++ {
+ h.land(t)
+ }
+ if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandParking {
+ t.Fatalf("the landing went past parking without RunHeld: %+v", m)
+ }
+ if got := h.leases.holding(); len(got) != 1 || got[0] != location {
+ t.Fatalf("the landing holds %v", got)
+ }
+ for _, k := range keys {
+ if _, ok := h.f.buckets["data-uk"][k]; !ok {
+ t.Fatalf("%s was parked under a lease nobody held", k)
+ }
+ }
+ })
+}
+
+// TestNoArmRunsWithoutTheThreeParkHooks (M17): every moves arm requires the
+// claim's park leases, view and authorizer, the landing (and the recovery's
+// install) claim Plan 12's RunHeld too, the finalization its local
+// authorizer; production's own wiring sets none of them.
+func TestNoArmRunsWithoutTheThreeParkHooks(t *testing.T) {
+ three := map[string]func(*moveCfg){
+ "no park leases": func(c *moveCfg) { c.ParkLeases = nil },
+ "no park view": func(c *moveCfg) { c.ParkView = nil },
+ "no park authorizer": func(c *moveCfg) { c.ParkAuthorize = nil },
+ }
+ runHeld := func(c *moveCfg) { c.ParkRunHeld = nil }
+ local := func(c *moveCfg) { c.ParkLocalAuthorize = nil }
+ arms := []struct {
+ name string
+ cfg func(t *testing.T) moveCfg
+ armed func(moveCfg) bool
+ extra map[string]func(*moveCfg)
+ }{
+ {"landing", func(t *testing.T) moveCfg { return newLandingHarness(t).lcfg }, landingArmed, map[string]func(*moveCfg){"no claim Plan 12 RunHeld": runHeld}},
+ {"rollback target", func(t *testing.T) moveCfg { h := newRollbackHarness(t); return h.rollbackCfg(t) }, rollbackTargetArmed, nil},
+ {"lane park (revert, abort, committed)", func(t *testing.T) moveCfg { return newRevertHarness(t).laneCfg() }, revertLaneArmed, nil},
+ {"recovery", func(t *testing.T) moveCfg { h := newRecoverHarness(t, true); return h.sourceCfg(t) }, recoverySourceArmed, map[string]func(*moveCfg){"no claim Plan 12 RunHeld": runHeld}},
+ {"source finalization", func(t *testing.T) moveCfg { return newFinalHarness(t).cfg }, sourceFinalArmed, map[string]func(*moveCfg){"no claim Plan 12 local authorizer": local}},
+ }
+ for _, a := range arms {
+ t.Run(a.name, func(t *testing.T) {
+ c := a.cfg(t)
+ if !a.armed(c) {
+ t.Fatal("fixture: the arm is not armed with every hook")
+ }
+ for name, unarm := range three {
+ u := c
+ unarm(&u)
+ if a.armed(u) {
+ t.Fatalf("%s: armed", name)
+ }
+ }
+ for name, unarm := range a.extra {
+ u := c
+ unarm(&u)
+ if a.armed(u) {
+ t.Fatalf("%s: armed", name)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/park/live_generations_test.go b/internal/park/live_generations_test.go
new file mode 100644
index 000000000..055725b8c
--- /dev/null
+++ b/internal/park/live_generations_test.go
@@ -0,0 +1,39 @@
+package park
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+)
+
+// TestLiveGenerationsWalksTheUnfinishedChain: the exported live set is the
+// engine's own: with the marker lagging at g1 and the holder at g3, the
+// unfinished cold chain g3 -> g2 -> g1 in the authoritative region is live;
+// a region that is not authoritative has none; a missing claim refuses.
+func TestLiveGenerationsWalksTheUnfinishedChain(t *testing.T) {
+ loc := "instances/h.example"
+ c := &hostclaim.Claim{Host: "h.example", Op: hostclaim.Op{Kind: hostclaim.KindServing}, Holder: &hostclaim.Holder{Location: loc, Generation: "g3",
+ Admissions: []hostclaim.AdmissionRecord{
+ {Generation: "g1", Mode: "cold", Region: "ie"},
+ {Generation: "g2", Predecessor: "g1", Mode: "cold", Region: "ie", PredecessorRegion: "ie"},
+ {Generation: "g3", Predecessor: "g2", Mode: "cold", Region: "ie", PredecessorRegion: "ie"},
+ }}}
+ v := View{Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: "g1", GenerationOf: func(Ref) (string, bool) { return "", true }}
+ live, err := LiveGenerations(c, v, loc)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, g := range []string{"g1", "g2", "g3"} {
+ if !live[g] {
+ t.Fatalf("live %v, missing %s", live, g)
+ }
+ }
+ v.Region = "uk"
+ if live, err := LiveGenerations(c, v, loc); err != nil || len(live) != 0 {
+ t.Fatalf("a non-authoritative region: %v %v", live, err)
+ }
+ if _, err := LiveGenerations(nil, v, loc); !errors.Is(err, ErrUnsafe) {
+ t.Fatalf("no claim: %v", err)
+ }
+}
diff --git a/internal/park/sweep.go b/internal/park/sweep.go
index 82097b102..92ceb109b 100644
--- a/internal/park/sweep.go
+++ b/internal/park/sweep.go
@@ -106,6 +106,20 @@ func protected(c *hostclaim.Claim, v View, location string) (map[string]bool, er
return nil, ErrUnsafe
}
+// LiveGenerations is the region-qualified live generation set the engine
+// protects at location: the holder's generation, the marker's, every
+// generation the pinned versions belong to, and the unfinished cold
+// admission chain walked back to the marker in the authoritative region.
+// It is protected itself, exported so a view that needs the live set (the
+// moves files live set) enumerates the same generations rather than a
+// parallel walk. An error is protected's refusal.
+func LiveGenerations(c *hostclaim.Claim, v View, location string) (map[string]bool, error) {
+ if c == nil {
+ return nil, ErrUnsafe
+ }
+ return protected(c, v, location)
+}
+
// Decide uses a complete namespace version listing supplied by Discover.
// LegacyIncarnation is a caller-supplied, evidenced archive label only. It is
// never used to discover locations, and its absence refuses legacy cleanup.
diff --git a/internal/provision/move_test.go b/internal/provision/move_test.go
index e7cdd0aa0..4f0ddd1ff 100644
--- a/internal/provision/move_test.go
+++ b/internal/provision/move_test.go
@@ -1415,6 +1415,50 @@ func TestStuckReturnTargetReasons(t *testing.T) {
}
}
+// TestStuckReleaseTargetFiles (Plan 5 Task 9a): the target's release finds
+// files it can never archive. The reason belongs to the target's releasing,
+// with the source still waiting, dirty decided or not; the target phase
+// stays releasing, the source's waiting is StuckAt, and a retry runs the
+// release again. Anywhere else it is a conflict.
+func TestStuckReleaseTargetFiles(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ at := func(target string, dirty bool) *Record {
+ rec := rollingBackRecord(t, g, now)
+ rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = target, dirty
+ return rec
+ }
+ for _, dirty := range []bool{false, true} {
+ rec := at(RbTargetReleasing, dirty)
+ if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); err != nil {
+ t.Fatalf("dirty %v: %v", dirty, err)
+ }
+ m, rb := rec.Move, rec.Move.Rollback
+ if m.State != MoveStuck || m.StuckReason != StuckReleaseTargetFiles || rb.StuckAt != RbSourceWaiting || rb.TargetPhase != RbTargetReleasing {
+ t.Fatalf("dirty %v: %+v %+v", dirty, m, rb)
+ }
+ if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); err != nil {
+ t.Fatalf("replay: %v", err)
+ }
+ if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != RbSourceWaiting || rb.TargetPhase != RbTargetReleasing || m.StuckReason != "" {
+ t.Fatalf("retry: %v %+v %+v", err, m, rb)
+ }
+ }
+ for _, target := range []string{RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning} {
+ if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(at(target, true)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("at %s: %v", target, err)
+ }
+ }
+ rec := at(RbTargetReleasing, false)
+ rec.Move.Rollback.SourcePhase = RbSourceClaimed
+ if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("with the source claimed: %v", err)
+ }
+ if StuckReleaseTargetFiles == StuckReturnTargetFiles {
+ t.Fatal("the release's reason must differ from the return's")
+ }
+}
+
func TestRolledBackCloseNeedsAllThreeStamps(t *testing.T) {
g := cutTestGuard()
now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index 7b38a2d85..e0af6c4e4 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -519,6 +519,12 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/host/movesevidence.go", "movesInstancesLocation", "key-site", "lit:instances", "4f79083f445c5892fb4933a696bc322eb081d0dd0c0b53c2134e8544c5a9df3e"},
// Reads the record through MovesRecordReader (GetRecord); RecordKey only checks the claim holder's key, and the literals only compare data-bucket locations. No write.
{"internal/host/movesevidence.go", "movesRetirementEvidence.retired", "key-site", "key:RecordKey,lit:instances", "d6a42cfd64728e581bb123d5743d3c74b1b6eb5abddb8972571f15cb42d84cb6"},
+ // The moves park view's record read (Plan 5 Task 9a): reads through MovesRecordReader (GetRecord); RecordKey only checks the claim holder's key. No write.
+ {"internal/host/movessources.go", "holderRecord", "key-site", "key:RecordKey", "84e840552703cd9863c9b27786baa864109e94d15b9e6f3e035c0da18622eb2e"},
+ // The moves lease set's scope shapes (Plan 5 Task 9a, M6): instances/<host>[/<incarnation>] and moves/<host>[/<op>] are data-bucket lock prefixes; a pure string check.
+ {"internal/host/movessources.go", "movesLeaseScope", "key-site", "lit:instances", "7a5b9e07b4bb754c9edf0bb5c3e828f79262d195d764e6d51608518e86ed6183"},
+ // The park view's read-only store over one manifest version (Plan 5 Task 9a, M10): Put refuses and writes nothing.
+ {"internal/host/movessources.go", "versionStore.Put", "nonrecord", "def:Put", "a540f7fd383ea728f93a9eacb0d03ec1b8dcefcf20ade117831d0dd28faad2da"},
// Copies config acknowledgements to moves/<account>/<app>/<host>/<op>/config-ack-source.json.
{"internal/host/movesfinal.go", "sourceFinal.configAck", "nonrecord", "sel:Put", "cca53109e6945a73b7f04705d8965b3b0997f2b7d2a82fa6d86cffa4b4174e93"},
// Stamps source_parked_at through SetRecordMove, updateRecord and WriteRecord.
@@ -551,8 +557,8 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/host/movesrollback.go", "rollbackRun.laneMoveJSON", "nonrecord", "sel:PutIfAbsentVersioned", "5fd541f4a1daf2600ff266583fc9627cc0cab35e276f1d9c8f097f128c2c4e53"},
// Advances the target rollback phase through SetRecordMove and its sequenced writer.
{"internal/host/movesrollback.go", "rollbackRun.phase", "record-writer", "key:SetRecordMove", "76f1efb1076029e5da4c0cc0d750fbac5e1c94a6014c2b1e4af241797350827b"},
- // Marks the return stuck through SetRecordMove; event and serial writes are not record objects.
- {"internal/host/movesrollback.go", "rollbackRun.stuck", "record-writer", "key:SetRecordMove", "84fcea399a55d77ceee17e35652ef269e5811453ffff62dff43208b68538b1ed"},
+ // Marks the return stuck through SetRecordMove; event and serial writes are not record objects. Plan 5 Task 9a: the release's archive refusal stops here too; only the log line changed (the target's half, not the return), the write is the same MoveMarkStuck.
+ {"internal/host/movesrollback.go", "rollbackRun.stuck", "record-writer", "key:SetRecordMove", "fc699dcb2fc95356e6721be4ca849f2bd361bbef5f639ce54eac5fc92de2a6d6"},
// Validates the pointer-digest marker key and Put/Delete method names without writing.
{"internal/hostops/mutation_classifier.go", "classifyPointerDigestMarker", "reference", "str:Put", "08111b5db5d98e788d1eef5861ad9c861cf926c5e1d5d54717fd650d2dcf9fa8"},
// Validates the promotion-record key (releases/<acct>/<app>/promoted/<version>, a release object, not a
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index 94542d08e..fdbe6cd23 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -1865,6 +1865,13 @@ const (
// archived (a special file, over the cap, too many entries, a path too
// long).
StuckReturnTargetFiles = "the target's files cannot be archived"
+ // StuckReleaseTargetFiles (Plan 5 Task 9a): the target that served
+ // stopped, and its files directory cannot be archived when it is
+ // digested for its stopped report (a special file, over the cap, too
+ // many entries, a path too long). A rollback whose target is at
+ // releasing, the source still waiting; the target keeps its phase and
+ // everything it holds, and a retry runs the release again.
+ StuckReleaseTargetFiles = "the target's files cannot be archived now that it has stopped"
)
// The refusals of this section that reach an operator through a door
@@ -2107,6 +2114,9 @@ func MoveRecordRecoveryInputs(g MoveGuard, rbID string, in MoveInputs) func(*Rec
// StuckReturnTargetFiles: a dirty rollback whose target is at returning
// (the source still waiting for it); the target phase is kept, so a
// retry runs the return again;
+// - StuckReleaseTargetFiles: a rollback whose target is at releasing,
+// the source still waiting (Plan 5 Task 9a); the target phase is kept,
+// so a retry runs the release again;
// - StuckRevertProbeBudget: a revert;
// - StuckRecoveryProbeAttempts: a revert too, whose probe has the recovery
// probe's nine attempts as well as its fifteen minutes (§7.7; Plan 4
@@ -2143,6 +2153,8 @@ func MoveMarkStuck(g MoveGuard, reason string) func(*Record) error {
why = StuckReturnTargetChanged
case StuckReturnTargetFiles:
why = StuckReturnTargetFiles
+ case StuckReleaseTargetFiles:
+ why = StuckReleaseTargetFiles
default:
return fmt.Errorf("provision: stuck reason %q has no recorded sentence", reason)
}
@@ -2164,6 +2176,8 @@ func MoveMarkStuck(g MoveGuard, reason string) func(*Record) error {
ok = rb.Dirty && rb.SourcePhase == RbSourceConfigured
case StuckReturnTargetDatabase, StuckReturnTargetChanged, StuckReturnTargetFiles:
ok = rb.Dirty && rb.TargetPhase == RbTargetReturning && rb.SourcePhase == RbSourceWaiting
+ case StuckReleaseTargetFiles:
+ ok = rb.TargetPhase == RbTargetReleasing && rb.SourcePhase == RbSourceWaiting
}
}
case MoveReverting: