amadan

carlos / platform Public

moves-offline-precopy: carlos move offline stage (pre-copy while serving) and parallel mirror copies

Squashed from moves-offline-precopy into main.
claude 771ca01b6be00cec299a8e265c34136daa1fe0d1 parent f550afa
37 files changed, +3513 −130
  • approved_moves_copy_test.go +4 −1
  • docs/site/cli.md +3 −0
  • docs/superpowers/notes/census-proposal-moves-offline-precopy.md +236 −0
  • docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +31 −2
  • internal/activator/backend_exec_test.go +18 −1
  • internal/console/apiinstancemove_offline.go +308 −41
  • internal/console/apiinstancemove_offline_stage_test.go +325 −0
  • internal/console/apiinstancemove_offline_test.go +7 −2
  • internal/console/authcli.go +1 −0
  • internal/console/claim_invariant_fixtures_test.go +7 −0
  • internal/console/svcroutes_test.go +1 −0
  • internal/console/testdata/claim-routes.txt +1 −0
  • internal/edgesfile/snapshot.go +3 −0
  • internal/host/edgesfile.go +3 −2
  • internal/host/movescut_test.go +2 −0
  • internal/host/movesoffline.go +15 −1
  • internal/host/movesoffline_seal.go +6 −0
  • internal/host/movesoffline_stage.go +321 −0
  • internal/host/movesoffline_stage_test.go +584 −0
  • internal/host/movesoffline_test.go +4 −4
  • internal/host/movespass_test.go +37 −9
  • internal/host/movesphase1_harness_test.go +6 −1
  • internal/lease/mirror.go +159 −21
  • internal/lease/mirror_parallel_test.go +427 −0
  • internal/lease/mirror_test.go +100 −14
  • internal/moves/offline.go +51 −5
  • internal/moves/offline_test.go +30 −0
  • internal/provision/moves_offline_precopy_census_test.go +49 −0
  • internal/provision/offline_move.go +105 −9
  • internal/provision/offline_stage_test.go +234 −0
  • internal/provision/plan14_root_reads_test.go +18 −0
  • internal/provision/plan9_b8_census_test.go +1 −1
  • internal/provision/provision.go +21 −0
  • internal/provision/testdata/plan14-root-reads-moves-offline-precopy.tsv +22 −0
  • move.go +134 −8
  • move_test.go +103 −1
  • offline_move_e2e_test.go +136 −7

CI passed — run details

CI log 
=== gate ===
cd launchctx && go vet ./... && go test ./... -count=1
ok  	amadan.net/carlos/platform.git/launchctx	0.012s
?   	amadan.net/carlos/platform.git/launchctx/controlname	[no test files]
?   	amadan.net/carlos/platform.git/launchctx/wireid	[no test files]
go vet ./...
gofmt -l .: clean
go test ./... -count=1 -timeout 30m
ok  	github.com/carlosframework/platform	109.827s
?   	github.com/carlosframework/platform/cmd/changelog	[no test files]
ok  	github.com/carlosframework/platform/cmd/console	3.066s
ok  	github.com/carlosframework/platform/cmd/moveprobe	93.325s
ok  	github.com/carlosframework/platform/cmd/zddprobe	0.713s
ok  	github.com/carlosframework/platform/internal/activator	250.834s
ok  	github.com/carlosframework/platform/internal/agentsock	0.014s
ok  	github.com/carlosframework/platform/internal/alerting	0.004s
ok  	github.com/carlosframework/platform/internal/analytics	0.019s
ok  	github.com/carlosframework/platform/internal/appconfig	2.493s
ok  	github.com/carlosframework/platform/internal/apperr	0.071s
ok  	github.com/carlosframework/platform/internal/applog	0.123s
ok  	github.com/carlosframework/platform/internal/awsinv	0.005s
ok  	github.com/carlosframework/platform/internal/boxchoice	0.005s
ok  	github.com/carlosframework/platform/internal/boxid	0.476s
?   	github.com/carlosframework/platform/internal/boxid/boxidtest	[no test files]
?   	github.com/carlosframework/platform/internal/boxid/internal/signtest	[no test files]
ok  	github.com/carlosframework/platform/internal/boxvolumes	0.008s
ok  	github.com/carlosframework/platform/internal/caps	0.013s
ok  	github.com/carlosframework/platform/internal/certwarm	0.002s
ok  	github.com/carlosframework/platform/internal/cgroup	0.701s
ok  	github.com/carlosframework/platform/internal/changelog	0.343s
ok  	github.com/carlosframework/platform/internal/claimactivation	0.822s
ok  	github.com/carlosframework/platform/internal/clamd	0.055s
?   	github.com/carlosframework/platform/internal/cliflag	[no test files]
ok  	github.com/carlosframework/platform/internal/console	445.303s
?   	github.com/carlosframework/platform/internal/controlname	[no test files]
?   	github.com/carlosframework/platform/internal/controlsock	[no test files]
ok  	github.com/carlosframework/platform/internal/dbdoor	5.745s
ok  	github.com/carlosframework/platform/internal/dbdoor/dbdoortest	0.010s
ok  	github.com/carlosframework/platform/internal/dbverify	0.004s
ok  	github.com/carlosframework/platform/internal/dnsrec	0.003s
ok  	github.com/carlosframework/platform/internal/docsite	0.006s
ok  	github.com/carlosframework/platform/internal/domains	2.396s
ok  	github.com/carlosframework/platform/internal/doorkey	0.006s
ok  	github.com/carlosframework/platform/internal/economics	0.004s
ok  	github.com/carlosframework/platform/internal/edge	45.002s
?   	github.com/carlosframework/platform/internal/edgeproof	[no test files]
ok  	github.com/carlosframework/platform/internal/edgesfile	0.008s
ok  	github.com/carlosframework/platform/internal/fleetapi	0.136s
ok  	github.com/carlosframework/platform/internal/gate	1.658s
ok  	github.com/carlosframework/platform/internal/genselect	0.015s
ok  	github.com/carlosframework/platform/internal/geo	0.014s
ok  	github.com/carlosframework/platform/internal/geoip	0.030s
ok  	github.com/carlosframework/platform/internal/historyreadback	0.154s
ok  	github.com/carlosframework/platform/internal/host	193.292s
ok  	github.com/carlosframework/platform/internal/hostactors	5.847s
ok  	github.com/carlosframework/platform/internal/hostactors/awsmembership	0.017s
ok  	github.com/carlosframework/platform/internal/hostbox	2.740s
ok  	github.com/carlosframework/platform/internal/hostclaim	0.593s
ok  	github.com/carlosframework/platform/internal/hostenv	0.016s
ok  	github.com/carlosframework/platform/internal/hostevidence	0.016s
ok  	github.com/carlosframework/platform/internal/hostexec	5.129s
ok  	github.com/carlosframework/platform/internal/hostfence	0.031s
ok  	github.com/carlosframework/platform/internal/hostfiles	10.552s
ok  	github.com/carlosframework/platform/internal/hostfloor	0.016s
ok  	github.com/carlosframework/platform/internal/hostguard	1.032s
ok  	github.com/carlosframework/platform/internal/hostops	1.641s
?   	github.com/carlosframework/platform/internal/hostops/fencerelease	[no test files]
ok  	github.com/carlosframework/platform/internal/hostops/internal/fencerelease	0.016s
ok  	github.com/carlosframework/platform/internal/hostquarantine	0.017s
ok  	github.com/carlosframework/platform/internal/hostrelocate	0.528s
ok  	github.com/carlosframework/platform/internal/hostresolve	0.146s
ok  	github.com/carlosframework/platform/internal/hostwrite	0.018s
ok  	github.com/carlosframework/platform/internal/infraview	0.019s
ok  	github.com/carlosframework/platform/internal/instancectl	5.239s
ok  	github.com/carlosframework/platform/internal/instanceexec	2.773s
ok  	github.com/carlosframework/platform/internal/instancefs	0.020s
ok  	github.com/carlosframework/platform/internal/instancekey	0.018s
ok  	github.com/carlosframework/platform/internal/instanceupdates	0.025s
ok  	github.com/carlosframework/platform/internal/layout	1.633s
ok  	github.com/carlosframework/platform/internal/lbhealth	0.018s
ok  	github.com/carlosframework/platform/internal/lease	10.092s
ok  	github.com/carlosframework/platform/internal/lease/leasetest	0.380s
ok  	github.com/carlosframework/platform/internal/ledger	0.017s
ok  	github.com/carlosframework/platform/internal/mail	0.184s
ok  	github.com/carlosframework/platform/internal/maintenance	0.085s
ok  	github.com/carlosframework/platform/internal/maintenanceexec	5.593s
ok  	github.com/carlosframework/platform/internal/maintenanceguard	0.086s
ok  	github.com/carlosframework/platform/internal/metrics	0.005s
ok  	github.com/carlosframework/platform/internal/moves	21.171s
ok  	github.com/carlosframework/platform/internal/moves/archive	0.871s
ok  	github.com/carlosframework/platform/internal/objstore	0.215s
ok  	github.com/carlosframework/platform/internal/ops	0.170s
ok  	github.com/carlosframework/platform/internal/park	0.220s
ok  	github.com/carlosframework/platform/internal/plans	0.005s
ok  	github.com/carlosframework/platform/internal/privatefile	0.007s
ok  	github.com/carlosframework/platform/internal/provision	39.396s
ok  	github.com/carlosframework/platform/internal/publicrelease	0.008s
?   	github.com/carlosframework/platform/internal/regionname	[no test files]
ok  	github.com/carlosframework/platform/internal/registry	5.489s
ok  	github.com/carlosframework/platform/internal/relay	7.240s
ok  	github.com/carlosframework/platform/internal/release	24.980s
ok  	github.com/carlosframework/platform/internal/rollout	3.843s
ok  	github.com/carlosframework/platform/internal/rolloutwords	0.004s
ok  	github.com/carlosframework/platform/internal/rootaws	0.007s
ok  	github.com/carlosframework/platform/internal/rootparticipation	0.058s
ok  	github.com/carlosframework/platform/internal/rootstate	0.026s
ok  	github.com/carlosframework/platform/internal/runs	0.587s
ok  	github.com/carlosframework/platform/internal/s3compat	0.005s
ok  	github.com/carlosframework/platform/internal/safefile	0.004s
ok  	github.com/carlosframework/platform/internal/schedule	2.584s
ok  	github.com/carlosframework/platform/internal/seeds	0.013s
ok  	github.com/carlosframework/platform/internal/sim/hostlife	10.317s
ok  	github.com/carlosframework/platform/internal/sim/litestreamfacts	3.697s
ok  	github.com/carlosframework/platform/internal/status	0.062s
ok  	github.com/carlosframework/platform/internal/steering	0.006s
ok  	github.com/carlosframework/platform/internal/stopevidence	0.202s
ok  	github.com/carlosframework/platform/internal/svcrole	0.002s
ok  	github.com/carlosframework/platform/internal/telemetry	0.004s
ok  	github.com/carlosframework/platform/internal/termfence	0.004s
ok  	github.com/carlosframework/platform/internal/tunnelclient	0.004s
ok  	github.com/carlosframework/platform/internal/unixdial	0.003s
ok  	github.com/carlosframework/platform/internal/updatewords	0.005s
ok  	github.com/carlosframework/platform/internal/usage	0.005s
ok  	github.com/carlosframework/platform/internal/version	0.002s
ok  	github.com/carlosframework/platform/internal/vet	1.589s
=== cross ===
--- shard 0 of 5 ---
cross-compiling darwin/arm64
--- shard 1 of 5 ---
cross-compiling darwin/amd64
--- shard 2 of 5 ---
cross-compiling linux/arm64
linux/arm64 carlos binary: 49873058 bytes (ceiling 52000000)
--- shard 3 of 5 ---
cross-compiling windows/amd64
--- shard 4 of 5 ---
cross-compiling windows/arm64
=== minio-contract ===
minio contract: arm64, minio RELEASE.2025-09-07T16-13-09Z, mc RELEASE.2025-08-13T08-35-41Z
  minio: cached, digest verified
  mc: cached, digest verified
  minio healthy on 127.0.0.1:20999
Added `local` successfully.
Bucket created successfully `local/contract-bucket`.
Bucket created successfully `local/denied-bucket`.
Created policy `contract-only` successfully.
Added user `scopeduser` successfully.
Attached Policies: [contract-only]
To User: scopeduser
  buckets and scoped user ready
=== RUN   TestDeliveredContractMinio
=== RUN   TestDeliveredContractMinio/put
=== RUN   TestDeliveredContractMinio/get_round-trips_the_body
=== RUN   TestDeliveredContractMinio/list_shows_the_key
=== RUN   TestDeliveredContractMinio/head_and_list_agree_on_the_version
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict
=== RUN   TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed
=== RUN   TestDeliveredContractMinio/delete_removes_it
=== RUN   TestDeliveredContractMinio/cross-bucket_denial
--- PASS: TestDeliveredContractMinio (0.02s)
    --- PASS: TestDeliveredContractMinio/put (0.00s)
    --- PASS: TestDeliveredContractMinio/get_round-trips_the_body (0.00s)
    --- PASS: TestDeliveredContractMinio/list_shows_the_key (0.00s)
    --- PASS: TestDeliveredContractMinio/head_and_list_agree_on_the_version (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/not_by_PutObjectTagging (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_its_own_object/and_not_by_x-amz-tagging_on_upload (0.00s)
    --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route (0.01s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CreateMultipartUpload (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/not_by_CopyObject_with_a_REPLACE_tagging_directive (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/but_multipart_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_copy_itself_is_allowed (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/a_POST-policy_forged_tag_with_a_matching_version_is_still_not_a_verdict (0.00s)
        --- PASS: TestDeliveredContractMinio/the_app_cannot_tag_by_any_other_route/and_an_ordinary_tagged-with-something-else_upload_is_allowed (0.00s)
    --- PASS: TestDeliveredContractMinio/delete_removes_it (0.00s)
    --- PASS: TestDeliveredContractMinio/cross-bucket_denial (0.00s)
PASS
ok  	github.com/carlosframework/platform/internal/objstore	0.026s
=== RUN   TestT4StorageProbeMinio
2026/10/21 13:54:04 moves: T4: STORE: list: operation error S3: ListObjectsV2, https response error StatusCode: 403, RequestID: 18E08FE02158D3D9, HostID: 7c420053e0fbcd7f41653fdd92ffa2802c903d05303b1aa15adc2e8bb65d183f, api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your key and signing method.
--- PASS: TestT4StorageProbeMinio (0.01s)
PASS
ok  	github.com/carlosframework/platform/internal/host	0.022s
minio contract: PASS
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go
index 99219804e..7d15e7bba 100644
--- a/approved_moves_copy_test.go
+++ b/approved_moves_copy_test.go
@@ -96,7 +96,7 @@ var movesCopyIDs = []string{
"MV-O20", "MV-O21", "MV-O22", "MV-O23", "MV-O24", "MV-O25", "MV-O26", "MV-O27", "MV-O28", "MV-O29", "MV-O30", "MV-O31",
"MV-O32", "MV-O33",
"MV-O34", "MV-O35", "MV-O36", "MV-O37", "MV-O38", "MV-O39", "MV-O40", "MV-O41", "MV-O42", "MV-O43",
- "MV-O50", "MV-O51", "MV-O52", "MV-O53", "MV-O54", "MV-O55", "MV-O56", "MV-O57", "MV-O58", "MV-O59", "MV-O60", "MV-O61", "MV-O62", "MV-O63", "MV-O64", "MV-O65",
+ "MV-O50", "MV-O51", "MV-O52", "MV-O53", "MV-O54", "MV-O55", "MV-O57", "MV-O58", "MV-O59", "MV-O60", "MV-O61", "MV-O62", "MV-O63", "MV-O64", "MV-O65",
"MV-O66", "MV-O67", "MV-O68", "MV-O69", "MV-O70", "MV-O71", "MV-O72", "MV-O73", "MV-O74", "MV-O75", "MV-O76", "MV-O77", "MV-O78", "MV-O79", "MV-O80", "MV-O81",
"MV-O82", "MV-O83", "MV-O84", "MV-O85", "MV-O86", "MV-O87", "MV-O88", "MV-O89", "MV-O90", "MV-O91", "MV-O92", "MV-O93", "MV-O94", "MV-O95", "MV-O96", "MV-O97",
"MV-O98",
@@ -106,6 +106,8 @@ var movesCopyIDs = []string{
"MV-O158", "MV-O159", "MV-O160", "MV-O161", "MV-O162", "MV-O163", "MV-O164", "MV-O165", "MV-O166", "MV-O167", "MV-O168", "MV-O169", "MV-O170", "MV-O171",
"MV-O172", "MV-O173", "MV-O174", "MV-O175", "MV-O176", "MV-O177",
"MV-O178",
+ "MV-O179", "MV-O180", "MV-O181", "MV-O182", "MV-O183", "MV-O184", "MV-O185", "MV-O186", "MV-O187", "MV-O188", "MV-O189",
+ "MV-O190", "MV-O191", "MV-O192",
}
// copyRow is one row of the record. The File cell is the file the sentence is
@@ -1273,6 +1275,7 @@ var movesScanFiles = []string{
"internal/host/movesoffline.go",
"internal/host/movesoffline_seal.go",
"internal/host/movesoffline_land.go",
+ "internal/host/movesoffline_stage.go",
"internal/host/movesphase1.go",
"internal/host/movespolicy.go",
"internal/host/movesprobe.go",
diff --git a/docs/site/cli.md b/docs/site/cli.md
index 918eb7768..2508d98af 100644
--- a/docs/site/cli.md
+++ b/docs/site/cli.md
@@ -2415,6 +2415,7 @@ Operators who are members of the instance's account.
`carlos move offline` moves one instance with planned downtime, one step at a time. It is for operators, works with host claims and Phase 1 moves off, and never acts on its own: each verb does one step and says which verb comes next.
```sh
+carlos move offline stage --host <host> --to <region>
carlos move offline start --host <host> --to <region>
carlos move offline seal --host <host>
carlos move offline flip --host <host>
@@ -2426,6 +2427,8 @@ carlos move offline status --host <host>
The instance is offline from `start` until `open`. Until `open`, `abort` puts it back where it was with nothing lost. After `open`, moving it back is a new move.
+`stage` is optional and comes before `start`. It copies the instance's data to the region it moves to while the instance keeps serving, and each time it runs it copies only what is new. The more it has copied, the less is left to copy while the instance is offline. Until `start`, `abort` removes the staged copy and the instance is never offline.
+
## On the host
These are what systemd starts. You will rarely type them, but knowing what
diff --git a/docs/superpowers/notes/census-proposal-moves-offline-precopy.md b/docs/superpowers/notes/census-proposal-moves-offline-precopy.md
new file mode 100644
index 000000000..f61fc041e
--- /dev/null
+++ b/docs/superpowers/notes/census-proposal-moves-offline-precopy.md
@@ -0,0 +1,236 @@
+# Census: parallel mirror rounds and the offline move's stage door
+
+Status: RULED. census-ruling-moves-offline-precopy, by the coordinator as census
+owner for this branch, 2026-10-21: accepted as proposed. Every row carries
+`rev3=census-ruling-moves-offline-precopy`.
+
+The branch is `moves-offline-precopy`, from main 3d444a1b. It was ruled at
+21cd69a4 (after Astra's round 1 fixes, 692d41cf), then re-ruled at 981294cf
+(round 2, 0a6bb375) merged with main f550afa2 (see "Round 2 and the merge of
+main" below). It does two things. It runs a mirror
+round's copies and deletes side by side in `lease.MirrorPrefixKeep`. And it
+adds `carlos move offline stage`, which pre-copies the replica while the
+instance keeps serving, as a new record state, offline-staging.
+
+The ruling adds:
+
+- `internal/provision/testdata/plan14-root-reads-moves-offline-precopy.tsv`,
+ loaded last in `TestPlan14RootReadCensus`, after design-plans-and-caps, once
+ design-plans-and-caps' seal has been checked;
+- `internal/provision/moves_offline_precopy_census_test.go`, the record-write
+ (9B8) overlay on design-plans-and-caps' inventory, which
+ `TestPlan9B8RecordWriteCensus` now reads;
+- one line in `internal/console/testdata/claim-routes.txt`.
+
+No earlier overlay is edited.
+
+## The footing
+
+Every unit here runs in the box agent (`/opt/carlos/carlos agent`,
+carlos-edge.service, `User=carlos`) or the console (cmd/console serve, user
+carlos-console). None makes a root decision. So each stays accepted legacy at
+the disposition census-ruling-moves-offline gave it or its sibling: J1
+(root-route) for registry and route reads, J4 (root-input) for file reads.
+
+## Root-read census
+
+| unit | change | executing identity | disposition |
+|---|---|---|---|
+| `internal/host/movesoffline.go` `movesWorker.offlineSourceRevert` | changed legacy: calls `offlineUnstage` before it reports reverted. Read sites unchanged; `whole:Route@701` moved to `@843` (the same `rt` read, shifted by the inserted lines) | the agent, on abort, through the armed moves worker | re-frozen, J1 |
+| `internal/provision/offline_move.go` `offlineBegin` | new: OfflineStart's body moved unchanged, with the state as a parameter. Its one site, `file:Open@198`, is `rec.ActiveRun.Open()`, i.e. `RunState.Open()` (queued, picked or cancel), a method, not a file read | the console, through `SetRecordMove` | accepted legacy, J4 (as OfflineStart) |
+| `internal/provision/offline_move.go` `OfflineStart` | retired: still declared, reads nothing. It calls `offlineBegin(…, OfflineStopping)` and is behaviour-identical to main: same refusals in the same order, same stamps, and the `StagedAt` stamp fires only for staging | the console | H3 census-ruling removal (the carlos-run `getJSON` form) |
+
+### What offlineUnstage reads and writes
+
+It does nothing unless the record carries `staged_at` and this box is the
+stager the stage door pinned on the record (`move.stager`). On that one box it
+reads only the object store:
+
+- its own stager report (`OfflineReportKey`);
+- this box's receipts object.
+
+It then:
+
+- deletes `moves/<host>/` in the staging bucket the stage door fixed on the
+ record (`move.stage_bucket`), through a mirror for `move.stage_region`
+ (round 2). It refuses a bare `moves/`, a missing bucket and this box's own
+ bucket, and in those cases acknowledges nothing. `offlineSealRevert`
+ already deletes the same prefix the same way;
+- clears `move-receipts/<host>.json` in `cfg.OwnBucket`, the same bucket the
+ stage round writes them to (`tombstoneAt`);
+- writes its stager report as reverted.
+
+It reads no local file. The host comes from the record. The stored bucket and
+region come from the target box's edges file at stage time, written to the
+record by the console through `SetRecordMove`. The operator names only a
+region code, and the tenant controls none of these paths.
+
+### Envelopes re-pinned (at 21cd69a4; superseded by the round 2 table below)
+
+None of these changes adds a read site. Every changed production file was
+read for this ruling.
+
+| envelope | what changed | hash |
+|---|---|---|
+| `.` | move.go: the stage subcommand and its status and abort lines; no os, file, exec or env use | `44fc0f3e…8bdd` |
+| `internal/console` | the stage door; `offlineBeginChecks`, shared by start and stage; the stager pin; the finish gate on the stager's reverted report; the status view | `6ae90250…92d6` |
+| `internal/host` | `movesoffline_stage.go`, new: reads only the object store, and imports `os` only for `os.ErrNotExist`. `edgesfile.go` publishes `offline-stage-1`. offlineRound has a staging branch | `e9234db8…8f47` |
+| `internal/lease` | mirror.go: `forEachParallel`; copies, then deletes, side by side; the receipts book read before the workers start | `980f38c4…62a14` |
+| `internal/moves` | the stager role, the staged phase, the report's progress fields, `OfflineStager` | `33c7b507…e071` |
+| `internal/provision` | `OfflineStage` (which needs a stager), `OfflineRestage`, `OfflineStartStaged`; `MoveState.StagedAt` and `MoveState.Stager` | `8c868695…4b64` |
+
+## Record-write census (9B8)
+
+Two sites change, each keeping main's class. No site is added.
+
+| file:function | kinds | class | why |
+|---|---|---|---|
+| `internal/console/authcli.go:Server.registerCLIRoutes` | lit:instances,lit:sidecars | key-site (main's) | one route line for the stage door, beside the six offline doors; no write added. Digest `2aee959d…c459` |
+| `internal/lease/mirror.go:MirrorPrefixKeep` | sel:CopyVersioned | versioned-transport (main's) | see below. Digest `dff4d03f…fb19` |
+
+MirrorPrefixKeep plans its skips in key order first. It then runs the copies
+on up to `DefaultMirrorWorkers` (8) goroutines and, only once every copy has
+landed, the deletes. Its writes are the same as before:
+
+- Copy or CopyVersioned into the destination prefix;
+- Delete of destination keys the source no longer has;
+- receipts through the book (`set`, `drop`, `save`). Every update is under one
+ mutex, and `receiptLedger.save` is still the one Put, unchanged.
+
+After round 1, the book is read once, before the workers start, whenever the
+round has copies to make. So `set` and `drop` touch memory only, and no
+request is made under the mutex.
+
+The stager pin is a new field on `MoveState`. It is written through the typed
+`provision.SetRecordMove` (`offlineApply`), like every other move field, under
+the same one-move lock. That is not a new write site.
+
+## Claim route census
+
+`POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/stage` joins
+`claim-routes.txt`.
+
+- **Access.** The door is operator-only: `offlineGate` calls
+ `moveOperatorGate`, and any token that is not an operator's gets a 404.
+- **Claims consoles.** With host claims or Phase 1 on, it is refused with
+ MV-O20.
+- **Tests.** Its claim-refusal fixture and its `svcroutes_test.go` exemption
+ match the six sibling offline doors exactly.
+
+## Round 2 and the merge of main
+
+### The merge (d5b020ba)
+
+Main (f550afa2) added two overlays, run-lifecycle-logs-325 and
+design-plans-and-caps, and a 9B8 inventory, `plansCapsWriteCensusInventory`.
+They touch none of this branch's units. The merge keeps both of main's
+overlays and loads this branch's last, after design-plans-and-caps, whose seal
+(`4f5e0ed6…df17`) is checked first. This branch's 9B8 overlay now sits on
+`plansCapsWriteCensusInventory`.
+
+Three envelopes (`.`, internal/console, internal/host) and registerCLIRoutes
+are changed by both sides. Because this branch's overlay loads last, it pins
+them on the merged tree.
+
+### Round 2 (0a6bb375): what changed, and the census reading
+
+- **The stage door fixes the staging destination on the record:**
+ `MoveState.StageBucket` and `StageRegion`, from the one target box's edges
+ file. The door refuses a source box's own bucket (MV-O191).
+ `provision.OfflineStage` takes a `StageTarget` and refuses an empty stager
+ or bucket. These are record fields written through the typed
+ `SetRecordMove` (`offlineBegin`, `offlineApply`), so there is no new write
+ site.
+- **The stager re-reads the record** before a round, and once before its
+ delete phase (`stageDeleteGate`). `provision.GetRecord` is an object-store
+ read, not a local file, and the scan finds no root-read unit in
+ `movesoffline_stage.go`. A move that has left staging starts no round and
+ deletes nothing.
+- **The stager acknowledges it has stopped** (`offlineStagerStopped`). It
+ writes its own stager report through `writeOfflineReport`, an existing
+ nonrecord site, and is called from `offlineSourceStep` in stopping through
+ landed.
+- **offlineUnstage deletes from the stored destination:** `moves/<host>/` in
+ `move.stage_bucket`, through a mirror for `move.stage_region`. It refuses,
+ and acknowledges nothing, when no bucket is stored or the stored bucket is
+ this box's own. It reads only the object store.
+- **offlineTakeSeal** gains one refusal before its mirror: a staged move
+ seals only into the stored bucket and region. This is a comparison of
+ record and edges-file fields. Its 16 read sites are unchanged, each shifted
+ by 487 bytes. Its Puts (nonrecord, under the move's `moves/@offline/`
+ prefix) are unchanged.
+- **The console:**
+ - Start and seal refuse when the pinned stager is missing from the fleet
+ (MV-O190), and stage, start and seal refuse changed coordinates (MV-O192).
+ - Seal waits for a fresh acknowledgement from the stager (M-C10, or MV-O178
+ when it is stale).
+ - These checks read only the fleet and the report evidence the doors
+ already read.
+- **internal/edgesfile:** `PlacementView` gains `DataBucketRegion`, a JSON
+ field. No read site.
+
+### Rows and sites changed by round 2 and the merge
+
+| census | unit or envelope | ruling |
+|---|---|---|
+| Plan 14 | `internal/host/movesoffline_seal.go` `movesWorker.offlineTakeSeal` | re-frozen at J1, census-ruling-moves-offline's disposition; digest `f606873f…cdaa` |
+| Plan 14 | `movesWorker.offlineSourceRevert` | basis updated: offlineUnstage now uses the stored destination; digest and sites unchanged |
+| Plan 14 | envelopes `.`, internal/console, internal/edgesfile, internal/host, internal/provision | re-pinned on the merged tree; internal/lease and internal/moves unchanged since round 1 |
+| 9B8 | `movesWorker.offlineTakeSeal` | nonrecord (census-ruling-moves-offline's), sel:Put; digest `f606873f…cdaa` |
+| 9B8 | `Server.registerCLIRoutes` | key-site, pinned on the merged tree; digest `a0aff01b…2e87` |
+
+| envelope | hash |
+|---|---|
+| `.` | `649bb6a5…eabd` |
+| `internal/console` | `7bdd3caa…81c9` |
+| `internal/edgesfile` | `71692438…811e` |
+| `internal/host` | `d58c8e60…e219` |
+| `internal/lease` | `980f38c4…62a14` |
+| `internal/moves` | `33c7b507…e071` |
+| `internal/provision` | `210b1740…34dc` |
+
+## The seal
+
+Three accepted-legacy rows add frozen declarations (offlineSourceRevert,
+offlineBegin and offlineTakeSeal), so the legacy seal moves from
+`4f5e0ed6…df17` (main, after design-plans-and-caps) to
+`940ed1bba8d911e0f57af83ecac253c77ba5e14dad590258e0ee03ec98ccb818`. A removal
+is not in the seal. The test checks design-plans-and-caps' seal before it
+loads this overlay. Each seal was read from a temporary `t.Logf` of
+`rootReadLegacySeal(inv)`, which was then removed.
+
+Earlier seals of this overlay:
+
+- `965fda6e…c81`, on its own after scan-demand-revival;
+- `8ef09f41…1a4d`, after the merge, before round 2.
+
+## Ruling (the coordinator, 2026-10-21): census-ruling-moves-offline-precopy, ACCEPTED as proposed
+
+1. offlineSourceRevert and offlineUnstage: ACCEPTED. Object-store reads only,
+ on the pinned stager only, deleting only in the staging destination fixed
+ on the record; there is no root read of an operator- or tenant-controlled
+ path.
+2. offlineBegin: ACCEPTED (J4, as OfflineStart). OfflineStart is retired by
+ the H3 removal, since it is behaviour-identical.
+3. offlineBegin's `file:Open@198` is `rec.ActiveRun.Open()`: ACCEPTED.
+4. The envelopes: ACCEPTED, re-pinned on the merged tree (981294cf with main
+ f550afa2), internal/edgesfile included.
+5. MirrorPrefixKeep: ACCEPTED at versioned-transport. Its record writes still
+ go through the book.
+6. registerCLIRoutes: ACCEPTED at key-site.
+7. The stage route: ACCEPTED. It is operator-only, refused with MV-O20 on
+ claims consoles, and has its fixture.
+
+8. offlineTakeSeal (round 2): ACCEPTED, re-frozen at J1 and nonrecord in
+ 9B8. It adds one refusal and no read or write.
+9. The round 2 record fields and reads (`StageBucket`, `StageRegion`, the
+ record re-reads, the stop acknowledgement): ACCEPTED. The fields go through
+ `SetRecordMove`, the reads go to the object store, and the report write
+ goes through `writeOfflineReport`.
+10. The merge with main f550afa2: ACCEPTED. Both overlays are kept, this
+ branch's is loaded last, and the seal is recomputed.
+
+The first verification was at 25c0db26. It was re-verified at 21cd69a4 for the
+round 1 fixes (the stager pin, cleanup by `staged_at`, the receipts preload),
+then at 981294cf merged with main f550afa2 for the round 2 fixes (the stager
+in the seal barrier, the fixed staging destination, the record re-reads).
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
index 10640ff9c..56eaba0c4 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
@@ -543,7 +543,7 @@ Two more refusals of the console doors (Task 6), for cases MV-O20 to MV-O31 do n
| MV-O50 | internal/console/apiinstancemove_offline.go | <host> is steered or runs under a unit. offline moves cannot carry either yet | approved |
| MV-O51 | internal/console/apiinstancemove_offline.go | <box> already serves <host>. an offline move needs the region it moves to to be empty of it | approved |
-Seven more lines of `carlos move offline` (`move.go`, Task 7), for places MV-O1 to MV-O10 do not reach: what `abort` says once every box has put the instance back (the plan's runbook has the operator run `finish` next, and no row said so), the lines `status` prints for each box and for each manifest item (`<status>` is ok, waiting, failed or unchecked; `<detail>` and `<phase>` are box diagnostics, not copy; an item with no detail uses the shorter line), the usage line, and the line `status` prints for a path a source box could not park and left where it was (MV-O58; `<reason>` is the box's own reason, not copy). A box whose report is not current says so: `flip` and `open` refuse on a stale report, so `status` has to show it. MV-O56 follows MV-O57 in the table only by number.
+Seven more lines of `carlos move offline` (`move.go`, Task 7), for places MV-O1 to MV-O10 do not reach: what `abort` says once every box has put the instance back (the plan's runbook has the operator run `finish` next, and no row said so), the lines `status` prints for each box and for each manifest item (`<status>` is ok, waiting, failed or unchecked; `<detail>` and `<phase>` are box diagnostics, not copy; an item with no detail uses the shorter line), the usage line, and the line `status` prints for a path a source box could not park and left where it was (MV-O58; `<reason>` is the box's own reason, not copy). A box whose report is not current says so: `flip` and `open` refuse on a stale report, so `status` has to show it. MV-O56 was the usage line before `stage`; MV-O181 replaced it, and Luna approved its withdrawal on 2026-10-21.
| id | file | text | status |
|---|---|---|---|
@@ -552,7 +552,6 @@ Seven more lines of `carlos move offline` (`move.go`, Task 7), for places MV-O1
| MV-O54 | move.go | <box> is <phase> as the <role>. Its report is not current. | approved |
| MV-O55 | move.go | Item <id> is <status>: <detail>. | approved |
| MV-O57 | move.go | Item <id> is <status>. | approved |
-| MV-O56 | move.go | carlos move offline needs start, seal, flip, open, finish, abort or status. | approved |
| MV-O58 | move.go | <box> left <path> in place: <reason>. | approved |
| MV-O65 | move.go | <box> could not clean up <host>: <reason>. Run carlos move offline status --host <host> for each item. | approved |
@@ -688,3 +687,33 @@ A door's answer to a box report it cannot believe for its age (harness Run 9, fi
| id | file | text | status |
|---|---|---|---|
| MV-O178 | internal/console/apiinstancemove_offline.go | the report from <box> is not current. it reports again within a minute. try again then | approved |
+
+## Offline move: stage (pre-copy)
+
+`carlos move offline stage` copies the instance's replica to the region it moves to while it keeps serving, so the seal has less to copy with the instance offline (branch `moves-offline-precopy`). Paul handed the review of these rows to Luna. Luna approved MV-O179 to MV-O189 on 2026-10-21, with two rewrites applied as given (MV-O182 and MV-O183).
+
+MV-O179 and MV-O180 are console door refusals (`internal/console/apiinstancemove_offline.go`), lower case like the other door sentences. MV-O179 answers a `stage` or `start` that names another region than the one the instance is staged for (`<to>` is that region's name). MV-O180 is the stage door's own box rule: every box in both regions must publish `offline-stage-1`.
+
+MV-O181 is the usage line with `stage` added. It replaces MV-O56, whose withdrawal Luna approved with it. MV-O182 to MV-O188 are `carlos move offline` output (`move.go`): what `stage` says when it posts and when its round has run, what `status` says while the move is staging, and what `abort` says for a move that was staged and never started (nothing is moved back, only the staged copy is removed). In MV-O183, `<staged>` and `<total>` are object counts, `<staged_mb>` and `<total_mb>` are megabytes with one decimal, and `<time>` is the round's end as `2006-01-02 15:04 UTC`. In MV-O185, `<reason>` is the stager box's own diagnostic, not copy. MV-O189 is the CLI page's paragraph for `stage`.
+
+| id | file | text | status |
+|---|---|---|---|
+| MV-O179 | internal/console/apiinstancemove_offline.go | <host> is staged for <to>. abort the move to move it to another region | approved |
+| MV-O180 | internal/console/apiinstancemove_offline.go | <box> cannot stage an offline move. it runs an older release | approved |
+| MV-O181 | move.go | carlos move offline needs stage, start, seal, flip, open, finish, abort or status. | approved |
+| MV-O182 | move.go | Staging <host> in <to>. <host> keeps serving while the data is copied. | approved |
+| MV-O183 | move.go | Staged <staged> of <total> objects for <host> in <to> (<staged_mb> of <total_mb> MB) at <time>. | approved |
+| MV-O184 | move.go | <host> keeps serving. Run carlos move offline start --host <host> --to <to> when you are ready to take it offline. | approved |
+| MV-O185 | move.go | The last staging round failed: <reason>. Run carlos move offline stage --host <host> --to <to> to try again. | approved |
+| MV-O186 | move.go | Waiting for the first staging round of <host>. | approved |
+| MV-O187 | move.go | Removing the staged copy of <host> from <to>. | approved |
+| MV-O188 | move.go | The staged copy of <host> is removed. Run carlos move offline finish --host <host> to end the move. | approved |
+| MV-O189 | docs/site/cli.md | `stage` is optional and comes before `start`. It copies the instance's data to the region it moves to while the instance keeps serving, and each time it runs it copies only what is new. The more it has copied, the less is left to copy while the instance is offline. Until `start`, `abort` removes the staged copy and the instance is never offline. | approved |
+
+Three more console door refusals for a staged move (`internal/console/apiinstancemove_offline.go`, Astra round 2), approved by Luna 2026-10-21 (MV-O192 as Luna rewrote it). MV-O190 is what `start` and `seal` answer when the box that staged the instance (the stager, pinned on the record at the stage) no longer publishes an edges file in the source region: its absence is not proof that it stopped copying, so the step waits for it or the operator aborts. `<box>` is the stager, `<from>` the source region's name. MV-O191 is the stage door's refusal of a target whose data bucket is a source box's own, where a staged copy would replace the serving replica. MV-O192 is what `stage`, `start` and `seal` answer once the target publishes another data bucket or region than the one the stage fixed on the record: the staged copy is in the stored bucket and is never followed elsewhere.
+
+| id | file | text | status |
+|---|---|---|---|
+| MV-O190 | internal/console/apiinstancemove_offline.go | <box>, which staged <host>, no longer publishes an edges file in <from>. bring it back or abort the move | approved |
+| MV-O191 | internal/console/apiinstancemove_offline.go | <to> publishes the same data bucket as <from>. a staged copy there would replace the serving one | approved |
+| MV-O192 | internal/console/apiinstancemove_offline.go | <to> now publishes a different data bucket or region from what was recorded when <host> was staged. abort the move and stage it again | approved |
diff --git a/internal/activator/backend_exec_test.go b/internal/activator/backend_exec_test.go
index c3c5b9b5c..60eec7b27 100644
--- a/internal/activator/backend_exec_test.go
+++ b/internal/activator/backend_exec_test.go
@@ -288,7 +288,9 @@ func TestRestoreIgnoresMoveReceipts(t *testing.T) {
}
// fakeMirrorStore is the test double for lease.MirrorStore, bucket→key→object.
+// A mirror round copies and deletes side by side, so every method holds mu.
type fakeMirrorStore struct {
+ mu sync.Mutex
buckets map[string]map[string]lease.MirrorObject
failList bool
}
@@ -297,12 +299,19 @@ func newFakeMirrorStore() *fakeMirrorStore {
return &fakeMirrorStore{buckets: map[string]map[string]lease.MirrorObject{}}
}
func (f *fakeMirrorStore) put(bucket, key, etag string) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.putLocked(bucket, key, etag)
+}
+func (f *fakeMirrorStore) putLocked(bucket, key, etag string) {
if f.buckets[bucket] == nil {
f.buckets[bucket] = map[string]lease.MirrorObject{}
}
f.buckets[bucket][key] = lease.MirrorObject{ETag: etag}
}
func (f *fakeMirrorStore) List(_ context.Context, bucket, prefix string) (map[string]lease.MirrorObject, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
if f.failList {
return nil, context.DeadlineExceeded
}
@@ -315,6 +324,8 @@ func (f *fakeMirrorStore) List(_ context.Context, bucket, prefix string) (map[st
return out, nil
}
func (f *fakeMirrorStore) Copy(_ context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
o, ok := f.buckets[srcBucket][srcKey]
if !ok || (srcETag != "" && o.ETag != srcETag) {
return "", lease.ErrCopySourceChanged
@@ -326,16 +337,22 @@ func (f *fakeMirrorStore) Copy(_ context.Context, srcBucket, srcKey, srcETag, ds
return o.ETag, nil
}
func (f *fakeMirrorStore) Get(_ context.Context, bucket, key string) ([]byte, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
if _, ok := f.buckets[bucket][key]; !ok {
return nil, lease.ErrMirrorNotFound
}
return []byte("{}"), nil
}
func (f *fakeMirrorStore) Put(_ context.Context, bucket, key string, _ []byte) error {
- f.put(bucket, key, "receipts")
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.putLocked(bucket, key, "receipts")
return nil
}
func (f *fakeMirrorStore) Delete(_ context.Context, bucket, key string) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
delete(f.buckets[bucket], key)
return nil
}
diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go
index 74d178a72..dc51ef24d 100644
--- a/internal/console/apiinstancemove_offline.go
+++ b/internal/console/apiinstancemove_offline.go
@@ -29,7 +29,8 @@ import (
// offline-operator-move.md, Task 6). One hibernating instance moves between
// two regions with planned downtime, an operator at each step:
//
-// POST .../instances/{host}/move/offline {"region":"uk"} start
+// POST .../instances/{host}/move/offline/stage {"region":"uk"} pre-copy: -> staging
+// POST .../instances/{host}/move/offline {"region":"uk"} start (staging -> stopping)
// POST .../instances/{host}/move/offline/seal stopping -> sealing
// POST .../instances/{host}/move/offline/flip sealed -> landing
// POST .../instances/{host}/move/offline/open landed -> open
@@ -52,6 +53,12 @@ import (
// enforced guard are all off (internal/host, Task 3).
const offlineMoveCapability = "offline-move-1"
+// offlineStageCapability is the one a box publishes beside it when it can
+// stage (internal/host movesoffline_stage.go): run the pre-copy as the
+// stager and remove it on abort. A box without it treats a staging record
+// as nothing to do, so the stage door refuses a region holding one.
+const offlineStageCapability = "offline-stage-1"
+
// offlineReportFresh is how recently a box's report must be written for a
// door to believe it. Boxes rewrite their reports on every moves pass, and a
// source or sealer box every 15 s or so while its report is one a door reads
@@ -469,62 +476,64 @@ func (s *Server) offlineClaimEvidence(ctx context.Context, d offlineDoor) *offli
return nil
}
-func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Request) {
- d, ok := s.offlineGate(w, r)
- if !ok {
- return
- }
+// offlineBeginChecks is what start and stage check before either writes:
+// the instance's class, its claims, the region asked for, the app's
+// regions and the fleet's box rules (offlineSelect). It answers the region
+// code and the fleet it read, or false once it has answered the request.
+// A staged move is checked again in full when it starts: the fleet and the
+// claims may have changed since the stage.
+func (s *Server) offlineBeginChecks(w http.ResponseWriter, r *http.Request, d offlineDoor) (string, offlineFleet, bool) {
rec, host := d.rec, d.host
if s.refuseSeedMove(w, rec) {
- return
+ return "", offlineFleet{}, false
}
if rec.Role != "" {
jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region")
- return
+ return "", offlineFleet{}, false
}
if len(rec.Replicas) > 0 {
jsonError(w, http.StatusBadRequest, "a home with replicas does not move. remove its replicas first")
- return
+ return "", offlineFleet{}, false
}
if rec.Steering != "" || rec.Unit != "" {
jsonError(w, http.StatusBadRequest, fmt.Sprintf("%s is steered or runs under a unit. offline moves cannot carry either yet", host))
- return
+ return "", offlineFleet{}, false
}
ctx := r.Context()
if ref := s.offlineClaimEvidence(ctx, d); ref != nil {
writeRefusal(w, ref)
- return
+ return "", offlineFleet{}, false
}
var req struct {
Region string `json:"region"`
}
if !readJSON(w, r, &req) {
- return
+ return "", offlineFleet{}, false
}
region := strings.TrimSpace(req.Region)
to := s.homeRegionIn(region)
if region == "" || !provision.ValidRegion(to) {
jsonError(w, http.StatusBadRequest, "invalid region")
- return
+ return "", offlineFleet{}, false
}
if to == rec.Region {
jsonError(w, http.StatusConflict, "the instance is already in "+region)
- return
+ return "", offlineFleet{}, false
}
m, err := provision.GetManifest(ctx, s.cfg.Releases, d.acct.Sqid, d.app)
if err != nil && !errors.Is(err, os.ErrNotExist) {
log.Printf("console: offline move %s: manifest: %v", host, err)
jsonError(w, http.StatusServiceUnavailable, "could not read the app's instances manifest. try again")
- return
+ return "", offlineFleet{}, false
}
if m != nil && !s.moveRegionAllowed(m.Regions, to) {
jsonError(w, http.StatusBadRequest, "region "+region+" is not one of this app's regions: "+strings.Join(m.Regions, ", "))
- return
+ return "", offlineFleet{}, false
}
fleet, ok := s.readOfflineFleet(ctx)
if !ok {
jsonError(w, http.StatusConflict, "the fleet's edges files could not all be read. try again")
- return
+ return "", offlineFleet{}, false
}
if to == "" || rec.Region == "" {
// A box that sets CARLOS_REGION to the home's name is home to this
@@ -533,22 +542,173 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re
for _, f := range fleet.snap.Files {
if f.Placement != "" && homePlacementCode(s.moveHomeName(), f.Placement) == "" {
jsonError(w, http.StatusConflict, f.Box+" sets CARLOS_REGION, so it cannot take a move into the home region")
- return
+ return "", offlineFleet{}, false
}
}
}
if ref := s.offlineSelect(ctx, fleet, host, rec.Region, to); ref != nil {
writeRefusal(w, ref)
+ return "", offlineFleet{}, false
+ }
+ return to, fleet, true
+}
+
+// offlineStagedFor is the refusal of a start or a stage that names another
+// region than the one the instance is staged for (MV-O179).
+func (s *Server) offlineStagedFor(host, to string) *offlineRefusal {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s is staged for %s. abort the move to move it to another region", host, s.offlineName(to))}
+}
+
+// offlineStaged is the record's offline move when it is staging.
+func offlineStaged(rec *provision.Record) (*provision.MoveState, bool) {
+ if mv := rec.Move; mv.Offline() && mv.State == provision.OfflineStaging {
+ return mv, true
+ }
+ return nil, false
+}
+
+// offlineStagerGone is MV-O190: the pinned stager of a staged move no
+// longer publishes an edges file in the source region. Its disappearance
+// is never proof that it stopped copying, so start and seal refuse.
+func (s *Server) offlineStagerGone(box, host, from string) *offlineRefusal {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s, which staged %s, no longer publishes an edges file in %s. bring it back or abort the move", box, host, from)}
+}
+
+// offlineStageCoords refuses a staged move whose target publishes other
+// staging coordinates than the ones the stage fixed on the record (MV-O192,
+// Astra round 2, finding 2): the staged copy is in the stored bucket, and
+// nothing is ever retargeted. offlineSelect has already required exactly
+// one target box.
+func (s *Server) offlineStageCoords(f offlineFleet, mv *provision.MoveState, host string) *offlineRefusal {
+ if mv.StagedAt == nil {
+ return nil
+ }
+ tgt := f.boxesAt(mv.To)
+ if len(tgt) != 1 || tgt[0].DataBucket != mv.StageBucket || tgt[0].DataBucketRegion != mv.StageRegion {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s now publishes a different data bucket or region from what was recorded when %s was staged. abort the move and stage it again", s.offlineName(mv.To), host)}
+ }
+ return nil
+}
+
+// offlineStagerPresent refuses a staged move whose pinned stager is not in
+// the source region's fleet now (MV-O190).
+func (s *Server) offlineStagerPresent(f offlineFleet, mv *provision.MoveState, host string) *offlineRefusal {
+ if mv.StagedAt == nil {
+ return nil
+ }
+ for _, b := range f.boxesAt(mv.From) {
+ if b.Box == mv.Stager {
+ return nil
+ }
+ }
+ return s.offlineStagerGone(mv.Stager, host, s.offlineName(mv.From))
+}
+
+func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Request) {
+ d, ok := s.offlineGate(w, r)
+ if !ok {
+ return
+ }
+ to, fleet, ok := s.offlineBeginChecks(w, r, d)
+ if !ok {
+ return
+ }
+ // A staged instance starts the move it is staged for: staging to
+ // stopping, the instance offline from now. The lock is the stage's.
+ if mv, staged := offlineStaged(d.rec); staged {
+ if to != mv.To {
+ writeRefusal(w, s.offlineStagedFor(d.host, mv.To))
+ return
+ }
+ if ref := s.offlineStagerPresent(fleet, mv, d.host); ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ if ref := s.offlineStageCoords(fleet, mv, d.host); ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ s.offlineAdvance(w, r, d, provision.OfflineStartStaged(mv.ID, d.addr, time.Now()))
return
}
+ s.offlineBegin(w, r, d, func(id string) func(*provision.Record) error {
+ return provision.OfflineStart(id, d.rec.Region, to, d.addr, time.Now())
+ })
+}
+
+// handleAPIInstanceOfflineStage is the pre-copy door. On a record at rest it
+// creates the move in offline-staging, checked exactly as start is and
+// under the same one-move lock, and the source region's stager runs one
+// round on the serial bump. On a record already staging for that region it
+// moves staged_at forward (OfflineRestage), so the stager runs one more
+// round; each round copies only what is new (the receipts).
+func (s *Server) handleAPIInstanceOfflineStage(w http.ResponseWriter, r *http.Request) {
+ d, ok := s.offlineGate(w, r)
+ if !ok {
+ return
+ }
+ to, fleet, ok := s.offlineBeginChecks(w, r, d)
+ if !ok {
+ return
+ }
+ if mv, staged := offlineStaged(d.rec); staged {
+ if to != mv.To {
+ writeRefusal(w, s.offlineStagedFor(d.host, mv.To))
+ return
+ }
+ if ref := s.offlineStageCoords(fleet, mv, d.host); ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ s.offlineAdvance(w, r, d, provision.OfflineRestage(mv.ID, time.Now()))
+ return
+ }
+ for _, b := range append(fleet.boxesAt(d.rec.Region), fleet.boxesAt(to)...) {
+ if !slices.Contains(b.Capabilities, offlineStageCapability) {
+ jsonError(w, http.StatusConflict, fmt.Sprintf("%s cannot stage an offline move. it runs an older release", b.Box))
+ return
+ }
+ }
+ // The stager is pinned here, for the whole move: the one box that ever
+ // writes this move's staging prefix, and the one whose cleanup an abort
+ // waits for (Astra round 1, finding 2). offlineSelect has already
+ // required at least one source box.
+ var names []string
+ for _, b := range fleet.boxesAt(d.rec.Region) {
+ names = append(names, b.Box)
+ }
+ // The staging destination is fixed here too, before any copy (Astra
+ // round 2, finding 2): the one target box's data bucket and region, as
+ // offlineSelect found it. A target that publishes a source box's own
+ // bucket is refused: the activator there would promote the staged copy
+ // over the serving replica on its next wake (MV-O191).
+ tgt := fleet.boxesAt(to)[0]
+ for _, b := range fleet.boxesAt(d.rec.Region) {
+ if b.DataBucket == tgt.DataBucket {
+ jsonError(w, http.StatusConflict, fmt.Sprintf("%s publishes the same data bucket as %s. a staged copy there would replace the serving one", s.offlineName(to), s.offlineName(d.rec.Region)))
+ return
+ }
+ }
+ st := provision.StageTarget{Stager: moves.OfflineStager(names), Bucket: tgt.DataBucket, Region: tgt.DataBucketRegion}
+ s.offlineBegin(w, r, d, func(id string) func(*provision.Record) error {
+ return provision.OfflineStage(id, d.rec.Region, to, d.addr, st, time.Now())
+ })
+}
+
+// offlineBegin mints the move's id and writes the transition begin makes
+// with it (OfflineStart or OfflineStage) under the one-move lock, then
+// bumps the serial and answers 202 with the status view.
+func (s *Server) offlineBegin(w http.ResponseWriter, r *http.Request, d offlineDoor, begin func(id string) func(*provision.Record) error) {
+ ctx := r.Context()
+ host := d.host
id, err := moves.NewMoveID(rand.Reader)
if err != nil {
log.Printf("console: offline move %s: move id: %v", host, err)
jsonError(w, http.StatusBadGateway, "could not mint a move id")
return
}
- start := provision.OfflineStart(id, rec.Region, to, d.addr, time.Now())
- probe := *rec
+ start := begin(id)
+ probe := *d.rec
if err := start(&probe); err != nil {
jsonError(w, http.StatusConflict, offlineConflict(err))
return
@@ -708,6 +868,7 @@ func (s *Server) offlineReleaseLock(ctx context.Context, host, id string) {
// The verbs.
const (
+ offlineVerbStart = "start"
offlineVerbSeal = "seal"
offlineVerbFlip = "flip"
offlineVerbOpen = "open"
@@ -718,6 +879,8 @@ const (
// wait for a box, not for the operator.
func offlineVerbOf(state string) string {
switch state {
+ case provision.OfflineStaging:
+ return offlineVerbStart
case provision.OfflineStopping:
return offlineVerbSeal
case provision.OfflineSealed:
@@ -746,6 +909,23 @@ func offlineOpenMut(id string, now time.Time) func(*provision.Record) error {
func (s *Server) offlineStepFor(ctx context.Context, d offlineDoor, mv *provision.MoveState, ev offlineEvidence, now time.Time) (offlineStep, *offlineRefusal) {
host, from := d.host, s.offlineName(mv.From)
switch mv.State {
+ case provision.OfflineStaging:
+ // What start would check of the fleet now; its request checks
+ // (the class, the claims, the region) are the start door's.
+ fleet, ok := s.readOfflineFleet(ctx)
+ if !ok {
+ return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}
+ }
+ if ref := s.offlineSelect(ctx, fleet, host, mv.From, mv.To); ref != nil {
+ return offlineStep{}, ref
+ }
+ if ref := s.offlineStagerPresent(fleet, mv, host); ref != nil {
+ return offlineStep{}, ref
+ }
+ if ref := s.offlineStageCoords(fleet, mv, host); ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: provision.OfflineStartStaged(mv.ID, d.addr, now)}, nil
case provision.OfflineStopping:
fleet, ok := s.readOfflineFleet(ctx)
if !ok {
@@ -795,6 +975,14 @@ func (s *Server) offlineStepFor(ctx context.Context, d offlineDoor, mv *provisio
if ref := offlineFinishCheck(ev, mv, host, "", sealerPhase, moves.OfflinePhaseReverted, boxes); ref != nil {
return offlineStep{}, ref
}
+ // A staged move also waits for its pinned stager to say the staged
+ // copy is gone, whatever its last round reported (Astra round 1,
+ // findings 1 and 2).
+ if mv.StagedAt != nil {
+ if rep, ok := ev.get(moves.OfflineRoleStager, mv.Stager); !ok || rep.Phase != moves.OfflinePhaseReverted {
+ return offlineStep{}, offlineStateRefusal(mv.State, host)
+ }
+ }
return offlineStep{mut: provision.OfflineFinish(mv.ID, now)}, nil
}
return offlineStep{}, offlineStateRefusal(mv.State, host)
@@ -808,6 +996,28 @@ func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provis
if len(src) == 0 {
return "", nil, &offlineRefusal{status: http.StatusBadRequest, msg: "no eligible box in " + from + " publishes an edges file"}
}
+ // A staged move's pinned stager is part of the barrier whatever the
+ // fleet says (Astra round 2, finding 1): it must still be in the
+ // source region's fleet (so it is one of the boxes below, and of the
+ // boxes the seal names), and it must have said, after the move left
+ // staging and under its pass lock, that no stage round is in flight
+ // and none will start (its stager report, phase stopped, fresh). The
+ // staging destination must still be the one the stage fixed.
+ if mv.StagedAt != nil {
+ if ref := s.offlineStagerPresent(f, mv, host); ref != nil {
+ return "", nil, ref
+ }
+ if ref := s.offlineStageCoords(f, mv, host); ref != nil {
+ return "", nil, ref
+ }
+ ack, ok := ev.get(moves.OfflineRoleStager, mv.Stager)
+ if ok && ack.Phase == moves.OfflinePhaseStopped && !offlineFresh(ack.OfflineReport, now) {
+ return "", nil, offlineStaleRefusal(mv.Stager)
+ }
+ if !ok || ack.Phase != moves.OfflinePhaseStopped {
+ return "", nil, s.offlineWaitRefusal(mv.Stager, from, host)
+ }
+ }
var data []string
for _, b := range src {
rep, ok := ev.get(moves.OfflineRoleSource, b.Box)
@@ -1056,25 +1266,77 @@ type offlineItemView struct {
// verb would be refused with. CanAbort says abort is legal. Failed says the
// sealer or the target reported it could not do its step: abort is the way on.
type offlineView struct {
- Host string `json:"host"`
- State string `json:"state"`
- ID string `json:"id,omitempty"`
- From string `json:"from,omitempty"`
- To string `json:"to,omitempty"`
- Sealer string `json:"sealer,omitempty"`
- Boxes []string `json:"boxes,omitempty"`
- RequestedBy string `json:"requested_by,omitempty"`
- RequestedAt int64 `json:"requested_at,omitempty"`
- UpdatedAt int64 `json:"updated_at,omitempty"`
- OpenedAt int64 `json:"opened_at,omitempty"`
- Next string `json:"next,omitempty"`
- Waiting string `json:"waiting,omitempty"`
- CanAbort bool `json:"can_abort,omitempty"`
- Failed bool `json:"failed,omitempty"`
- Left []offlineLeftView `json:"left,omitempty"`
- Reports []offlineReportView `json:"reports,omitempty"`
- Items []offlineItemView `json:"items,omitempty"`
- Last *provision.MoveOutcome `json:"last,omitempty"`
+ Host string `json:"host"`
+ State string `json:"state"`
+ ID string `json:"id,omitempty"`
+ From string `json:"from,omitempty"`
+ To string `json:"to,omitempty"`
+ Sealer string `json:"sealer,omitempty"`
+ Stager string `json:"stager,omitempty"`
+ Boxes []string `json:"boxes,omitempty"`
+ RequestedBy string `json:"requested_by,omitempty"`
+ RequestedAt int64 `json:"requested_at,omitempty"`
+ UpdatedAt int64 `json:"updated_at,omitempty"`
+ OpenedAt int64 `json:"opened_at,omitempty"`
+ Next string `json:"next,omitempty"`
+ Waiting string `json:"waiting,omitempty"`
+ CanAbort bool `json:"can_abort,omitempty"`
+ // NeverOffline: the move was staged and never started, so its
+ // instance has kept serving throughout (an abort from staging).
+ NeverOffline bool `json:"never_offline,omitempty"`
+ // StagedAt is the record's staged_at in Unix nanoseconds: the latest
+ // stage request. Stage.Request equals it once the stager has answered
+ // that request.
+ StagedAt int64 `json:"staged_at,omitempty"`
+ Failed bool `json:"failed,omitempty"`
+ Left []offlineLeftView `json:"left,omitempty"`
+ Stage *offlineStageView `json:"stage,omitempty"`
+ Reports []offlineReportView `json:"reports,omitempty"`
+ Items []offlineItemView `json:"items,omitempty"`
+ Last *provision.MoveOutcome `json:"last,omitempty"`
+}
+
+// offlineStageView is the pre-copy's progress, from the latest stager
+// report: when its last round ended, the replica as that round listed it,
+// how much of it the target's staging prefix holds, and what the round
+// copied. Phase failed carries the box's own diagnostic in Error. Stager
+// reports are rewritten each moves pass, not every few seconds, so their
+// age is shown and never judged.
+type offlineStageView struct {
+ Box string `json:"box"`
+ Phase string `json:"phase"`
+ At int64 `json:"at"`
+ ReplicaObjects int `json:"replica_objects"`
+ ReplicaBytes int64 `json:"replica_bytes"`
+ StagedObjects int `json:"staged_objects"`
+ StagedBytes int64 `json:"staged_bytes"`
+ RoundObjects int `json:"round_objects"`
+ RoundBytes int64 `json:"round_bytes"`
+ Error string `json:"error,omitempty"`
+ // Request is the staged_at (Unix nanoseconds) the round answered.
+ Request int64 `json:"request,omitempty"`
+}
+
+// offlineStageOf is the stage view of the newest stager report, nil when no
+// box has staged this move.
+func offlineStageOf(ev offlineEvidence) *offlineStageView {
+ var latest *offlineReport
+ for _, r := range ev.byRole(moves.OfflineRoleStager) {
+ if latest == nil || r.At.After(latest.At) {
+ latest = &r
+ }
+ }
+ if latest == nil {
+ return nil
+ }
+ r := latest.OfflineReport
+ v := &offlineStageView{Box: r.Box, Phase: r.Phase, At: r.At.Unix(),
+ ReplicaObjects: r.ReplicaObjects, ReplicaBytes: r.ReplicaBytes, StagedObjects: r.StagedObjects, StagedBytes: r.StagedBytes,
+ RoundObjects: r.RoundObjects, RoundBytes: r.RoundBytes, Error: r.StageError}
+ if !r.StageRequest.IsZero() {
+ v.Request = r.StageRequest.UnixNano()
+ }
+ return v
}
// offlineViewOf builds the status view of a record. Evidence it cannot read
@@ -1088,13 +1350,17 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli
now := time.Now()
v.State, v.ID = mv.State, mv.ID
v.From, v.To = s.offlineName(mv.From), s.offlineName(mv.To)
- v.Sealer, v.Boxes = mv.Sealer, mv.Boxes
+ v.Sealer, v.Boxes, v.Stager = mv.Sealer, mv.Boxes, mv.Stager
+ v.NeverOffline = !mv.StartedOffline()
+ if mv.StagedAt != nil {
+ v.StagedAt = mv.StagedAt.UnixNano()
+ }
v.RequestedBy, v.RequestedAt, v.UpdatedAt = mv.RequestedBy, mv.RequestedAt.Unix(), mv.UpdatedAt.Unix()
if mv.OpenedAt != nil {
v.OpenedAt = mv.OpenedAt.Unix()
}
switch mv.State {
- case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
+ case provision.OfflineStaging, provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
v.CanAbort = true
}
ev, err := s.readOfflineEvidence(ctx, rec.Host, mv.ID)
@@ -1107,6 +1373,7 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli
v.Reports = append(v.Reports, offlineReportView{OfflineReport: r.OfflineReport, Fresh: offlineFresh(r.OfflineReport, now)})
}
v.Items = offlineItems(ev, mv)
+ v.Stage = offlineStageOf(ev)
for _, r := range ev.byRole(moves.OfflineRoleSource) {
for _, p := range r.Retired {
if p.Action == moves.RetiredLeft {
diff --git a/internal/console/apiinstancemove_offline_stage_test.go b/internal/console/apiinstancemove_offline_stage_test.go
new file mode 100644
index 000000000..a22255413
--- /dev/null
+++ b/internal/console/apiinstancemove_offline_stage_test.go
@@ -0,0 +1,325 @@
+package console
+
+import (
+ "context"
+ "net/http"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// The pre-copy door (offline-staging), in-process against the offline
+// doors' console fixture: the boxes' side is played by the reports they
+// write, the record moves only through provision's offline transitions.
+
+func msgO179(to string) string {
+ return offlineHost + " is staged for " + to + ". abort the move to move it to another region"
+}
+
+func msgO180(box string) string {
+ return box + " cannot stage an offline move. it runs an older release"
+}
+
+// stage runs the stage door on a record at rest and keeps the id.
+func (w *offlineWorld) stage() offlineView {
+ w.t.Helper()
+ before := serialBytes(w.t, w.fx.rel)
+ v := w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging)
+ if v.ID == "" {
+ w.t.Fatalf("stage answered no id: %+v", v)
+ }
+ if serialBytes(w.t, w.fx.rel) == before {
+ w.t.Fatal("stage did not bump the serial")
+ }
+ w.id = v.ID
+ return v
+}
+
+// staged writes the stager's report as the stager writes it after a round.
+func (w *offlineWorld) staged(box string, edit func(*moves.OfflineReport)) {
+ w.t.Helper()
+ w.report(moves.OfflineRoleStager, box, moves.OfflinePhaseStaged, func(r *moves.OfflineReport) {
+ r.Held, r.Fenced, r.Quiesced = false, false, false
+ r.ReplicaObjects, r.ReplicaBytes, r.StagedObjects, r.StagedBytes, r.RoundObjects, r.RoundBytes = 305, 1_580_000_000, 300, 1_570_000_000, 12, 9_000_000
+ if edit != nil {
+ edit(r)
+ }
+ })
+}
+
+func TestOfflineStageThenStartThenSeal(t *testing.T) {
+ us := offlineBox{Box: "us-box", Placement: "us", Bucket: "data-us"}
+ w := newOfflineWorld(t, ieOff("ie-box"), ukOff("uk-box"), us)
+ v := w.stage()
+ rec := w.record()
+ if rec.Region != "ie" || rec.Move.State != provision.OfflineStaging || rec.Move.To != "uk" || rec.Move.StagedAt == nil || rec.Move.Stager != "ie-box" {
+ t.Fatalf("staged record: %q %+v", rec.Region, rec.Move)
+ }
+ if !v.CanAbort || v.Next != offlineVerbStart || v.Stage != nil {
+ t.Fatalf("staged view: next %q, can abort %v, stage %+v", v.Next, v.CanAbort, v.Stage)
+ }
+ if l, ok := w.lock(); !ok || l.Host != offlineHost || l.Op != w.id {
+ t.Fatalf("lock %+v %v, want this move's", l, ok)
+ }
+
+ if !v.NeverOffline || v.StagedAt != rec.Move.StagedAt.UnixNano() {
+ t.Fatalf("staged view: never offline %v, staged at %d (record %v)", v.NeverOffline, v.StagedAt, rec.Move.StagedAt)
+ }
+
+ // Again: one more round. The request moves forward, the rest of the
+ // move does not, and the serial moves.
+ serial := serialBytes(t, w.fx.rel)
+ again := w.ok("/stage", `{"region":"uk"}`, provision.OfflineStaging)
+ if again.ID != w.id || again.StagedAt <= v.StagedAt {
+ t.Fatalf("a second stage answered move %s staged at %d, want %s after %d", again.ID, again.StagedAt, w.id, v.StagedAt)
+ }
+ if serialBytes(t, w.fx.rel) == serial {
+ t.Fatal("a second stage did not bump the serial")
+ }
+ if after := w.record(); after.Move.State != provision.OfflineStaging || !after.Move.RequestedAt.Equal(rec.Move.RequestedAt) || after.Move.ID != w.id {
+ t.Fatalf("a second stage changed the move: %+v", after.Move)
+ }
+ v = again
+
+ // Another region: refused for stage and for start, nothing written.
+ w.refused(http.MethodPost, "/stage", `{"region":"us"}`, http.StatusConflict, msgO179("uk"))
+ w.refused(http.MethodPost, "", `{"region":"us"}`, http.StatusConflict, msgO179("uk"))
+ // The other verbs wait for start.
+ for _, verb := range []string{"/seal", "/flip", "/open", "/finish"} {
+ w.refused(http.MethodPost, verb, "", http.StatusConflict, msgO25(provision.OfflineStaging))
+ }
+
+ // The status shows the stager's progress, and which request it answered.
+ w.staged("ie-box", func(r *moves.OfflineReport) { r.StageRequest = time.Unix(0, v.StagedAt).UTC() })
+ st := w.status()
+ if st.Stage == nil || st.Stage.Box != "ie-box" || st.Stage.Phase != moves.OfflinePhaseStaged || st.Stage.StagedObjects != 300 ||
+ st.Stage.ReplicaObjects != 305 || st.Stage.StagedBytes != 1_570_000_000 || st.Stage.RoundBytes != 9_000_000 || st.Stage.At == 0 ||
+ st.Stage.Request != v.StagedAt {
+ t.Fatalf("stage view %+v (request %d)", st.Stage, v.StagedAt)
+ }
+
+ // Start: staging to stopping, the same move and lock, offline from now.
+ started := w.ok("", `{"region":"uk"}`, provision.OfflineStopping)
+ if started.ID != w.id {
+ t.Fatalf("start answered move %s, want the staged %s", started.ID, w.id)
+ }
+ rec = w.record()
+ if !rec.Move.RequestedAt.After(*rec.Move.StagedAt) || rec.Move.RequestedBy == "" || started.NeverOffline {
+ t.Fatalf("started record %+v (never offline %v)", rec.Move, started.NeverOffline)
+ }
+ if l, _ := w.lock(); l.Op != w.id {
+ t.Fatalf("lock %+v after start", l)
+ }
+ if st := w.status(); st.Stage == nil || st.Stage.StagedObjects != 300 {
+ t.Fatalf("the stage's progress is gone from the status after start: %+v", st.Stage)
+ }
+ // From here it is the ordinary move: stage and start cannot be posted
+ // again, and seal follows the stop.
+ w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, "a move to uk is already "+provision.OfflineStopping)
+ w.stopped(nil, "ie-box")
+ w.stagerStopped("ie-box", time.Now())
+ w.ok("/seal", "", provision.OfflineSealing)
+}
+
+// The stage door's own rule: every box in both regions can stage. A box
+// that does not publish offline-stage-1 would leave a staging record to
+// nobody, so the door refuses before anything is written. The start door
+// does not ask it.
+func TestOfflineStageRefusesABoxThatCannotStage(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ boxes []offlineBox
+ box string
+ }{
+ {"source", []offlineBox{{Box: "ie-box", Placement: "ie", Bucket: "data-ie", NoStageCap: true}, ukOff("uk-box")}, "ie-box"},
+ {"target", []offlineBox{ieOff("ie-box"), {Box: "uk-box", Placement: "uk", Bucket: "data-uk", NoStageCap: true}}, "uk-box"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ w := newOfflineWorld(t, c.boxes...)
+ w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO180(c.box))
+ if rec := w.record(); rec.Move != nil {
+ t.Fatalf("a refused stage wrote %+v", rec.Move)
+ }
+ if _, ok := w.lock(); ok {
+ t.Fatal("a refused stage took the lock")
+ }
+ w.start()
+ })
+ }
+ // The start door's refusals hold for stage too (one of them here).
+ w := newOfflineWorld(t, ieOff("ie-box"), offlineBox{Box: "uk-box", Placement: "uk", Bucket: "data-uk", NoCap: true})
+ w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO21("uk-box"))
+}
+
+// Abort from staging: reverting, the region untouched; finish waits for
+// every source box's reverted (the stager's comes after it has deleted the
+// staged copy), then ends a move that was never offline and frees the lock.
+func TestOfflineAbortFromStagingDoor(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.stage()
+ w.staged("ie-box", nil)
+ if v := w.ok("/abort", "", provision.OfflineReverting); !v.NeverOffline {
+ t.Fatal("an abort from staging does not say the instance was never offline")
+ }
+ if rec := w.record(); rec.Region != "ie" || rec.Move.State != provision.OfflineReverting {
+ t.Fatalf("after abort: %q %+v", rec.Region, rec.Move)
+ }
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
+ // Every source box has reverted, but the pinned stager has not said
+ // the staged copy is gone (Astra round 1, findings 1 and 2): no finish,
+ // whatever its last round reported, or another box's stager report.
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
+ w.report(moves.OfflineRoleStager, "ie-other", moves.OfflinePhaseReverted, nil)
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
+ w.report(moves.OfflineRoleStager, "ie-box", moves.OfflinePhaseReverted, nil)
+ v := w.ok("/finish", "", "none")
+ if v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeAborted || v.Last.OfflineS != 0 {
+ t.Fatalf("last = %+v", v.Last)
+ }
+ if _, ok := w.lock(); ok {
+ t.Fatal("the lock outlived the aborted stage")
+ }
+ // The instance can be staged again.
+ w.stage()
+}
+
+// A failed round shows in the status with the box's own diagnostic.
+func TestOfflineStageViewShowsAFailedRound(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.stage()
+ w.staged("ie-box", func(r *moves.OfflineReport) {
+ r.Phase, r.StageError, r.At = moves.OfflinePhaseFailed, "mirror: copy instances/x: boom", time.Now().UTC()
+ })
+ st := w.status()
+ if st.Stage == nil || st.Stage.Phase != moves.OfflinePhaseFailed || st.Stage.Error != "mirror: copy instances/x: boom" || st.Stage.StagedObjects != 300 {
+ t.Fatalf("stage view %+v", st.Stage)
+ }
+}
+
+// The stage door pins the stager: the lexically first eligible source box,
+// whatever order the fleet lists them in.
+func TestOfflineStagePinsTheStager(t *testing.T) {
+ w := newOfflineWorld(t, ieOff("ie-c"), ieOff("ie-a"), ukOff("uk-box"))
+ v := w.stage()
+ if rec := w.record(); rec.Move.Stager != "ie-a" || v.Stager != "ie-a" {
+ t.Fatalf("stager %q (view %q), want ie-a", rec.Move.Stager, v.Stager)
+ }
+}
+
+// ---- Astra round 2 ----
+
+func msgO190(box string) string {
+ return box + ", which staged " + offlineHost + ", no longer publishes an edges file in ie. bring it back or abort the move"
+}
+
+const (
+ msgO191 = "uk publishes the same data bucket as ie. a staged copy there would replace the serving one"
+ msgO192 = "uk now publishes a different data bucket or region from what was recorded when " + offlineHost + " was staged. abort the move and stage it again"
+)
+
+// unpublish removes a box from the fleet's evidence, its edges file and
+// its status report: the box left the fleet (it may still be running).
+func (w *offlineWorld) unpublish(box string) {
+ w.t.Helper()
+ for _, key := range []string{edgesFilePrefix + box + ".json", "status/boxes/" + box + ".json"} {
+ if err := w.fx.rel.Delete(context.Background(), key); err != nil {
+ w.t.Fatal(err)
+ }
+ }
+}
+
+// stagerStopped writes the pinned stager's stop acknowledgement.
+func (w *offlineWorld) stagerStopped(box string, at time.Time) {
+ w.t.Helper()
+ w.report(moves.OfflineRoleStager, box, moves.OfflinePhaseStopped, func(r *moves.OfflineReport) {
+ r.Held, r.Fenced, r.Quiesced, r.At = false, false, false, at
+ })
+}
+
+// Finding 1: seal waits for the pinned stager's fresh acknowledgement that
+// no round is in flight and none will start, beside every box's stop, and
+// names the stager among the boxes it captures.
+func TestOfflineSealWaitsForThePinnedStager(t *testing.T) {
+ w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
+ w.stage()
+ w.ok("", `{"region":"uk"}`, provision.OfflineStopping)
+ w.stopped(nil, "ie-a", "ie-b")
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, "Waiting for ie-a in ie to confirm it has stopped "+offlineHost+". "+offlineHost+" is offline.")
+ w.stagerStopped("ie-a", time.Now().Add(-10*time.Minute))
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO178("ie-a"))
+ w.stagerStopped("ie-a", time.Now())
+ w.ok("/seal", "", provision.OfflineSealing)
+ if rec := w.record(); !slices.Contains(rec.Move.Boxes, "ie-a") {
+ t.Fatalf("the seal's boxes %v leave out the stager", rec.Move.Boxes)
+ }
+}
+
+// Finding 1: a pinned stager that no longer publishes an edges file is never
+// taken to have stopped. start and seal refuse with MV-O190.
+func TestOfflineStagedMoveRefusesAMissingStager(t *testing.T) {
+ w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
+ w.stage()
+ w.unpublish("ie-a")
+ w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO190("ie-a"))
+ if st := w.status(); st.Next != "" || st.Waiting != msgO190("ie-a") {
+ t.Fatalf("status next %q waiting %q", st.Next, st.Waiting)
+ }
+
+ w2 := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
+ w2.stage()
+ w2.ok("", `{"region":"uk"}`, provision.OfflineStopping)
+ w2.stopped(nil, "ie-a", "ie-b")
+ w2.stagerStopped("ie-a", time.Now())
+ w2.unpublish("ie-a")
+ w2.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO190("ie-a"))
+}
+
+// Finding 2: the stage fixes its destination on the record, and refuses a
+// target that publishes a source box's own bucket.
+func TestOfflineStageFixesItsDestination(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.stage()
+ if m := w.record().Move; m.StageBucket != "data-uk" || m.StageRegion != "eu-west-2" {
+ t.Fatalf("staging destination %q %q", m.StageBucket, m.StageRegion)
+ }
+ w2 := newOfflineWorld(t, ieOff("ie-box"), offlineBox{Box: "uk-box", Placement: "uk", Bucket: "data-ie"})
+ w2.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO191)
+}
+
+// Finding 2: once staged, a target that publishes other coordinates is
+// refused by stage, start and seal, never followed.
+func TestOfflineStagedMoveRefusesChangedCoordinates(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ target offlineBox
+ }{
+ {"another bucket", offlineBox{Box: "uk-box", Placement: "uk", Bucket: "data-uk-new"}},
+ {"another box", offlineBox{Box: "uk-new", Placement: "uk", Bucket: "data-uk-new"}},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.stage()
+ if c.target.Box != "uk-box" {
+ w.unpublish("uk-box")
+ }
+ w.fleet(c.target)
+ w.refused(http.MethodPost, "/stage", `{"region":"uk"}`, http.StatusConflict, msgO192)
+ w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO192)
+
+ w2 := newOfflineWorld(t, oneEach...)
+ w2.stage()
+ w2.ok("", `{"region":"uk"}`, provision.OfflineStopping)
+ w2.stopped(nil, "ie-box")
+ w2.stagerStopped("ie-box", time.Now())
+ if c.target.Box != "uk-box" {
+ w2.unpublish("uk-box")
+ }
+ w2.fleet(c.target)
+ w2.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO192)
+ })
+ }
+}
diff --git a/internal/console/apiinstancemove_offline_test.go b/internal/console/apiinstancemove_offline_test.go
index e588d9da8..7b6d0c144 100644
--- a/internal/console/apiinstancemove_offline_test.go
+++ b/internal/console/apiinstancemove_offline_test.go
@@ -65,7 +65,8 @@ func offlineClaimFor(t *testing.T, fx *consoleFixture, host, box, region string)
// snapshot.
type offlineBox struct {
Box, Placement, Bucket string
- NoCap bool // no offline-move-1
+ NoCap bool // no offline-move-1 (nor offline-stage-1)
+ NoStageCap bool // no offline-stage-1
ClaimsOn bool // its status report says claims are on
NoMetrics bool // its status report carries no metrics
NotSharedPool bool // serves_shared_pool false
@@ -142,6 +143,9 @@ func (w *offlineWorld) fleet(boxes ...offlineBox) {
caps := []string{"region-ha-1"}
if !b.NoCap {
caps = append(caps, offlineMoveCapability)
+ if !b.NoStageCap {
+ caps = append(caps, offlineStageCapability)
+ }
}
hosts := map[string]any{}
for _, h := range b.Hosts {
@@ -153,7 +157,7 @@ func (w *offlineWorld) fleet(boxes ...offlineBox) {
}
body, err := json.Marshal(map[string]any{
"v": 1, "box": b.Box, "box_id": b.Box, "eip": "1.2.3.4", "region": aws, "placement": b.Placement,
- "data_bucket": b.Bucket, "hibernation_configured": !b.NoHibernation, "serves_shared_pool": !b.NotSharedPool,
+ "data_bucket": b.Bucket, "data_bucket_region": aws, "hibernation_configured": !b.NoHibernation, "serves_shared_pool": !b.NotSharedPool,
"accepts_inbound": true, "secrets_key_fp": fp, "capabilities": caps, "hosts": hosts,
})
if err != nil {
@@ -438,6 +442,7 @@ func msgO31(box string) string {
var offlineRoutes = []struct{ method, path, body string }{
{http.MethodGet, "", ""},
{http.MethodPost, "", `{"region":"uk"}`},
+ {http.MethodPost, "/stage", `{"region":"uk"}`},
{http.MethodPost, "/seal", ""},
{http.MethodPost, "/flip", ""},
{http.MethodPost, "/open", ""},
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index b75f82a83..c597c3e56 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -720,6 +720,7 @@ func (s *Server) registerCLIRoutes() {
// and refused with MV-O20 on a console with host claims or Phase 1 on.
s.cliRoute("GET /api/cli/apps/{account}/{app}/instances/{host}/move/offline", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStatus)
s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStart)
+ s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/stage", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStage)
s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbSeal))
s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbFlip))
s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbOpen))
diff --git a/internal/console/claim_invariant_fixtures_test.go b/internal/console/claim_invariant_fixtures_test.go
index 88b156d31..d09d1d291 100644
--- a/internal/console/claim_invariant_fixtures_test.go
+++ b/internal/console/claim_invariant_fixtures_test.go
@@ -1490,6 +1490,13 @@ var claimFixtures = map[string]claimFixture{
status: http.StatusConflict,
msg: claimOfflineMoveUnsupportedMsg,
}},
+ "route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/stage": {refused: &claimRefusalFixture{
+ apps: claimRefs("bab/titogo"),
+ setup: func(t *testing.T, cx *claimCase) { cx.movableInstance(t) },
+ do: claimRefusedAPI(http.MethodPost, "/api/cli/apps/bab/titogo/instances/acme.go.tito.io/move/offline/stage", `{"region":"uk"}`),
+ status: http.StatusConflict,
+ msg: claimOfflineMoveUnsupportedMsg,
+ }},
"route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal": {refused: &claimRefusalFixture{
apps: claimRefs("bab/titogo"),
setup: func(t *testing.T, cx *claimCase) { cx.offlineMove(t, provision.OfflineStopping) },
diff --git a/internal/console/svcroutes_test.go b/internal/console/svcroutes_test.go
index 804f1f078..94d5b365f 100644
--- a/internal/console/svcroutes_test.go
+++ b/internal/console/svcroutes_test.go
@@ -540,6 +540,7 @@ var exemptFromLegacyPin = map[string]bool{
// refuses with MV-O20 on a console with host claims or Phase 1 on.
"GET /api/cli/apps/{account}/{app}/instances/{host}/move/offline": true,
"POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline": true,
+ "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/stage": true,
"POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal": true,
"POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip": true,
"POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open": true,
diff --git a/internal/console/testdata/claim-routes.txt b/internal/console/testdata/claim-routes.txt
index faf01f510..932579962 100644
--- a/internal/console/testdata/claim-routes.txt
+++ b/internal/console/testdata/claim-routes.txt
@@ -195,6 +195,7 @@ route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/finish
route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip
route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open
route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal
+route POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/stage
route POST /api/cli/apps/{account}/{app}/instances/{host}/retry
route POST /api/cli/apps/{account}/{app}/instances/{host}/runs
route POST /api/cli/apps/{account}/{app}/instances/{host}/runs/{id}/ack
diff --git a/internal/edgesfile/snapshot.go b/internal/edgesfile/snapshot.go
index cdaf1333a..bbfd5c6b5 100644
--- a/internal/edgesfile/snapshot.go
+++ b/internal/edgesfile/snapshot.go
@@ -37,6 +37,9 @@ type PlacementView struct {
// coordinates, and doubles as positive hosting evidence from a writer
// too old to publish HibernationConfigured.
DataBucket string `json:"data_bucket"`
+ // DataBucketRegion is that bucket's AWS region, as the box publishes it
+ // beside DataBucket. An offline move's stage fixes both on the record.
+ DataBucketRegion string `json:"data_bucket_region"`
// HibernationConfigured is the box's own answer to "is my hibernation
// activator wired" (internal/host/edgesfile.go). A POINTER because
// absent and false are different facts and the difference is the whole
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index 150f394a0..e30d70329 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -172,7 +172,8 @@ func edgesFileCapabilities(getenv func(string) string) []string {
return []string{capRegionHA1}
}
-// withOfflineCapability is caps with offline-move-1 when capable. The loop
+// withOfflineCapability is caps with offline-move-1 and offline-stage-1 when
+// capable (a box that can take part in an offline move can stage one). The loop
// passes armedOfflineCapable(): the armed moves worker's own answer, the one
// the reconcile hooks act on, so a box never publishes the capability
// without the hold that makes it safe (a target that published it with no
@@ -181,7 +182,7 @@ func withOfflineCapability(caps []string, capable bool) []string {
if !capable {
return caps
}
- return append(slices.Clone(caps), capOfflineMove1)
+ return append(slices.Clone(caps), capOfflineMove1, capOfflineStage1)
}
// headroomCapabilities is what headroomFor is asked about: region HA's
diff --git a/internal/host/movescut_test.go b/internal/host/movescut_test.go
index 0f3947853..f687f9906 100644
--- a/internal/host/movescut_test.go
+++ b/internal/host/movescut_test.go
@@ -578,10 +578,12 @@ func TestCutRevertsAfterThreeDeferredRounds(t *testing.T) {
if strings.HasSuffix(srcKey, lastTXKey) {
// The source changes under the copy: the conditional copy is
// refused and the object deferred.
+ h.f.bmu.Lock()
n++
o := h.f.buckets["data-ie"][srcKey]
o.ETag = "changed-" + string(rune('a'+n))
h.f.buckets["data-ie"][srcKey] = o
+ h.f.bmu.Unlock()
}
}
h.cut(t)
diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go
index d272036c8..a78e946af 100644
--- a/internal/host/movesoffline.go
+++ b/internal/host/movesoffline.go
@@ -330,6 +330,14 @@ func (w *movesWorker) offlineRound(ctx context.Context, s release.Store, rec pro
}
return nil
}
+ if m.State == provision.OfflineStaging {
+ // The pre-copy (movesoffline_stage.go): nothing holds or fences
+ // the instance, and only the stager acts.
+ if source {
+ return w.offlineStageStep(ctx, s, rec)
+ }
+ return nil
+ }
var errs []error
if source {
errs = append(errs, w.offlineSourceStep(ctx, s, rec))
@@ -535,7 +543,9 @@ func (w *movesWorker) offlineSourceStep(ctx context.Context, s release.Store, re
}
switch rec.Move.State {
case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
- return w.offlineSourceStop(ctx, s, rec)
+ // The pinned stager's stop acknowledgement first: the seal door
+ // needs it beside this box's stop (movesoffline_stage.go).
+ return errors.Join(w.offlineStagerStopped(ctx, s, rec), w.offlineSourceStop(ctx, s, rec))
case provision.OfflineOpen:
return w.offlineSourcePark(ctx, s, rec)
case provision.OfflineFinishing:
@@ -1078,6 +1088,10 @@ func (w *movesWorker) offlineSourceRevert(ctx context.Context, s release.Store,
if rt != nil && rt.HoldOp == m.ID {
return fmt.Errorf("offline move %s: %s is still held by the move", m.ID, host)
}
+ // A staged move's pre-copy is removed before this box says reverted.
+ if err := w.offlineUnstage(ctx, s, rec); err != nil {
+ return err
+ }
r, err := w.sourceReport(host, m, rt, false, moves.OfflinePhaseReverted)
if err != nil {
return err
diff --git a/internal/host/movesoffline_seal.go b/internal/host/movesoffline_seal.go
index a0056e640..f72f38d61 100644
--- a/internal/host/movesoffline_seal.go
+++ b/internal/host/movesoffline_seal.go
@@ -351,6 +351,12 @@ func (w *movesWorker) offlineTakeSeal(ctx context.Context, s release.Store, rec
if err != nil {
return seal, err
}
+ // A staged move hands over in the destination its stage fixed (Astra
+ // round 2, finding 2). A target that publishes anything else now is
+ // not sealed into; the seal door refuses it first (MV-O192).
+ if m.StagedAt != nil && (tgt.DataBucket != m.StageBucket || tgt.DataBucketRegion != m.StageRegion) {
+ return seal, fmt.Errorf("the target now publishes %s (%s), not the staging bucket %s (%s) the stage fixed", tgt.DataBucket, tgt.DataBucketRegion, m.StageBucket, m.StageRegion)
+ }
mirror, err := w.offlineMirror(ctx, tgt.DataBucketRegion)
if err != nil {
return seal, err
diff --git a/internal/host/movesoffline_stage.go b/internal/host/movesoffline_stage.go
new file mode 100644
index 000000000..53690e6ad
--- /dev/null
+++ b/internal/host/movesoffline_stage.go
@@ -0,0 +1,321 @@
+package host
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The pre-copy of an offline move (offline-staging): while the instance
+// keeps serving, one source box (the stager) runs mirror rounds of its
+// replica into the target's staging prefix, so the seal's own round, with
+// the instance offline, copies only what changed since.
+//
+// A stage round is the seal's round with nothing else: the same source
+// (instances/<host>/ in this box's bucket), the same destination
+// (moves/<host>/ in the target's data bucket), the same receipts
+// (move-receipts/<host>.json in this box's bucket) and the same mirror pair.
+// The staging prefix and the receipts are keyed by host, not by move, so
+// the seal finds what the stage copied with no hand-over: lease.MirrorPrefix
+// skips an object only when its ETag equals the source's or a receipt for
+// this exact pair still matches both sides, deletes every staged key the
+// source no longer has (litestream's retention between stage and seal),
+// and copies everything else. Nothing a stage round left behind is trusted
+// beyond that, so a stale prefix is corrected, never sealed as it is.
+//
+// The round holds nothing, fences nothing and takes no lease: it reads the
+// replica (its listing and the copies' reads) and writes only the target's
+// staging prefix and this box's receipts object. The lease, lock.json
+// included, is never touched (MirrorPrefix skips lock.json on both sides).
+//
+// One round runs per stage request: the record's staged_at, which the stage
+// door sets and every later stage moves forward (provision.OfflineRestage).
+// The stager report names the staged_at its round answered
+// (StageRequest), and a pass whose stager report already answers the
+// record's runs nothing, so an idle staging costs one report read a pass and
+// no S3 request; a failed round is retried by running stage again. Rounds
+// stop when the record leaves staging: the pass that runs a round holds the
+// worker's pass lock, so this box's stopped report, which the seal door
+// waits for, is written only after the last round it ran has ended.
+
+// capOfflineStage1 is the edges-file capability a box publishes beside
+// offline-move-1 when it can stage: run the pre-copy as the stager, and
+// clean it up on abort. The console's stage door refuses a region holding a
+// box that does not publish it.
+const capOfflineStage1 = "offline-stage-1"
+
+// offlineStageStep is the stager's step in offline-staging: one round, then
+// the stager report. The stager is the box the stage door pinned on the
+// record (MoveState.Stager); no other box ever writes this move's staging
+// prefix, whatever the fleet looks like now (Astra round 1, finding 2).
+func (w *movesWorker) offlineStageStep(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ if cfg.Box == "" {
+ return errors.New("offline move: this box has no label to report under")
+ }
+ if m.Stager == "" || m.Stager != cfg.Box {
+ return nil
+ }
+ // The record again, now, under this box's pass lock: the pass's listing
+ // may predate the start, and once the move has left staging no round
+ // ever starts (Astra round 2, finding 1).
+ cur, err := provision.GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
+ if err != nil {
+ return fmt.Errorf("offline move %s: read the record before a stage round: %w", m.ID, err)
+ }
+ if cur.Move == nil || cur.Move.ID != m.ID || cur.Move.State != provision.OfflineStaging {
+ return nil
+ }
+ rec, m = *cur, cur.Move
+ var request time.Time
+ if m.StagedAt != nil {
+ request = m.StagedAt.UTC()
+ }
+ prev, err := w.ownStagerReport(ctx, s, rec)
+ if err != nil {
+ return fmt.Errorf("offline move %s: read the stager report: %w", m.ID, err)
+ }
+ if prev != nil && prev.StageRequest.Equal(request) && (prev.Phase == moves.OfflinePhaseStaged || prev.Phase == moves.OfflinePhaseFailed) {
+ return nil // this request has had its round
+ }
+ r := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleStager, Phase: moves.OfflinePhaseStaged, StageRequest: request}
+ p, err := w.offlineStageRound(ctx, s, rec)
+ r.At = cfg.now().UTC()
+ if err != nil {
+ // The progress stays what the last good round found; the error is a
+ // box diagnostic the status view shows as it is.
+ if prev != nil {
+ p = stageProgress{replicaObjects: prev.ReplicaObjects, replicaBytes: prev.ReplicaBytes, stagedObjects: prev.StagedObjects, stagedBytes: prev.StagedBytes}
+ }
+ p.fill(&r)
+ r.Phase, r.StageError = moves.OfflinePhaseFailed, err.Error()
+ return errors.Join(fmt.Errorf("offline move %s: staging %s: %w", m.ID, host, err), writeOfflineReport(ctx, s, r))
+ }
+ p.fill(&r)
+ if p.roundObjects > 0 {
+ log.Printf("moves: %s: offline move %s: staged round: %d object(s) changed, %d bytes copied; %d of %d objects staged", host, m.ID, p.roundObjects, p.roundBytes, p.stagedObjects, p.replicaObjects)
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// stageProgress is what one stage round found.
+type stageProgress struct {
+ replicaObjects, stagedObjects, roundObjects int
+ replicaBytes, stagedBytes, roundBytes int64
+}
+
+func (p stageProgress) fill(r *moves.OfflineReport) {
+ r.ReplicaObjects, r.ReplicaBytes = p.replicaObjects, p.replicaBytes
+ r.StagedObjects, r.StagedBytes = p.stagedObjects, p.stagedBytes
+ r.RoundObjects, r.RoundBytes = p.roundObjects, p.roundBytes
+}
+
+// offlineStageRound is one pre-copy round and the listing that measures it.
+// It writes only to the staging destination the stage fixed on the record
+// (Astra round 2, finding 2), and only while the target still publishes
+// exactly that bucket and region; a target that publishes anything else is
+// refused, never followed. The round's delete phase reads the record again
+// first and stops if the move has left staging (finding 1).
+func (w *movesWorker) offlineStageRound(ctx context.Context, s release.Store, rec provision.Record) (stageProgress, error) {
+ cfg := w.cfg
+ host, m := rec.Host, rec.Move
+ if m.StageBucket == "" {
+ return stageProgress{}, errors.New("the record names no staging bucket")
+ }
+ tgt, err := w.offlineTarget(ctx, s, m)
+ if err != nil {
+ return stageProgress{}, err
+ }
+ if tgt.DataBucket != m.StageBucket || tgt.DataBucketRegion != m.StageRegion {
+ return stageProgress{}, fmt.Errorf("the target now publishes %s (%s), not the staging bucket %s (%s) the stage fixed", tgt.DataBucket, tgt.DataBucketRegion, m.StageBucket, m.StageRegion)
+ }
+ bucket, loc, _, err := cfg.tombstoneAt(host)
+ if err != nil {
+ return stageProgress{}, err
+ }
+ // The activator promotes a host's moves/<host>/ in its OWN bucket over
+ // its replica on every restore (promoteStagedReplica: staging wins). A
+ // staging bucket that is this box's own would have the serving
+ // instance restore from a copy behind its replica, so the round refuses
+ // (the stage door refuses it first).
+ if m.StageBucket == bucket {
+ return stageProgress{}, fmt.Errorf("the staging bucket is this box's own bucket %s; a staged copy there would replace the serving replica on its next wake", bucket)
+ }
+ mirror, err := w.offlineMirror(ctx, m.StageRegion)
+ if err != nil {
+ return stageProgress{}, err
+ }
+ gated := &stageDeleteGate{MirrorStore: mirror, check: func(ctx context.Context) error {
+ cur, err := provision.GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
+ if err != nil {
+ return fmt.Errorf("read the record before the delete phase: %w", err)
+ }
+ if cur.Move == nil || cur.Move.ID != m.ID || cur.Move.State != provision.OfflineStaging {
+ return errors.New("the move left staging during the round; its deletes are not made")
+ }
+ return nil
+ }}
+ srcPrefix := loc + "/"
+ dstPrefix := activator.MovesPrefix(host)
+ receipts := lease.Receipts{Bucket: bucket, Key: activator.MoveReceiptsKey(host)}
+ round, err := lease.MirrorPrefix(ctx, gated, bucket, srcPrefix, m.StageBucket, dstPrefix, receipts)
+ if err != nil {
+ return stageProgress{}, err
+ }
+ source, err := mirror.List(ctx, bucket, srcPrefix)
+ if err != nil {
+ return stageProgress{}, fmt.Errorf("list the replica: %w", err)
+ }
+ staged, err := mirror.List(ctx, m.StageBucket, dstPrefix)
+ if err != nil {
+ return stageProgress{}, fmt.Errorf("list the staged copy: %w", err)
+ }
+ p := stageProgress{roundObjects: round.Changed, roundBytes: round.BytesCopied}
+ for k, o := range source {
+ rel := strings.TrimPrefix(k, srcPrefix)
+ if rel == "lock.json" {
+ continue
+ }
+ p.replicaObjects++
+ p.replicaBytes += o.Size
+ if d, ok := staged[dstPrefix+rel]; ok && d.Size == o.Size {
+ p.stagedObjects++
+ p.stagedBytes += o.Size
+ }
+ }
+ return p, nil
+}
+
+// stageDeleteGate is a stage round's mirror with one check before the
+// delete phase: lease.MirrorPrefix deletes only after every copy has
+// landed, so the first Delete is that phase's start, and check runs there
+// once for all of the round's deletes. Its error fails every delete, so
+// the round deletes nothing.
+type stageDeleteGate struct {
+ lease.MirrorStore
+ check func(ctx context.Context) error
+ once sync.Once
+ err error
+}
+
+func (g *stageDeleteGate) Delete(ctx context.Context, bucket, key string) error {
+ g.once.Do(func() { g.err = g.check(ctx) })
+ if g.err != nil {
+ return g.err
+ }
+ return g.MirrorStore.Delete(ctx, bucket, key)
+}
+
+// offlineStagerStopped is the pinned stager's stop acknowledgement, written
+// in stopping and every later state up to landed, on every pass and every
+// reassert tick, always under this box's pass lock (Astra round 2, finding
+// 1). The record it was handed has left staging, which never comes back,
+// so no stage round starts from here on (offlineStageStep reads the record
+// again before one), and the round that holds the pass lock has ended
+// before this runs. The seal door waits for it, fresh. The progress of the
+// last round is kept.
+func (w *movesWorker) offlineStagerStopped(ctx context.Context, s release.Store, rec provision.Record) error {
+ m := rec.Move
+ if m.StagedAt == nil || m.Stager == "" || m.Stager != w.cfg.Box {
+ return nil
+ }
+ prev, err := w.ownStagerReport(ctx, s, rec)
+ if err != nil {
+ return fmt.Errorf("offline move %s: read the stager report: %w", m.ID, err)
+ }
+ r := moves.OfflineReport{Op: m.ID, Host: rec.Host, Box: w.cfg.Box, Role: moves.OfflineRoleStager, Phase: moves.OfflinePhaseStopped,
+ At: w.cfg.now().UTC(), StageRequest: m.StagedAt.UTC()}
+ if prev != nil {
+ stageProgress{replicaObjects: prev.ReplicaObjects, replicaBytes: prev.ReplicaBytes, stagedObjects: prev.StagedObjects,
+ stagedBytes: prev.StagedBytes, roundObjects: prev.RoundObjects, roundBytes: prev.RoundBytes}.fill(&r)
+ }
+ return writeOfflineReport(ctx, s, r)
+}
+
+// ownStagerReport is this box's stager report of the move, nil when it has
+// written none. One that does not read as a report is an error.
+func (w *movesWorker) ownStagerReport(ctx context.Context, s release.Store, rec provision.Record) (*moves.OfflineReport, error) {
+ key := moves.OfflineReportKey(rec.Host, rec.Move.ID, moves.OfflineRoleStager, w.cfg.Box)
+ if key == "" {
+ return nil, fmt.Errorf("offline move %s: %s has no stager report key", rec.Move.ID, rec.Host)
+ }
+ raw, _, err := s.Get(ctx, key)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(raw, &r); err != nil {
+ return nil, fmt.Errorf("%s: %w", key, err)
+ }
+ return &r, nil
+}
+
+// offlineUnstage is the stager's part of an abort of a staged move. A move
+// that carries staged_at may have staged copies whatever its reports say: a
+// round can land copies and die before its report (Astra round 1, finding
+// 1). So the pinned stager, and only it, deletes the target's staging
+// prefix and the copy receipts and says so in its stager report (phase
+// reverted), once, before its source step reports reverted. Its own last
+// round has ended by then: rounds run only in staging, and both run under
+// this box's pass lock. The console's finish of a staged move waits for
+// that report (finding 2). A move that was never staged does nothing here,
+// so an abort from stopping is what it was. The prefix deleted is this
+// host's in the target's bucket, never this box's own bucket, where a
+// moves/<host>/ prefix is not this move's (the stage round refuses to write
+// there for the same reason).
+func (w *movesWorker) offlineUnstage(ctx context.Context, s release.Store, rec provision.Record) error {
+ cfg := w.cfg
+ m, host := rec.Move, rec.Host
+ if m.StagedAt == nil || m.Stager == "" || m.Stager != cfg.Box {
+ return nil
+ }
+ own, err := w.ownStagerReport(ctx, s, rec)
+ if err != nil {
+ return fmt.Errorf("offline move %s: read the stager report: %w", m.ID, err)
+ }
+ if own != nil && own.Phase == moves.OfflinePhaseReverted {
+ return nil
+ }
+ // The destination the stage fixed on the record, never what the
+ // target publishes now (Astra round 2, finding 2): reverted is written
+ // only after that prefix is deleted. A stored bucket that is this box's
+ // own was refused at the stage, so it is never deleted from here, and
+ // nothing is acknowledged.
+ if m.StageBucket == "" {
+ return fmt.Errorf("offline move %s: the record names no staging bucket to clean", m.ID)
+ }
+ if m.StageBucket == cfg.OwnBucket {
+ return fmt.Errorf("offline move %s: the stored staging bucket %s is this box's own; nothing is deleted or acknowledged", m.ID, m.StageBucket)
+ }
+ mirror, err := w.offlineMirror(ctx, m.StageRegion)
+ if err != nil {
+ return err
+ }
+ prefix := activator.MovesPrefix(host)
+ if prefix == "moves/" || !strings.HasPrefix(prefix, "moves/") {
+ return fmt.Errorf("offline move %s: no staging prefix for %s", m.ID, host)
+ }
+ if err := lease.DeletePrefix(ctx, mirror, m.StageBucket, prefix); err != nil {
+ return fmt.Errorf("offline move %s: delete %s/%s: %w", m.ID, m.StageBucket, prefix, err)
+ }
+ log.Printf("moves: %s: offline move %s: the staged copy in %s/%s is deleted", host, m.ID, m.StageBucket, prefix)
+ clearOfflineReceipts(ctx, mirror, cfg.OwnBucket, host)
+ r := moves.OfflineReport{Op: m.ID, Host: host, Box: cfg.Box, Role: moves.OfflineRoleStager, Phase: moves.OfflinePhaseReverted, At: cfg.now().UTC()}
+ return writeOfflineReport(ctx, s, r)
+}
diff --git a/internal/host/movesoffline_stage_test.go b/internal/host/movesoffline_stage_test.go
new file mode 100644
index 000000000..047efc554
--- /dev/null
+++ b/internal/host/movesoffline_stage_test.go
@@ -0,0 +1,584 @@
+//go:build linux
+
+package host
+
+import (
+ "context"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// The pre-copy (offline-staging), through offlineRound over the seal world:
+// the S3 wire protocol, real litestream replicas, the real lease package and
+// the production edges-file writer. A stage round copies the replica into
+// the target's staging prefix with the instance serving; the seal's round
+// then copies only what changed since; abort from staging removes it all.
+
+// stagerEdges publishes b's edges file as an agent with an armed, capable
+// moves worker does: the hibernate coordinates and the offline capabilities
+// (offline-move-1, offline-stage-1). The stager is chosen from these.
+func (sw *sealWorld) stagerEdges(b *offBox) {
+ sw.t.Helper()
+ ident := edgesFile{Box: b.name, Placement: b.region, DataBucket: sealIE, DataBucketRegion: sealIERegion,
+ HibernationConfigured: true, ServesSharedPool: true, Capabilities: withOfflineCapability(nil, b.w.cfg.OfflineCapable)}
+ if err := edgesFileWrite(context.Background(), sw.s, b.reg, ident, "", &b.last, false); err != nil {
+ sw.t.Fatal(err)
+ }
+}
+
+// stagePass is sw.pass with the edges file stagerEdges writes.
+func (sw *sealWorld) stagePass(b *offBox) error {
+ sw.t.Helper()
+ b.reconcile()
+ sw.stagerEdges(b)
+ ctx := context.Background()
+ refreshOfflineView(ctx, sw.s, b.w.cfg.Offline)
+ return b.w.offlineRound(ctx, sw.s, sw.record())
+}
+
+func (sw *sealWorld) mustStagePass(b *offBox) {
+ sw.t.Helper()
+ if err := sw.stagePass(b); err != nil {
+ sw.t.Fatal(err)
+ }
+}
+
+func (sw *sealWorld) stage() { sw.stageBy("ie-a") }
+
+// stageBy stages with the stager the door pinned.
+func (sw *sealWorld) stageBy(stager string) {
+ sw.move(provision.OfflineStage(offID, "", "uk", "op@example.com", provision.StageTarget{Stager: stager, Bucket: sealUK, Region: sealUKRegion}, sw.clock()))
+}
+
+// restage is a second carlos move offline stage: one more round.
+func (sw *sealWorld) restage() {
+ sw.advance(time.Second)
+ sw.move(provision.OfflineRestage(offID, sw.clock()))
+}
+
+// copySpy records the source key of every server-side copy the fake S3
+// performs from now on.
+type copySpy struct {
+ mu sync.Mutex
+ keys []string
+}
+
+func (sw *sealWorld) spyCopies() *copySpy {
+ c := &copySpy{}
+ sw.fake.FailOn = func(op, key string) error {
+ if op == "copy" {
+ c.mu.Lock()
+ c.keys = append(c.keys, key)
+ c.mu.Unlock()
+ }
+ return nil
+ }
+ sw.t.Cleanup(func() { sw.fake.FailOn = nil })
+ return c
+}
+
+func (c *copySpy) take() []string {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ out := c.keys
+ c.keys = nil
+ return out
+}
+
+// replicaRel is the replica's listing by key relative to its prefix, lock
+// aside; stagedRel the staging prefix's the same way.
+func (sw *sealWorld) replicaRel() map[string]lease.MirrorObject {
+ prefix := sealPrefix + "/" + offHost + "/"
+ out := map[string]lease.MirrorObject{}
+ for k, o := range sw.list(sealIE, prefix) {
+ if rel := strings.TrimPrefix(k, prefix); rel != "lock.json" {
+ out[rel] = o
+ }
+ }
+ return out
+}
+
+func (sw *sealWorld) stagedRel() map[string]lease.MirrorObject {
+ out := map[string]lease.MirrorObject{}
+ for k, o := range sw.list(sealUK, activator.MovesPrefix(offHost)) {
+ out[strings.TrimPrefix(k, activator.MovesPrefix(offHost))] = o
+ }
+ return out
+}
+
+func sameSizes(t *testing.T, staged, replica map[string]lease.MirrorObject) {
+ t.Helper()
+ if len(staged) != len(replica) {
+ t.Fatalf("staged %d objects, replica %d", len(staged), len(replica))
+ }
+ for rel, o := range replica {
+ if s, ok := staged[rel]; !ok || s.Size != o.Size {
+ t.Fatalf("%s: staged %+v (%v), replica %+v", rel, s, ok, o)
+ }
+ }
+}
+
+func TestOfflineStageThenStartThenSealSkipsThePreCopy(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ ieB := sw.sourceBox("ie-b")
+ sw.instance(ie)
+ ctx := context.Background()
+ // A stale staged prefix an earlier aborted move left behind: a file
+ // under a replica key with other bytes, and one the replica never had.
+ // Neither may survive into the seal.
+ for rel := range sw.replicaRel() {
+ sw.fake.Put(sealUK, activator.MovesPrefix(offHost)+rel, []byte("stale bytes of an earlier move"))
+ break
+ }
+ sw.fake.Put(sealUK, activator.MovesPrefix(offHost)+"0000/stale-extra.ltx", []byte("stale"))
+ owner0, expires0, gen0 := sw.lock()
+
+ sw.stage()
+ spy := sw.spyCopies()
+ sw.stagerEdges(ie) // both boxes publish the capability,
+ sw.mustStagePass(ieB) // and ie-b is not the stager: it does nothing
+ if _, ok := sw.reportIf(moves.OfflineRoleStager, "ie-b"); ok || len(spy.take()) != 0 {
+ t.Fatal("a box that is not the stager staged")
+ }
+ sw.mustStagePass(ie)
+ first := spy.take()
+ replica := sw.replicaRel()
+ sameSizes(t, sw.stagedRel(), replica)
+ r := sw.report(moves.OfflineRoleStager, "ie-a")
+ if r.Phase != moves.OfflinePhaseStaged || r.ReplicaObjects != len(replica) || r.StagedObjects != len(replica) || r.RoundBytes == 0 || r.StagedBytes != r.ReplicaBytes || r.StageError != "" {
+ t.Fatalf("stager report %+v (replica %d objects)", r, len(replica))
+ }
+ if len(first) != len(replica) {
+ t.Fatalf("the first round made %d copies, want %d", len(first), len(replica))
+ }
+ if staged := sw.record().Move.StagedAt; staged == nil || !r.StageRequest.Equal(*staged) {
+ t.Fatalf("the round answered %v, the request is %v", r.StageRequest, staged)
+ }
+ // One round per request: an idle pass copies, lists and writes nothing.
+ snap := sw.fake.Snapshot()
+ sw.mustStagePass(ie)
+ if len(spy.take()) != 0 || sw.fake.Snapshot() != snap {
+ t.Fatal("a pass with no new stage request ran a round")
+ }
+ if again := sw.report(moves.OfflineRoleStager, "ie-a"); !again.At.Equal(r.At) {
+ t.Fatal("a pass with no new stage request rewrote the stager report")
+ }
+
+ // Staging held, fenced and leased nothing: the row serves, no fence,
+ // the lease is as it was, and a wake takes it.
+ if rt := ie.row(offHost); rt == nil || rt.HoldOp != "" {
+ t.Fatalf("row %+v during staging", rt)
+ }
+ if ie.svc.Fenced(offHost) {
+ t.Fatal("fenced during staging")
+ }
+ if owner, expires, gen := sw.lock(); owner != owner0 || !expires.Equal(expires0) || gen != gen0 {
+ t.Fatalf("the lease changed during staging: %s %s %d, was %s %s %d", owner, expires, gen, owner0, expires0, gen0)
+ }
+ h, err := acquireAsAWake(ctx)
+ if err != nil {
+ t.Fatalf("a wake during staging: %v", err)
+ }
+ h.Release(ctx)
+
+ // The instance keeps writing; a second round copies only what is new.
+ sw.writeRows(ie, sw.dbPath(ie), 20, 40, true)
+ before := sw.stagedRel()
+ sw.restage()
+ sw.mustStagePass(ie)
+ second := spy.take()
+ fresh := 0
+ for rel, o := range sw.replicaRel() {
+ if s, ok := before[rel]; !ok || s.Size != o.Size {
+ fresh++
+ }
+ }
+ if fresh == 0 || len(second) != fresh {
+ t.Fatalf("the second round made %d copies, want the %d new objects", len(second), fresh)
+ }
+ sameSizes(t, sw.stagedRel(), sw.replicaRel())
+
+ // A replica file that came and went between rounds (retention): the
+ // seal's round removes it from staging.
+ retired := sealPrefix + "/" + offHost + "/scratch/retired.bin"
+ sw.fake.Put(sealIE, retired, []byte("gone before the seal"))
+ sw.restage()
+ sw.mustStagePass(ie)
+ spy.take()
+ if _, ok := sw.stagedRel()["scratch/retired.bin"]; !ok {
+ t.Fatal("fixture: the retired file was not staged")
+ }
+ sw.fake.BareDelete(sealIE, retired)
+
+ // Start: the stage's objects are in place, and the seal's round copies
+ // only what changed since the last stage round.
+ preStaged := sw.stagedRel()
+ preReplica := sw.replicaRel()
+ sw.move(provision.OfflineStartStaged(offID, "op@example.com", sw.clock()))
+ ie.tick()
+ ieB.tick()
+ sw.move(provision.OfflineSeal(offID, "ie-a", []string{"ie-a", "ie-b"}))
+ spy.take()
+ sw.mustPass(ie)
+ if st := sw.record().Move.State; st != provision.OfflineSealed {
+ t.Fatalf("state %s, want sealed", st)
+ }
+ sealCopies := spy.take()
+ // Litestream's compaction decides how many files the checkpoint adds,
+ // so the count of the seal's copies is not fixed. What is: at least one
+ // object the stage copied is still in the replica, unchanged, and the
+ // seal copied none of those (the loop below).
+ kept := 0
+ for rel, o := range sw.replicaRel() {
+ if s, ok := preStaged[rel]; ok && s.Size == o.Size && preReplica[rel].ETag == o.ETag {
+ kept++
+ }
+ }
+ if kept == 0 {
+ t.Fatal("fixture: no pre-copied object survived to the seal, so nothing shows a skip")
+ }
+ t.Logf("the seal copied %d of %d objects; %d were pre-copied and skipped", len(sealCopies), len(sw.replicaRel()), kept)
+ for _, k := range sealCopies {
+ rel := strings.TrimPrefix(k, sealPrefix+"/"+offHost+"/")
+ s, staged := preStaged[rel]
+ o, listed := preReplica[rel]
+ if staged && listed && s.Size == o.Size && s.ETag == o.ETag {
+ t.Fatalf("the seal recopied %s, which the stage had copied", rel)
+ }
+ }
+ seal, _, ok := sw.sealRecord()
+ if !ok {
+ t.Fatal("no seal")
+ }
+ if _, ok := seal.Staged["scratch/retired.bin"]; ok {
+ t.Fatal("the seal kept a staged file the replica no longer has")
+ }
+ if _, ok := seal.Staged["0000/stale-extra.ltx"]; ok {
+ t.Fatal("the seal kept the stale prefix's extra file")
+ }
+ sameSizes(t, sw.stagedRel(), sw.replicaRel())
+ // The staged copy restores to every row written while staging.
+ restored := t.TempDir() + "/restored.db"
+ if ok, err := lease.Restore(ctx, restored, sealUK, strings.TrimSuffix(activator.MovesPrefix(offHost), "/"), sealUKRegion); err != nil || !ok {
+ t.Fatalf("restore the staged copy: %v %v", ok, err)
+ }
+ if n := countRows(t, restored); n != 40 {
+ t.Fatalf("the staged copy holds %d rows, want 40", n)
+ }
+}
+
+// Abort from staging: the stager deletes the staged copy and the receipts
+// before it reports reverted, every other source box reports reverted as
+// for any abort, and finish ends a move that was never offline. Nothing the
+// move did not write is touched.
+func TestOfflineAbortFromStagingRemovesTheStagedCopy(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.mustStagePass(ie)
+ if len(sw.stagedRel()) == 0 {
+ t.Fatal("fixture: nothing staged")
+ }
+ // Receipts, as a round over a multipart source leaves them, and
+ // neighbours the abort must not touch.
+ sw.fake.Put(sealIE, activator.MoveReceiptsKey(offHost), []byte(`{"v":1}`))
+ sw.fake.Put(sealUK, "moves/other.example.com/0000/x.ltx", []byte("x"))
+ replica := sw.replicaRel()
+
+ sw.abort()
+ sw.mustStagePass(ie)
+ if n := len(sw.stagedRel()); n != 0 {
+ t.Fatalf("%d staged objects left after the abort", n)
+ }
+ if _, ok := sw.fake.Current(sealIE, activator.MoveReceiptsKey(offHost)); ok {
+ t.Fatal("the receipts are still there")
+ }
+ if _, ok := sw.fake.Current(sealUK, "moves/other.example.com/0000/x.ltx"); !ok {
+ t.Fatal("the abort deleted another host's staging")
+ }
+ if got := sw.replicaRel(); len(got) != len(replica) {
+ t.Fatalf("the replica changed: %d objects, was %d", len(got), len(replica))
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-a"); r.Phase != moves.OfflinePhaseReverted {
+ t.Fatalf("stager report %+v", r)
+ }
+ if r := sw.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseReverted {
+ t.Fatalf("source report %+v", r)
+ }
+ // Again: nothing in the buckets changes.
+ snap := sw.fake.Snapshot()
+ sw.mustStagePass(ie)
+ if sw.fake.Snapshot() != snap {
+ t.Fatal("a second revert pass wrote to the buckets")
+ }
+ rec := sw.move(provision.OfflineFinish(offID, sw.clock()))
+ if rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != provision.OfflineOutcomeAborted || rec.LastMove.OfflineS != 0 {
+ t.Fatalf("finish: %+v", rec.LastMove)
+ }
+ if rt := ie.row(offHost); rt == nil || rt.HoldOp != "" {
+ t.Fatalf("row %+v after the abort", rt)
+ }
+}
+
+// A staging bucket that is the source's own would have the serving instance
+// restore from the staged copy (the activator's staging wins). The stage
+// door refuses it (MV-O191); a record that names it anyway (written by
+// hand) gets a refusing round that writes nothing, and a cleanup that
+// deletes and acknowledges nothing.
+func TestOfflineStageRefusesTheSourcesOwnBucket(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ sw.instance(ie)
+ sw.publishTarget(sealIE, sealIERegion)
+ sw.move(provision.OfflineStage(offID, "", "uk", "op@example.com", provision.StageTarget{Stager: "ie-a", Bucket: sealIE, Region: sealIERegion}, sw.clock()))
+ if err := sw.stagePass(ie); err == nil {
+ t.Fatal("staged into the source's own bucket")
+ }
+ if n := len(sw.list(sealIE, "moves/")); n != 0 {
+ t.Fatalf("%d objects staged into the source's bucket", n)
+ }
+ r := sw.report(moves.OfflineRoleStager, "ie-a")
+ if r.Phase != moves.OfflinePhaseFailed || !strings.Contains(r.StageError, "own bucket") {
+ t.Fatalf("stager report %+v", r)
+ }
+ sw.abort()
+ if err := sw.stagePass(ie); err == nil {
+ t.Fatal("an abort acknowledged a cleanup of the source's own bucket")
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-a"); r.Phase == moves.OfflinePhaseReverted {
+ t.Fatal("reverted written for a staging bucket that was never cleaned")
+ }
+}
+
+// Astra round 1, finding 1: a round that copied and then never wrote its
+// report (the box crashed, or the report write failed) still leaves its
+// copy to the abort. Cleanup follows the record's staged_at, never the
+// reports.
+func TestOfflineAbortCleansARoundThatNeverReported(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.stagerEdges(ie)
+ // The round's copies land; the box dies before its report.
+ if _, err := ie.w.offlineStageRound(context.Background(), sw.s, sw.record()); err != nil {
+ t.Fatal(err)
+ }
+ if len(sw.stagedRel()) == 0 {
+ t.Fatal("fixture: nothing staged")
+ }
+ if _, ok := sw.reportIf(moves.OfflineRoleStager, "ie-a"); ok {
+ t.Fatal("fixture: the round reported")
+ }
+ sw.abort()
+ sw.mustStagePass(ie)
+ if n := len(sw.stagedRel()); n != 0 {
+ t.Fatalf("%d staged objects left by a round that never reported", n)
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-a"); r.Phase != moves.OfflinePhaseReverted {
+ t.Fatalf("stager report %+v", r)
+ }
+}
+
+// Astra round 1, finding 2: a box that joins mid-move, lexically before the
+// stager, never stages and never cleans for it. Only the pinned stager
+// writes the staging prefix, so its cleanup, after its own last round, is
+// the one that counts.
+func TestOfflineAbortWaitsForThePinnedStager(t *testing.T) {
+ sw := newSealWorld(t)
+ sw.instance(sw.ie)
+ ieB := sw.sourceBox("ie-b")
+ sw.stageBy("ie-b")
+ sw.mustStagePass(ieB)
+ if r := sw.report(moves.OfflineRoleStager, "ie-b"); r.Phase != moves.OfflinePhaseStaged {
+ t.Fatalf("stager report %+v", r)
+ }
+ // ie-a joins, publishing the capability, and the operator stages again.
+ sw.stagerEdges(sw.ie)
+ sw.restage()
+ spy := sw.spyCopies()
+ sw.mustStagePass(sw.ie)
+ if _, ok := sw.reportIf(moves.OfflineRoleStager, "ie-a"); ok || len(spy.take()) != 0 {
+ t.Fatal("a box that is not the pinned stager staged")
+ }
+ sw.abort()
+ sw.mustStagePass(sw.ie) // ie-a's revert: not its copy to remove
+ if _, ok := sw.reportIf(moves.OfflineRoleStager, "ie-a"); ok {
+ t.Fatal("a box that is not the pinned stager acknowledged the cleanup")
+ }
+ // ie-b's round, which began before the abort, lands its copies now.
+ sw.fake.Put(sealIE, sealPrefix+"/"+offHost+"/scratch/late.bin", []byte("late"))
+ if _, err := ieB.w.offlineStageRound(context.Background(), sw.s, sw.record()); err != nil {
+ t.Fatal(err)
+ }
+ if len(sw.stagedRel()) == 0 {
+ t.Fatal("fixture: the late round staged nothing")
+ }
+ sw.mustStagePass(ieB) // the pinned stager's revert, after its last round
+ if n := len(sw.stagedRel()); n != 0 {
+ t.Fatalf("%d staged objects left after the pinned stager's cleanup", n)
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-b"); r.Phase != moves.OfflinePhaseReverted {
+ t.Fatalf("stager report %+v", r)
+ }
+}
+
+// ---- Astra round 2 ----
+
+// publishTarget republishes the target box's edges file with bucket in
+// region: a target replaced or reconfigured after the stage.
+func (sw *sealWorld) publishTarget(bucket, region string) {
+ sw.t.Helper()
+ var last []byte
+ if err := edgesFileWrite(context.Background(), sw.s, sw.uk.reg, edgesFile{Box: "uk-box", Placement: "uk", DataBucket: bucket, DataBucketRegion: region}, "", &last, false); err != nil {
+ sw.t.Fatal(err)
+ }
+}
+
+// Finding 1: once the move has left staging, the stager never starts a
+// round, even on a pass that read the record while it was still staging.
+func TestOfflineStageNoRoundAfterTheMoveLeftStaging(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.mustStagePass(ie)
+ sw.restage()
+ stale := sw.record() // a pass's listing, read while still staging
+ sw.move(provision.OfflineStartStaged(offID, "op@example.com", sw.clock()))
+ sw.fake.Put(sealIE, sealPrefix+"/"+offHost+"/scratch/new.bin", []byte("new"))
+ spy := sw.spyCopies()
+ if err := ie.w.offlineRound(context.Background(), sw.s, stale); err != nil {
+ t.Fatal(err)
+ }
+ if n := len(spy.take()); n != 0 {
+ t.Fatalf("a round made %d copies after the move left staging", n)
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-a"); r.StageRequest.Equal(*stale.Move.StagedAt) {
+ t.Fatal("the stager answered a request after the move left staging")
+ }
+}
+
+// Finding 1: a round the start overtakes deletes nothing. The delete phase
+// reads the record again first, and a move that has left staging stops the
+// round there: an object the seal's checkpoint added must never be removed
+// by a stage round.
+func TestOfflineStageRoundDeletesNothingOnceTheMoveLeftStaging(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.mustStagePass(ie)
+ // An object only the staging prefix has (the round would delete it),
+ // and a new replica object (the round copies it, and the start lands
+ // during that copy).
+ extra := activator.MovesPrefix(offHost) + "0000/added-by-the-seal.ltx"
+ sw.fake.Put(sealUK, extra, []byte("the seal's"))
+ sw.fake.Put(sealIE, sealPrefix+"/"+offHost+"/scratch/new.bin", []byte("new"))
+ sw.restage()
+ var once sync.Once
+ sw.fake.FailOn = func(op, key string) error {
+ if op == "copy" {
+ once.Do(func() {
+ go func() { // the operator's start, while the copy is in flight
+ _, _ = provision.SetRecordMove(context.Background(), sw.s, sw.acct, "app", offHost,
+ provision.OfflineStartStaged(offID, "op@example.com", time.Now()))
+ }()
+ time.Sleep(200 * time.Millisecond)
+ })
+ }
+ return nil
+ }
+ t.Cleanup(func() { sw.fake.FailOn = nil })
+ _ = ie.w.offlineRound(context.Background(), sw.s, sw.record())
+ if sw.record().Move.State != provision.OfflineStopping {
+ t.Fatal("fixture: the start did not land during the round")
+ }
+ if _, ok := sw.fake.Current(sealUK, extra); !ok {
+ t.Fatal("a stage round deleted from the staging prefix after the move left staging")
+ }
+}
+
+// Finding 1: in stopping and on, the pinned stager says, on every pass and
+// under its pass lock, that no round is in flight and none will start.
+func TestOfflineStagerAcknowledgesItHasStopped(t *testing.T) {
+ sw := newSealWorld(t)
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.mustStagePass(ie)
+ sw.move(provision.OfflineStartStaged(offID, "op@example.com", sw.clock()))
+ ie.tick()
+ r := sw.report(moves.OfflineRoleStager, "ie-a")
+ if r.Phase != moves.OfflinePhaseStopped || !r.At.Equal(sw.clock()) || r.StagedObjects == 0 {
+ t.Fatalf("stager report %+v, want stopped now with its progress kept", r)
+ }
+}
+
+// Finding 2: cleanup deletes from the destination the stage fixed on the
+// record, whatever the target publishes now, and says reverted only then.
+func TestOfflineAbortCleansTheStoredDestination(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ bucket, region string
+ }{
+ {"a replaced target", "data-uk-new", sealUKRegion},
+ {"a target that now publishes the source's own bucket", sealIE, sealIERegion},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ sw := newSealWorld(t)
+ sw.fake.AddBucket("data-uk-new")
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.mustStagePass(ie)
+ if len(sw.stagedRel()) == 0 {
+ t.Fatal("fixture: nothing staged")
+ }
+ sw.publishTarget(c.bucket, c.region)
+ sw.abort()
+ sw.mustStagePass(ie)
+ if n := len(sw.stagedRel()); n != 0 {
+ t.Fatalf("%d objects left in the stored staging bucket %s", n, sealUK)
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-a"); r.Phase != moves.OfflinePhaseReverted {
+ t.Fatalf("stager report %+v", r)
+ }
+ if got := sw.replicaRel(); len(got) == 0 {
+ t.Fatal("the cleanup touched the source's replica")
+ }
+ })
+ }
+}
+
+// Finding 2: a stage round never follows a target that now publishes other
+// coordinates than the stored ones. It refuses and writes nowhere.
+func TestOfflineStageRoundRefusesChangedCoordinates(t *testing.T) {
+ sw := newSealWorld(t)
+ sw.fake.AddBucket("data-uk-new")
+ ie := sw.ie
+ sw.instance(ie)
+ sw.stage()
+ sw.publishTarget("data-uk-new", sealUKRegion)
+ if err := sw.stagePass(ie); err == nil {
+ t.Fatal("staged with changed coordinates")
+ }
+ if n := len(sw.list("data-uk-new", "moves/")); n != 0 {
+ t.Fatalf("%d objects staged into the new bucket", n)
+ }
+ if n := len(sw.stagedRel()); n != 0 {
+ t.Fatalf("%d objects staged at all", n)
+ }
+ if r := sw.report(moves.OfflineRoleStager, "ie-a"); r.Phase != moves.OfflinePhaseFailed {
+ t.Fatalf("stager report %+v", r)
+ }
+}
diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go
index b44a184d9..dcc39c6f8 100644
--- a/internal/host/movesoffline_test.go
+++ b/internal/host/movesoffline_test.go
@@ -933,11 +933,11 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
if !armedOfflineCapable() {
t.Fatal("a capable armed worker does not publish")
}
- if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1}) {
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1, capOfflineStage1}) {
t.Fatalf("capable box: %v", got)
}
base := edgesFileCapabilities(env("CARLOS_REGION_HA", "1"))
- if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1}) || !slices.Equal(base, []string{capRegionHA1}) {
+ if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1, capOfflineStage1}) || !slices.Equal(base, []string{capRegionHA1}) {
t.Fatalf("region HA too: %v (base %v)", got, base)
}
for _, kv := range [][]string{
@@ -952,7 +952,7 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
}
incapable := wdCap.box("incapable", "", map[string]string{kv[0]: kv[1]})
armedMovesWorker.Store(incapable.w)
- if got := withOfflineCapability(edgesFileCapabilities(env(kv...)), armedOfflineCapable()); slices.Contains(got, capOfflineMove1) {
+ if got := withOfflineCapability(edgesFileCapabilities(env(kv...)), armedOfflineCapable()); slices.Contains(got, capOfflineMove1) || slices.Contains(got, capOfflineStage1) {
t.Fatalf("%s=%q publishes: %v", kv[0], kv[1], got)
}
}
@@ -1128,7 +1128,7 @@ func TestOfflineDormantIsMain(t *testing.T) {
if json.Unmarshal(b1, &m1) != nil || json.Unmarshal(b2, &m2) != nil {
t.Fatal("edges files do not parse")
}
- if caps, _ := m2["capabilities"].([]any); len(caps) != 1 || caps[0] != capOfflineMove1 {
+ if caps, _ := m2["capabilities"].([]any); len(caps) != 2 || caps[0] != capOfflineMove1 || caps[1] != capOfflineStage1 {
t.Fatalf("capabilities: %v", m2["capabilities"])
}
delete(m2, "capabilities")
diff --git a/internal/host/movespass_test.go b/internal/host/movespass_test.go
index e93de9a1a..1edc0acd6 100644
--- a/internal/host/movespass_test.go
+++ b/internal/host/movespass_test.go
@@ -25,7 +25,14 @@ import (
// a single-part object's ETag is its body, a multipart one's is
// body+"-27", and a copy lands single-part with the body as its ETag. An
// op log records ordering.
+//
+// A mirror round runs its copies and deletes side by side
+// (lease.DefaultMirrorWorkers), so buckets and ops are guarded by bmu
+// whenever a round can be running; a test reads them directly between
+// rounds. beforeCopy runs without the lock, so it may call the fake. Lock
+// order: mu, then bmu.
type fakeMoveMirror struct {
+ bmu sync.Mutex
buckets map[string]map[string]fakeMoveObject
ops []string
beforeCopy func(srcKey string)
@@ -62,12 +69,20 @@ func (f *fakeMoveMirror) putMultipart(bucket, key, body string, size int64) {
f.store(bucket, key, fakeMoveObject{lease.MirrorObject{ETag: body + "-27", Size: size}, body})
}
func (f *fakeMoveMirror) store(bucket, key string, o fakeMoveObject) {
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
+ f.storeLocked(bucket, key, o)
+}
+
+func (f *fakeMoveMirror) storeLocked(bucket, key string, o fakeMoveObject) {
if f.buckets[bucket] == nil {
f.buckets[bucket] = map[string]fakeMoveObject{}
}
f.buckets[bucket][key] = o
}
func (f *fakeMoveMirror) List(_ context.Context, bucket, prefix string) (map[string]lease.MirrorObject, error) {
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
out := map[string]lease.MirrorObject{}
for k, v := range f.buckets[bucket] {
if strings.HasPrefix(k, prefix) {
@@ -77,18 +92,25 @@ func (f *fakeMoveMirror) List(_ context.Context, bucket, prefix string) (map[str
return out, nil
}
func (f *fakeMoveMirror) Copy(_ context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {
- if f.beforeCopy != nil {
- f.beforeCopy(srcKey)
- }
+ f.bmu.Lock()
+ before := f.beforeCopy
+ f.bmu.Unlock()
+ if before != nil {
+ before(srcKey)
+ }
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
o, ok := f.buckets[srcBucket][srcKey]
if !ok || (srcETag != "" && o.ETag != srcETag) {
return "", lease.ErrCopySourceChanged
}
f.ops = append(f.ops, "copy "+dstBucket+"/"+dstKey)
- f.store(dstBucket, dstKey, fakeMoveObject{lease.MirrorObject{ETag: o.body, Size: o.Size}, o.body})
+ f.storeLocked(dstBucket, dstKey, fakeMoveObject{lease.MirrorObject{ETag: o.body, Size: o.Size}, o.body})
return o.body, nil
}
func (f *fakeMoveMirror) Delete(_ context.Context, bucket, key string) error {
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
f.ops = append(f.ops, "delete "+bucket+"/"+key)
delete(f.buckets[bucket], key)
return nil
@@ -99,6 +121,8 @@ func (f *fakeMoveMirror) Get(_ context.Context, bucket, key string) ([]byte, err
return nil, err
}
}
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
o, ok := f.buckets[bucket][key]
if !ok {
return nil, lease.ErrMirrorNotFound
@@ -106,14 +130,18 @@ func (f *fakeMoveMirror) Get(_ context.Context, bucket, key string) ([]byte, err
return []byte(o.body), nil
}
func (f *fakeMoveMirror) Put(_ context.Context, bucket, key string, body []byte) error {
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
f.ops = append(f.ops, "put "+bucket+"/"+key)
- f.store(bucket, key, fakeMoveObject{lease.MirrorObject{ETag: "receipts", Size: int64(len(body))}, string(body)})
+ f.storeLocked(bucket, key, fakeMoveObject{lease.MirrorObject{ETag: "receipts", Size: int64(len(body))}, string(body)})
return nil
}
// bodies is a prefix's content by relative key: what a byte-exact
// comparison compares (ETags differ across a multipart copy by design).
func (f *fakeMoveMirror) bodies(bucket, prefix string) map[string]string {
+ f.bmu.Lock()
+ defer f.bmu.Unlock()
out := map[string]string{}
for k, v := range f.buckets[bucket] {
if strings.HasPrefix(k, prefix) {
@@ -804,10 +832,10 @@ func TestMovesPassDeferredBytesCountAgainstTheBar(t *testing.T) {
h.moveRound(t, host)
big := "instances/" + host + "/ltx/1/0.ltx"
h.f.putMultipart("data-ie", big, "level-0b", 138<<20)
+ var once sync.Once
h.f.beforeCopy = func(k string) {
if k == big {
- h.f.putMultipart("data-ie", big, "level-0c", 138<<20)
- h.f.beforeCopy = nil
+ once.Do(func() { h.f.putMultipart("data-ie", big, "level-0c", 138<<20) })
}
}
if st := h.moveRound(t, host); st != provision.MovePreparing {
@@ -864,10 +892,10 @@ func TestMovesPassCutStopsOnADeferredCopy(t *testing.T) {
t.Fatal(err)
}
big := "instances/" + host + "/ltx/1/3.ltx"
+ var once sync.Once
h.f.beforeCopy = func(k string) {
if k == big {
- h.f.putMultipart("data-ie", big, "level-3b", 138<<20)
- h.f.beforeCopy = nil
+ once.Do(func() { h.f.putMultipart("data-ie", big, "level-3b", 138<<20) })
}
}
if err := movesPass(context.Background(), h.s, h.worker()); err != nil {
diff --git a/internal/host/movesphase1_harness_test.go b/internal/host/movesphase1_harness_test.go
index 6e91d0118..b3b0f814f 100644
--- a/internal/host/movesphase1_harness_test.go
+++ b/internal/host/movesphase1_harness_test.go
@@ -76,7 +76,10 @@ func (f *fakeMoveMirror) putAt(op string, signAt time.Time, bucket, key string,
if err := f.logAt(op, signAt, bucket, key); err != nil {
return "", err
}
- if _, ok := f.buckets[bucket][key]; ok && createOnly {
+ f.bmu.Lock()
+ _, exists := f.buckets[bucket][key]
+ f.bmu.Unlock()
+ if exists && createOnly {
return "", lease.ErrExists
}
ver := f.nextVersion()
@@ -99,7 +102,9 @@ func (f *fakeMoveMirror) setVersion(bucket, key, ver string) {
if f.history == nil {
f.history = map[string]fakeMoveObject{}
}
+ f.bmu.Lock()
f.history[bucket+"/"+key+"@"+ver] = f.buckets[bucket][key]
+ f.bmu.Unlock()
}
}
diff --git a/internal/lease/mirror.go b/internal/lease/mirror.go
index 2aeb7bcb6..a1aa65870 100644
--- a/internal/lease/mirror.go
+++ b/internal/lease/mirror.go
@@ -223,6 +223,10 @@ func MirrorPrefixKeep(ctx context.Context, m MirrorStore, srcBucket, srcPrefix,
if versioned {
round.Versions = map[string]string{}
}
+ // The plan: which objects this round copies. Deciding a skip reads only
+ // the two listings and the receipts, so it is done here, in key order,
+ // before any copy starts; the copies then run side by side.
+ var jobs []string
for _, r := range rel {
// The lease's lock.json is COORDINATION state, not data (live
// rehearsal catch, 2026-08-29): the renewer rewrites it on every
@@ -233,9 +237,8 @@ func MirrorPrefixKeep(ctx context.Context, m MirrorStore, srcBucket, srcPrefix,
if r == leaseLockName {
continue
}
- sk, dk := srcPrefix+r, dstPrefix+r
- s := src[sk]
- d, staged := dst[dk]
+ s := src[srcPrefix+r]
+ d, staged := dst[dstPrefix+r]
if staged && d.ETag == s.ETag && !versioned {
continue
}
@@ -257,23 +260,50 @@ func MirrorPrefixKeep(ctx context.Context, m MirrorStore, srcBucket, srcPrefix,
continue
}
}
+ jobs = append(jobs, r)
+ }
+ // The copies, up to mirrorWorkers(m) at once (#175 offline-move
+ // speedup: one server-side copy at a time took six minutes for a 1.6 GB
+ // replica while the instance was offline). Every update to round and
+ // to the book is made under mu, in whatever order the copies finish;
+ // neither depends on that order. The first failure stops the round: no
+ // further copy starts, the ones in flight are cancelled, and every copy
+ // that did land keeps its receipt (the best-effort save below).
+ //
+ // No network request is ever made under mu (Astra round 1, finding 3):
+ // a receipts read that stalled there would hold every other worker,
+ // the first failure's included, so the round could not stop. The book
+ // is read here, before any copy, whenever this round copies anything
+ // and keeps receipts; set and drop then touch memory only. (A round
+ // that copies nothing still reads no receipts.)
+ if len(jobs) > 0 {
+ if err := book.load(ctx); err != nil {
+ return round, err
+ }
+ }
+ var mu sync.Mutex
+ err = forEachParallel(ctx, mirrorWorkers(m), len(jobs), func(cctx context.Context, i int) error {
+ r := jobs[i]
+ sk, dk := srcPrefix+r, dstPrefix+r
+ s := src[sk]
var etag, ver string
var err error
if versioned {
- etag, ver, err = vc.CopyVersioned(ctx, srcBucket, sk, s.ETag, dstBucket, dk)
+ etag, ver, err = vc.CopyVersioned(cctx, srcBucket, sk, s.ETag, dstBucket, dk)
} else {
- etag, err = m.Copy(ctx, srcBucket, sk, s.ETag, dstBucket, dk)
+ etag, err = m.Copy(cctx, srcBucket, sk, s.ETag, dstBucket, dk)
}
+ mu.Lock()
+ defer mu.Unlock()
if errors.Is(err, ErrCopySourceChanged) {
book.drop(r)
round.Changed++
round.Deferred++
round.BytesDeferred += s.Size
- continue
+ return nil
}
if err != nil {
- book.save(ctx) // best effort: what already landed keeps its receipts
- return round, fmt.Errorf("mirror: copy %s: %w", sk, err)
+ return fmt.Errorf("mirror: copy %s: %w", sk, err)
}
if !UsableVersionID(ver) {
ver = "" // never recorded or sealed by
@@ -288,29 +318,45 @@ func MirrorPrefixKeep(ctx context.Context, m MirrorStore, srcBucket, srcPrefix,
if dstETag == "" {
dstETag = s.ETag
}
- // A failed read here loses this receipt (one recopy next
- // round), never a wrong skip.
- if err := book.set(ctx, r, copyReceipt{SrcETag: s.ETag, SrcSize: s.Size, DstETag: dstETag, DstVersionID: ver}); err != nil {
- return round, err
- }
- } else {
- book.drop(r)
+ // The book was read before the copies began, so this is a
+ // write to memory, never a request under mu.
+ return book.set(ctx, r, copyReceipt{SrcETag: s.ETag, SrcSize: s.Size, DstETag: dstETag, DstVersionID: ver})
}
+ book.drop(r)
+ return nil
+ })
+ if err != nil {
+ book.save(ctx) // best effort: what already landed keeps its receipts
+ return round, err
}
+ // The deletes, only once every copy has landed (as before), and side
+ // by side among themselves: their order never mattered.
+ var gone []string
for dk := range dst {
r := dk[len(dstPrefix):]
if r == leaseLockName {
continue
}
if _, ok := src[srcPrefix+r]; !ok {
- if err := m.Delete(ctx, dstBucket, dk); err != nil {
- book.save(ctx)
- return round, fmt.Errorf("mirror: delete %s: %w", dk, err)
- }
- book.drop(r)
- round.Changed++
+ gone = append(gone, r)
}
}
+ sort.Strings(gone)
+ err = forEachParallel(ctx, mirrorWorkers(m), len(gone), func(cctx context.Context, i int) error {
+ r := gone[i]
+ if err := m.Delete(cctx, dstBucket, dstPrefix+r); err != nil {
+ return fmt.Errorf("mirror: delete %s: %w", dstPrefix+r, err)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ book.drop(r)
+ round.Changed++
+ return nil
+ })
+ if err != nil {
+ book.save(ctx)
+ return round, err
+ }
book.prune(func(r string) bool { _, ok := src[srcPrefix+r]; return ok })
if err := book.save(ctx); err != nil {
return round, err
@@ -318,6 +364,98 @@ func MirrorPrefixKeep(ctx context.Context, m MirrorStore, srcBucket, srcPrefix,
return round, nil
}
+// DefaultMirrorWorkers is how many copies (and, after them, deletes) one
+// delta round runs at once. Each is one server-side request, so the round's
+// wall time is the slowest request of each batch rather than their sum.
+const DefaultMirrorWorkers = 8
+
+// MirrorConcurrency is the optional half of a MirrorStore that sets how many
+// requests a round runs at once. A store that does not implement it gets
+// DefaultMirrorWorkers; an answer below one is one (a strictly serial
+// round, as every round was before). Tests use it; production stores do not.
+type MirrorConcurrency interface {
+ MirrorWorkers() int
+}
+
+func mirrorWorkers(m MirrorStore) int {
+ if c, ok := m.(MirrorConcurrency); ok {
+ if n := c.MirrorWorkers(); n >= 1 {
+ return n
+ }
+ return 1
+ }
+ return DefaultMirrorWorkers
+}
+
+// forEachParallel calls do for every index below n, on up to workers
+// goroutines, starting them in index order. The first error do returns
+// stops it: no further index starts, the context handed to the calls still
+// in flight is cancelled, and once they have returned that first error is
+// the answer (any error a cancelled call returns after it is dropped). A ctx
+// that ends before every index has started is its own error. Calls that
+// were in flight when it stopped have returned by the time it does, so the
+// caller's state is final.
+func forEachParallel(ctx context.Context, workers, n int, do func(ctx context.Context, i int) error) error {
+ if n == 0 {
+ return nil
+ }
+ if workers < 1 {
+ workers = 1
+ }
+ if workers > n {
+ workers = n
+ }
+ cctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ var (
+ mu sync.Mutex
+ first error
+ wg sync.WaitGroup
+ )
+ next := make(chan int)
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for i := range next {
+ if err := do(cctx, i); err != nil {
+ mu.Lock()
+ if first == nil {
+ first = err
+ cancel()
+ }
+ mu.Unlock()
+ }
+ }
+ }()
+ }
+ started := 0
+feed:
+ for ; started < n; started++ {
+ // Checked first: a select with both cases ready picks at random,
+ // and a stopped round must not start another copy.
+ if cctx.Err() != nil {
+ break
+ }
+ select {
+ case next <- started:
+ case <-cctx.Done():
+ break feed
+ }
+ }
+ close(next)
+ wg.Wait()
+ if first != nil {
+ return first
+ }
+ if started < n {
+ if err := ctx.Err(); err != nil {
+ return fmt.Errorf("mirror: %w", err)
+ }
+ }
+ return nil
+}
+
// receiptLedger is one round's view of the receipts object. It reads the
// object lazily, on the first object whose ETags differ, so a round over
// single-part objects only costs no request it did not cost before.
diff --git a/internal/lease/mirror_parallel_test.go b/internal/lease/mirror_parallel_test.go
new file mode 100644
index 000000000..27bec5312
--- /dev/null
+++ b/internal/lease/mirror_parallel_test.go
@@ -0,0 +1,427 @@
+package lease
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// The parallel round (offline-move speedup): copies and deletes run up to
+// mirrorWorkers(m) at once. These tests hold the round to what the serial
+// one promised: the same end state, receipts for exactly what landed, the
+// first failure stopping the round, and no skip a receipt cannot prove.
+// Run them under -race: the round's own state is shared by its workers.
+
+// fillReplica stores n objects under instances/h/: every third one
+// multipart (the copy lands with a new ETag, so it needs a receipt), the
+// rest single-part. Sizes differ so byte totals catch a double count.
+func fillReplica(f *fakeMirror, n int, gen string) (keys []string, bytes int64) {
+ for i := 0; i < n; i++ {
+ key := fmt.Sprintf("instances/h/ltx/0/%s-%04d.ltx", gen, i)
+ size := int64(1000 + i)
+ if i%3 == 0 {
+ f.putMultipart("src", key, fmt.Sprintf("%s-body-%d", gen, i), size)
+ } else {
+ f.putSized("src", key, fmt.Sprintf("%s-etag-%d", gen, i), size)
+ }
+ keys = append(keys, key)
+ bytes += size
+ }
+ return keys, bytes
+}
+
+// stagedBodies is moves/h/ by key relative to it, body for body.
+func stagedBodies(f *fakeMirror) map[string]string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ out := map[string]string{}
+ for k, v := range f.buckets["dst"] {
+ if strings.HasPrefix(k, "moves/h/") {
+ out[strings.TrimPrefix(k, "moves/h/")] = v.body
+ }
+ }
+ return out
+}
+
+// sourceBodies is instances/h/ the same way, lock.json aside.
+func sourceBodies(f *fakeMirror) map[string]string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ out := map[string]string{}
+ for k, v := range f.buckets["src"] {
+ if rel, ok := strings.CutPrefix(k, "instances/h/"); ok && rel != "lock.json" {
+ out[rel] = v.body
+ }
+ }
+ return out
+}
+
+func sameBodies(t *testing.T, got, want map[string]string) {
+ t.Helper()
+ if len(got) != len(want) {
+ t.Fatalf("staged %d objects, source has %d", len(got), len(want))
+ }
+ for k, v := range want {
+ if got[k] != v {
+ t.Fatalf("staged %s = %q, source %q", k, got[k], v)
+ }
+ }
+}
+
+func copiesIn(ops []string) int {
+ n := 0
+ for _, op := range ops {
+ if strings.HasPrefix(op, "copy ") {
+ n++
+ }
+ }
+ return n
+}
+
+// A parallel round ends where a serial one does: every object staged body
+// for body, the same counts and bytes, receipts for exactly the multipart
+// objects, and a caught-up round after it. It really runs side by side,
+// and never more than the store allows.
+func TestMirrorPrefixParallelMatchesSerial(t *testing.T) {
+ const n = 120
+ var rounds [2]MirrorRound
+ var books [2]map[string]copyReceipt
+ for i, workers := range []int{1, 6} {
+ f := newFakeMirror()
+ f.workers = workers
+ _, bytes := fillReplica(f, n, "a")
+ f.put("src", "instances/h/lock.json", "renewal")
+ // Slow copies, so the workers overlap.
+ f.onCopy = func(context.Context, string) error { time.Sleep(time.Millisecond); return nil }
+ r := mirrorRound(t, f, testReceipts)
+ if r.Changed != n || r.BytesCopied != bytes || r.Deferred != 0 {
+ t.Fatalf("workers=%d: round = %+v, want %d copies of %d bytes", workers, r, n, bytes)
+ }
+ sameBodies(t, stagedBodies(f), sourceBodies(f))
+ if f.peak > workers || (workers > 1 && f.peak < 2) {
+ t.Fatalf("workers=%d: peak concurrency %d", workers, f.peak)
+ }
+ book := readReceipts(t, f)
+ if len(book) != n/3 {
+ t.Fatalf("workers=%d: %d receipts, want one per multipart object (%d)", workers, len(book), n/3)
+ }
+ if again := mirrorRound(t, f, testReceipts); again.Changed != 0 {
+ t.Fatalf("workers=%d: caught-up round = %+v", workers, again)
+ }
+ rounds[i], books[i] = r, book
+ }
+ if rounds[0].Changed != rounds[1].Changed || rounds[0].BytesCopied != rounds[1].BytesCopied {
+ t.Fatalf("serial %+v and parallel %+v rounds differ", rounds[0], rounds[1])
+ }
+ for k, v := range books[0] {
+ if books[1][k] != v {
+ t.Fatalf("receipt %s: serial %+v, parallel %+v", k, v, books[1][k])
+ }
+ }
+}
+
+// The versioned mode under workers: every copied object's version lands in
+// Versions, and the receipts carry them, so the next round names them all
+// without copying.
+func TestMirrorPrefixParallelVersionedRound(t *testing.T) {
+ f := &fakeVersionedMirror{fakeMirror: newFakeMirror()}
+ f.workers = 8
+ keys, _ := fillReplica(f.fakeMirror, 64, "a")
+ r1, err := MirrorPrefix(context.Background(), f, "src", "instances/h/", "dst", "moves/h/", testReceipts)
+ if err != nil || len(r1.Versions) != len(keys) {
+ t.Fatalf("round 1 = %d versions, %v; want %d", len(r1.Versions), err, len(keys))
+ }
+ seen := map[string]bool{}
+ for _, v := range r1.Versions {
+ if seen[v] {
+ t.Fatalf("version %s handed to two copies", v)
+ }
+ seen[v] = true
+ }
+ r2, err := MirrorPrefix(context.Background(), f, "src", "instances/h/", "dst", "moves/h/", testReceipts)
+ if err != nil || r2.Changed != 0 || len(r2.Versions) != len(keys) {
+ t.Fatalf("round 2 = %+v, %v; want nothing copied and every version named", r2, err)
+ }
+ for k, v := range r1.Versions {
+ if r2.Versions[k] != v {
+ t.Fatalf("%s: round 2 names %q, round 1 copied %q", k, r2.Versions[k], v)
+ }
+ }
+}
+
+// The first failure stops the round: it is the error, copies still in
+// flight see their context cancelled, nothing new starts, and every copy
+// that landed keeps its receipt, so the next round copies only what did
+// not land.
+func TestMirrorPrefixParallelFailureMidRound(t *testing.T) {
+ f := newFakeMirror()
+ f.workers = 4
+ // All multipart: a landed copy is skipped next round only by its receipt.
+ for i := 0; i < 40; i++ {
+ f.putMultipart("src", fmt.Sprintf("instances/h/ltx/1/%04d.ltx", i), fmt.Sprintf("b%d", i), 100)
+ }
+ bad := "instances/h/ltx/1/0010.ltx"
+ var cancelled atomic.Int32
+ later := make(chan struct{}, 64) // a copy after the bad one has started
+ f.onCopy = func(ctx context.Context, key string) error {
+ switch {
+ case key == bad:
+ // Fail only once a later copy is in flight, so the cancel
+ // has something to reach.
+ select {
+ case <-later:
+ case <-time.After(5 * time.Second):
+ }
+ return errors.New("injected copy failure")
+ case key > bad:
+ later <- struct{}{}
+ select {
+ case <-ctx.Done():
+ cancelled.Add(1)
+ return ctx.Err()
+ case <-time.After(5 * time.Second):
+ return errors.New("the round did not cancel a copy in flight")
+ }
+ }
+ return nil
+ }
+ r, err := MirrorPrefix(context.Background(), f, "src", "instances/h/", "dst", "moves/h/", testReceipts)
+ if err == nil || !strings.Contains(err.Error(), bad) || errors.Is(err, context.Canceled) {
+ t.Fatalf("round error = %v, want the injected failure on %s", err, bad)
+ }
+ landed := stagedBodies(f)
+ if r.Changed != len(landed) || r.BytesCopied != int64(len(landed))*100 {
+ t.Fatalf("round = %+v, but %d objects landed", r, len(landed))
+ }
+ if len(landed) == 0 || len(landed) > 10 {
+ t.Fatalf("%d objects landed, want those before the failure only", len(landed))
+ }
+ if cancelled.Load() == 0 {
+ t.Fatal("no copy in flight was cancelled")
+ }
+ // Nothing past the workers in flight at the failure ever started.
+ if started := copiesIn(f.ops) + int(cancelled.Load()) + 1; started > 10+1+f.workers {
+ t.Fatalf("%d copies started, want the round to stop dispatching at the failure", started)
+ }
+ book := readReceipts(t, f)
+ for rel := range landed {
+ if _, ok := book[rel]; !ok {
+ t.Fatalf("%s landed without a receipt", rel)
+ }
+ }
+ if len(book) != len(landed) {
+ t.Fatalf("%d receipts for %d landed copies", len(book), len(landed))
+ }
+ f.onCopy = nil
+ before := len(f.ops)
+ next := mirrorRound(t, f, testReceipts)
+ if next.Changed != 40-len(landed) {
+ t.Fatalf("next round = %+v, want exactly the %d that did not land", next, 40-len(landed))
+ }
+ for _, op := range f.ops[before:] {
+ for rel := range landed {
+ if strings.HasPrefix(op, "copy instances/h/"+rel+" ") {
+ t.Fatalf("a landed copy was recopied: %s", op)
+ }
+ }
+ }
+ sameBodies(t, stagedBodies(f), sourceBodies(f))
+}
+
+// A caller's context that ends mid-round is the round's error, never a
+// silent partial success.
+func TestMirrorPrefixParallelContextEnds(t *testing.T) {
+ f := newFakeMirror()
+ f.workers = 2
+ fillReplica(f, 30, "a")
+ ctx, cancel := context.WithCancel(context.Background())
+ var n atomic.Int32
+ f.onCopy = func(context.Context, string) error {
+ if n.Add(1) == 3 {
+ cancel()
+ }
+ return nil
+ }
+ if _, err := MirrorPrefix(ctx, f, "src", "instances/h/", "dst", "moves/h/", testReceipts); !errors.Is(err, context.Canceled) {
+ t.Fatalf("round error = %v, want the context's", err)
+ }
+ if len(stagedBodies(f)) == 30 {
+ t.Fatal("every copy ran after the context ended")
+ }
+}
+
+// The pre-copy (offline move stage): rounds while the instance serves, then
+// the seal's round. The seal finds what stage copied and copies only what is
+// new since; its receipts prove every multipart skip.
+func TestMirrorPrefixReceiptsSkipAfterAPreCopy(t *testing.T) {
+ f := newFakeMirror()
+ _, staged := fillReplica(f, 90, "a")
+ if r := mirrorRound(t, f, testReceipts); r.BytesCopied != staged {
+ t.Fatalf("stage round = %+v, want %d bytes", r, staged)
+ }
+ // The instance kept writing: new LTX files.
+ _, fresh := fillReplica(f, 6, "b")
+ before := len(f.ops)
+ seal := mirrorRound(t, f, testReceipts)
+ if seal.BytesCopied != fresh || seal.Changed != 6 || seal.Deferred != 0 {
+ t.Fatalf("seal round = %+v, want only the 6 new objects (%d bytes)", seal, fresh)
+ }
+ for _, op := range f.ops[before:] {
+ if strings.HasPrefix(op, "copy ") && !strings.Contains(op, "/b-") {
+ t.Fatalf("the seal recopied a pre-copied object: %s", op)
+ }
+ }
+ sameBodies(t, stagedBodies(f), sourceBodies(f))
+}
+
+// Between stage and seal, litestream's retention and compaction delete
+// replica files and write new ones. The seal's round deletes what the source
+// no longer has and drops those receipts, so the staged listing is the
+// source's again (the seal's sameListing).
+func TestMirrorPrefixSourceDeletionsBetweenStageAndSeal(t *testing.T) {
+ f := newFakeMirror()
+ keys, _ := fillReplica(f, 30, "a")
+ mirrorRound(t, f, testReceipts)
+ // Compaction: twelve level-0 files folded into one level-1 file.
+ for _, k := range keys[:12] {
+ f.remove("src", k)
+ }
+ f.putMultipart("src", "instances/h/ltx/1/compacted.ltx", "compacted", 12000)
+ r := mirrorRound(t, f, testReceipts)
+ if r.Changed != 13 || r.BytesCopied != 12000 {
+ t.Fatalf("seal round = %+v, want 12 deletes and the one copy", r)
+ }
+ sameBodies(t, stagedBodies(f), sourceBodies(f))
+ for rel := range readReceipts(t, f) {
+ if _, ok := sourceBodies(f)[rel]; !ok {
+ t.Fatalf("a receipt outlived its source object: %s", rel)
+ }
+ }
+ if again := mirrorRound(t, f, testReceipts); again.Changed != 0 {
+ t.Fatalf("round after = %+v", again)
+ }
+}
+
+// A staged prefix left by an earlier move is never trusted as such. A stale
+// object skips only when its ETag equals the source's (the same bytes) or a
+// receipt for this exact pair still matches both sides; anything else is
+// copied again, and every key the source lacks is deleted.
+func TestMirrorPrefixStaleStagedPrefixIsNotTrusted(t *testing.T) {
+ f := newFakeMirror()
+ f.putMultipart("src", "instances/h/ltx/1/a.ltx", "new-a", 500)
+ f.putMultipart("src", "instances/h/ltx/1/b.ltx", "new-b", 500)
+ f.putSized("src", "instances/h/ltx/0/c.ltx", "new-c", 50)
+ f.putSized("src", "instances/h/ltx/0/d.ltx", "same-d", 50)
+ // The stale prefix: a under the old move's copy of other bytes, b with
+ // the same size as the source and a stale receipt, c changed, d byte
+ // identical, and an extra the source never had.
+ f.putSized("dst", "moves/h/ltx/1/a.ltx", "old-a", 500)
+ f.putSized("dst", "moves/h/ltx/1/b.ltx", "old-b", 500)
+ f.putSized("dst", "moves/h/ltx/0/c.ltx", "old-c", 50)
+ f.putSized("dst", "moves/h/ltx/0/d.ltx", "same-d", 50)
+ f.putSized("dst", "moves/h/ltx/0/extra.ltx", "old-x", 50)
+ // An old book for this pair whose receipt for b names the source as it
+ // is now but a destination ETag the staged object does not carry.
+ stale := (`{"v":1,"src_bucket":"src","src_prefix":"instances/h/","dst_bucket":"dst","dst_prefix":"moves/h/",` +
+ `"objects":{"ltx/1/b.ltx":{"src_etag":"new-b-27","src_size":500,"dst_etag":"new-b"},"ltx/0/extra.ltx":{"src_etag":"x","src_size":50,"dst_etag":"old-x"}}}`)
+ f.store("src", testReceipts.Key, fakeObject{MirrorObject{ETag: "book", Size: int64(len(stale))}, stale})
+
+ r := mirrorRound(t, f, testReceipts)
+ if r.Changed != 4 || r.BytesCopied != 1050 {
+ t.Fatalf("round = %+v, want a, b and c copied and the extra deleted", r)
+ }
+ sameBodies(t, stagedBodies(f), sourceBodies(f))
+ book := readReceipts(t, f)
+ if _, ok := book["ltx/0/extra.ltx"]; ok {
+ t.Fatal("the stale receipt of a deleted key survived")
+ }
+ if got := book["ltx/1/b.ltx"]; got.DstETag != "new-b" || got.SrcETag != "new-b-27" {
+ t.Fatalf("b's receipt = %+v", got)
+ }
+
+ // A book written for another pair (another move's target) is no
+ // evidence at all, even when its receipt would match.
+ g := newFakeMirror()
+ g.putMultipart("src", "instances/h/ltx/1/a.ltx", "a", 500)
+ g.putSized("dst", "moves/h/ltx/1/a.ltx", "a", 500)
+ other := `{"v":1,"src_bucket":"src","src_prefix":"instances/h/","dst_bucket":"other-dst","dst_prefix":"moves/h/",` +
+ `"objects":{"ltx/1/a.ltx":{"src_etag":"a-27","src_size":500,"dst_etag":"a"}}}`
+ g.store("src", testReceipts.Key, fakeObject{MirrorObject{ETag: "book", Size: int64(len(other))}, other})
+ if r := mirrorRound(t, g, testReceipts); r.Changed != 1 {
+ t.Fatalf("round over another pair's book = %+v, want the object copied again", r)
+ }
+}
+
+// forEachParallel on its own: an error stops dispatch, and an already-ended
+// context starts nothing.
+func TestForEachParallel(t *testing.T) {
+ var ran atomic.Int32
+ err := forEachParallel(context.Background(), 1, 10, func(_ context.Context, i int) error {
+ ran.Add(1)
+ if i == 3 {
+ return errors.New("boom")
+ }
+ return nil
+ })
+ if err == nil || err.Error() != "boom" || ran.Load() > 5 {
+ t.Fatalf("err %v after %d calls, want boom after at most 5", err, ran.Load())
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ ran.Store(0)
+ err = forEachParallel(ctx, 4, 10, func(context.Context, int) error { ran.Add(1); return nil })
+ if !errors.Is(err, context.Canceled) || ran.Load() > 4 {
+ t.Fatalf("err %v after %d calls on an ended context", err, ran.Load())
+ }
+ if err := forEachParallel(context.Background(), 8, 0, nil); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Astra round 1, finding 3: no network request is made while the round's
+// accounting is locked. A receipts read that stalls once a copy has landed
+// (the lazy load a receipt write used to make under the lock) must not keep
+// another copy's failure from stopping the round: the book is read before
+// any copy starts.
+func TestMirrorPrefixAReceiptsReadNeverBlocksTheFirstFailure(t *testing.T) {
+ f := newFakeMirror()
+ f.workers = 2
+ f.putMultipart("src", "instances/h/ltx/1/a.ltx", "a", 100) // lands, needs a receipt
+ f.putSized("src", "instances/h/ltx/0/b.ltx", "b", 10) // fails
+ var landed atomic.Bool
+ release := make(chan struct{})
+ t.Cleanup(func() { close(release) })
+ f.onCopy = func(_ context.Context, key string) error {
+ if strings.HasSuffix(key, "b.ltx") {
+ // Fail only once a's copy has landed and its receipt is due.
+ for deadline := time.Now().Add(5 * time.Second); !landed.Load() && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
+ }
+ time.Sleep(20 * time.Millisecond)
+ return errors.New("injected copy failure")
+ }
+ landed.Store(true)
+ return nil
+ }
+ f.onGet = func(string) {
+ if landed.Load() {
+ <-release // a receipts read that stalls after a copy
+ }
+ }
+ done := make(chan error, 1)
+ go func() {
+ _, err := MirrorPrefix(context.Background(), f, "src", "instances/h/", "dst", "moves/h/", testReceipts)
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ if err == nil || !strings.Contains(err.Error(), "injected copy failure") {
+ t.Fatalf("round error = %v, want the injected failure", err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("a receipts read made under the round's lock kept the first failure from stopping the round")
+ }
+}
diff --git a/internal/lease/mirror_test.go b/internal/lease/mirror_test.go
index 2e9bae104..d1afab49e 100644
--- a/internal/lease/mirror_test.go
+++ b/internal/lease/mirror_test.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"strings"
+ "sync"
"testing"
)
@@ -15,16 +16,31 @@ import (
// single-part: the destination's ETag is the body, whatever the source's
// was. The fakes before #175 kept the source ETag on copy, and hid the bug
// that made every multipart object look changed on every round.
+//
+// A round runs its copies side by side, so every method holds mu, and the
+// hooks (beforeCopy, onCopy) run without it: a hook may call the fake.
+// workers is the round's concurrency (MirrorConcurrency); zero is the
+// production default.
type fakeMirror struct {
+ mu sync.Mutex
buckets map[string]map[string]fakeObject
ops []string
failOn string
// beforeCopy runs at the top of Copy, before the precondition check:
// the window between a round's listing and its copy.
beforeCopy func(srcKey string)
- failPut bool
- failGet bool
- puts int
+ // onCopy runs after the precondition check and before the object
+ // lands; ctx is the copy's own. An error it returns is the copy's.
+ onCopy func(ctx context.Context, srcKey string) error
+ // onGet runs at the top of Get, without the lock.
+ onGet func(key string)
+ failPut bool
+ failGet bool
+ puts int
+ gets int
+ workers int
+ // inFlight and peak count the copies running at once.
+ inFlight, peak int
}
type fakeObject struct {
@@ -34,6 +50,13 @@ type fakeObject struct {
func newFakeMirror() *fakeMirror { return &fakeMirror{buckets: map[string]map[string]fakeObject{}} }
+func (f *fakeMirror) MirrorWorkers() int {
+ if f.workers == 0 {
+ return DefaultMirrorWorkers
+ }
+ return f.workers
+}
+
func (f *fakeMirror) put(bucket, key, etag string) { f.putSized(bucket, key, etag, 0) }
// putSized stores a single-part object: its ETag is its body.
@@ -48,13 +71,36 @@ func (f *fakeMirror) putMultipart(bucket, key, body string, size int64) {
}
func (f *fakeMirror) store(bucket, key string, o fakeObject) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.storeLocked(bucket, key, o)
+}
+
+func (f *fakeMirror) storeLocked(bucket, key string, o fakeObject) {
if f.buckets[bucket] == nil {
f.buckets[bucket] = map[string]fakeObject{}
}
f.buckets[bucket][key] = o
}
+// remove deletes one object, as a test's own hand does.
+func (f *fakeMirror) remove(bucket, key string) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ delete(f.buckets[bucket], key)
+}
+
+// object is one stored object, as a test reads it.
+func (f *fakeMirror) object(bucket, key string) (fakeObject, bool) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ o, ok := f.buckets[bucket][key]
+ return o, ok
+}
+
func (f *fakeMirror) List(_ context.Context, bucket, prefix string) (map[string]MirrorObject, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
out := map[string]MirrorObject{}
for k, v := range f.buckets[bucket] {
if strings.HasPrefix(k, prefix) {
@@ -64,32 +110,64 @@ func (f *fakeMirror) List(_ context.Context, bucket, prefix string) (map[string]
return out, nil
}
-func (f *fakeMirror) Copy(_ context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {
- if f.failOn == srcKey {
+func (f *fakeMirror) Copy(ctx context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {
+ f.mu.Lock()
+ failOn, before, on := f.failOn, f.beforeCopy, f.onCopy
+ f.inFlight++
+ if f.inFlight > f.peak {
+ f.peak = f.inFlight
+ }
+ f.mu.Unlock()
+ defer func() {
+ f.mu.Lock()
+ f.inFlight--
+ f.mu.Unlock()
+ }()
+ if failOn == srcKey {
return "", fmt.Errorf("injected copy failure on %s", srcKey)
}
- if f.beforeCopy != nil {
- f.beforeCopy(srcKey)
+ if before != nil {
+ before(srcKey)
}
+ f.mu.Lock()
o, ok := f.buckets[srcBucket][srcKey]
+ f.mu.Unlock()
if !ok {
return "", fmt.Errorf("no such key %s", srcKey)
}
if srcETag != "" && o.ETag != srcETag {
return "", ErrCopySourceChanged
}
+ if on != nil {
+ if err := on(ctx, srcKey); err != nil {
+ return "", err
+ }
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
f.ops = append(f.ops, "copy "+srcKey+" -> "+dstBucket+"/"+dstKey)
- f.store(dstBucket, dstKey, fakeObject{MirrorObject{ETag: o.body, Size: o.Size}, o.body})
+ f.storeLocked(dstBucket, dstKey, fakeObject{MirrorObject{ETag: o.body, Size: o.Size}, o.body})
return o.body, nil
}
func (f *fakeMirror) Delete(_ context.Context, bucket, key string) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
f.ops = append(f.ops, "delete "+bucket+"/"+key)
delete(f.buckets[bucket], key)
return nil
}
func (f *fakeMirror) Get(_ context.Context, bucket, key string) ([]byte, error) {
+ f.mu.Lock()
+ on := f.onGet
+ f.mu.Unlock()
+ if on != nil {
+ on(key)
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.gets++
if f.failGet {
return nil, fmt.Errorf("injected get failure on %s", key)
}
@@ -101,12 +179,14 @@ func (f *fakeMirror) Get(_ context.Context, bucket, key string) ([]byte, error)
}
func (f *fakeMirror) Put(_ context.Context, bucket, key string, body []byte) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
if f.failPut {
return fmt.Errorf("injected put failure on %s", key)
}
f.puts++
f.ops = append(f.ops, "put "+bucket+"/"+key)
- f.store(bucket, key, fakeObject{MirrorObject{ETag: fmt.Sprint("put-", f.puts), Size: int64(len(body))}, string(body)})
+ f.storeLocked(bucket, key, fakeObject{MirrorObject{ETag: fmt.Sprint("put-", f.puts), Size: int64(len(body))}, string(body)})
return nil
}
@@ -390,10 +470,10 @@ func TestMirrorPrefixSourceChangedMidRoundIsDeferred(t *testing.T) {
f := newFakeMirror()
f.putMultipart("src", "instances/h/ltx/1/a.ltx", "v1", 100<<20)
f.putSized("src", "instances/h/ltx/0/b.ltx", "b", 10)
+ var once sync.Once
f.beforeCopy = func(k string) {
if k == "instances/h/ltx/1/a.ltx" {
- f.putMultipart("src", k, "v2", 120<<20)
- f.beforeCopy = nil
+ once.Do(func() { f.putMultipart("src", k, "v2", 120<<20) })
}
}
r := mirrorRound(t, f, testReceipts)
@@ -495,16 +575,22 @@ func (f *fakeVersionedMirror) CopyVersioned(ctx context.Context, sb, sk, se, db,
if err != nil {
return "", "", err
}
- f.n++
- return etag, fmt.Sprintf("v%d", f.n), nil
+ return etag, f.next(), nil
}
func (f *fakeVersionedMirror) PutVersioned(ctx context.Context, b, k string, body []byte) (string, error) {
if err := f.fakeMirror.Put(ctx, b, k, body); err != nil {
return "", err
}
+ return f.next(), nil
+}
+
+// next is the next version id; copies run side by side, so under mu.
+func (f *fakeVersionedMirror) next() string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
f.n++
- return fmt.Sprintf("v%d", f.n), nil
+ return fmt.Sprintf("v%d", f.n)
}
func TestMirrorPrefixRecordsDestinationVersions(t *testing.T) {
diff --git a/internal/moves/offline.go b/internal/moves/offline.go
index ea6f56fd4..670a9d1c6 100644
--- a/internal/moves/offline.go
+++ b/internal/moves/offline.go
@@ -118,11 +118,15 @@ func OfflineSourced() []OfflineItem {
}
// The roles a report names: a source-region box that stopped the host, the
-// one source box that seals, and the target box that lands.
+// one source box that seals, the target box that lands, and the one source
+// box that pre-copies the replica while the move is offline-staging (the
+// stager, which the stage door picks with OfflineStager and pins on the
+// record as move.stager).
const (
OfflineRoleSource = "source"
OfflineRoleSealer = "sealer"
OfflineRoleTarget = "target"
+ OfflineRoleStager = "stager"
)
// The phases a report names.
@@ -136,6 +140,8 @@ const (
OfflinePhaseLifted = "lifted"
OfflinePhaseReverted = "reverted"
OfflinePhaseFailed = "failed"
+ // OfflinePhaseStaged is a stager's report after a pre-copy round.
+ OfflinePhaseStaged = "staged"
)
// OfflineReportVersion is OfflineReport.V.
@@ -213,15 +219,37 @@ type OfflineReport struct {
// Retired is the parked report's inventory of what the source kept.
Retired []RetiredPath `json:"retired,omitempty"`
+
+ // The stager's progress, rewritten after every pre-copy round while
+ // the move is offline-staging (At is when the round ended): the
+ // replica as the round's own listing after it found it, lock.json
+ // aside; how much of it the staged copy holds, key and size for key
+ // and size; and what the round itself copied. A failed round keeps
+ // the last progress it had and says why in StageError, a box
+ // diagnostic and never copy.
+ ReplicaObjects int `json:"replica_objects,omitempty"`
+ ReplicaBytes int64 `json:"replica_bytes,omitempty"`
+ StagedObjects int `json:"staged_objects,omitempty"`
+ StagedBytes int64 `json:"staged_bytes,omitempty"`
+ RoundObjects int `json:"round_objects,omitempty"`
+ RoundBytes int64 `json:"round_bytes,omitempty"`
+ StageError string `json:"stage_error,omitempty"`
+ // StageRequest is the record's staged_at the round answered: one round
+ // per stage request (provision.OfflineStage, OfflineRestage), and a
+ // stager runs none while its report already answers the record's.
+ StageRequest time.Time `json:"stage_request,omitzero"`
}
var (
- offlineRoles = map[string]bool{OfflineRoleSource: true, OfflineRoleSealer: true, OfflineRoleTarget: true}
+ offlineRoles = map[string]bool{OfflineRoleSource: true, OfflineRoleSealer: true, OfflineRoleTarget: true, OfflineRoleStager: true}
offlinePhases = map[string]bool{OfflinePhaseStopped: true, OfflinePhaseSealed: true, OfflinePhaseLanded: true, OfflinePhaseOpened: true,
- OfflinePhaseParked: true, OfflinePhaseGone: true, OfflinePhaseLifted: true, OfflinePhaseReverted: true, OfflinePhaseFailed: true}
+ OfflinePhaseParked: true, OfflinePhaseGone: true, OfflinePhaseLifted: true, OfflinePhaseReverted: true, OfflinePhaseFailed: true,
+ OfflinePhaseStaged: true}
)
-// offlineCarries reports whether a role's report may carry the item.
+// offlineCarries reports whether a role's report may carry the item. A
+// stager's report carries none: the pre-copy decides nothing the doors
+// check, and the seal's own round proves the db item.
func offlineCarries(role string, it OfflineItem) bool {
switch role {
case OfflineRoleSource, OfflineRoleSealer:
@@ -302,7 +330,8 @@ func offlineSegment(s string) bool {
// OfflineReportKey is where a box writes its report:
// control/offline-moves/<host>/<op>/<role>/<box>.json. It is "" when host
-// fails provision.ValidHostShape, role is not source, sealer or target, or op
+// fails provision.ValidHostShape, role is not source, sealer, target or
+// stager, or op
// or box is not a plain name; a caller that gets "" has a bug and writes
// nothing. The prefix is outside every create-only and no-delete prefix, so
// a box may rewrite its own report.
@@ -371,6 +400,23 @@ func ValidateOfflineSeal(r OfflineReport, host, op string) error {
return nil
}
+// OfflineStager is the stage door's choice of the box that pre-copies the
+// replica while a move is offline-staging: the lexically first of boxes,
+// the source region's eligible boxes as the fleet's edges files list them,
+// "" for none. The door pins it on the record (MoveState.Stager) and it
+// never changes for the move: a box that joins later never stages or
+// cleans for it, so exactly one box ever writes the staging prefix (Astra
+// round 1, finding 2).
+func OfflineStager(boxes []string) string {
+ stager := ""
+ for _, b := range boxes {
+ if b != "" && (stager == "" || b < stager) {
+ stager = b
+ }
+ }
+ return stager
+}
+
// OfflineActiveKey is the one lock of the offline move: while it names a
// host whose record still carries that move, no other offline move starts.
const OfflineActiveKey = "control/offline-moves/active.json"
diff --git a/internal/moves/offline_test.go b/internal/moves/offline_test.go
index 50fbc9e13..46e8eca3e 100644
--- a/internal/moves/offline_test.go
+++ b/internal/moves/offline_test.go
@@ -493,3 +493,33 @@ func TestOfflineSealKeyAndValidation(t *testing.T) {
t.Errorf("evidence = %q", ev)
}
}
+
+// The stager (offline-staging's pre-copy): its report is a report like any
+// other, under its own role, and carries no manifest item; the stager is the
+// lexically first box, whatever order the listing gave.
+func TestOfflineStagerReport(t *testing.T) {
+ r := OfflineReport{V: OfflineReportVersion, Op: offTestOp, Host: offTestHost, Box: "box-a", Role: OfflineRoleStager, Phase: OfflinePhaseStaged,
+ ReplicaObjects: 10, ReplicaBytes: 100, StagedObjects: 9, StagedBytes: 90, RoundObjects: 3, RoundBytes: 30}
+ if err := ValidateOfflineReport(r); err != nil {
+ t.Fatal(err)
+ }
+ if got := OfflineReportKey(offTestHost, offTestOp, OfflineRoleStager, "box-a"); got != "control/offline-moves/"+offTestHost+"/"+offTestOp+"/stager/box-a.json" {
+ t.Fatalf("key = %q", got)
+ }
+ r.Items = map[string]Check{"db": {OK: true}}
+ if ValidateOfflineReport(r) == nil {
+ t.Fatal("a stager report carrying an item was accepted")
+ }
+ body, err := json.Marshal(OfflineReport{V: 1})
+ if err != nil || strings.Contains(string(body), "replica_objects") || strings.Contains(string(body), "stage_error") {
+ t.Fatalf("the stager's fields are not omitempty: %s", body)
+ }
+ for _, c := range []struct {
+ boxes []string
+ want string
+ }{{nil, ""}, {[]string{"box-c", "box-a", "box-b"}, "box-a"}, {[]string{"", "box-z"}, "box-z"}} {
+ if got := OfflineStager(c.boxes); got != c.want {
+ t.Errorf("OfflineStager(%v) = %q, want %q", c.boxes, got, c.want)
+ }
+ }
+}
diff --git a/internal/provision/moves_offline_precopy_census_test.go b/internal/provision/moves_offline_precopy_census_test.go
new file mode 100644
index 000000000..cb47a8312
--- /dev/null
+++ b/internal/provision/moves_offline_precopy_census_test.go
@@ -0,0 +1,49 @@
+package provision
+
+// movesOfflinePrecopyWriteCensusOverlay is moves-offline-precopy's
+// record-write sites (parallel copies in lease.MirrorPrefix and the carlos
+// move offline stage door), census-ruling-moves-offline-precopy: RULED by the
+// coordinator, as census owner for this branch (2026-10-21), accepted as
+// classified (docs/superpowers/notes/census-proposal-moves-offline-precopy.md).
+//
+// Three sites change and keep their class; no site is added.
+// registerCLIRoutes gains the stage door's route line beside the six
+// offline doors (key-site, main's), pinned on the merged tree with
+// design-plans-and-caps' two account plan doors. offlineTakeSeal
+// (nonrecord, census-ruling-moves-offline's) gains one refusal before its
+// mirror after Astra's round 2: a staged move seals only into the staging
+// bucket and region fixed on the record; its Puts and their keys under the
+// move's moves/@offline/ prefix are unchanged. MirrorPrefixKeep plans its skips first
+// and then runs the copies and, after every copy has landed, the deletes on
+// up to DefaultMirrorWorkers goroutines; it still writes only the
+// destination prefix (Copy or CopyVersioned, Delete) and the receipts
+// through the book (set, drop and save, every update under one mutex,
+// receiptLedger.save the one Put, unchanged), so versioned-transport stands.
+// After Astra's round 1 (692d41cf) the book is read once before the workers
+// start whenever the round copies anything, so set and drop touch memory
+// only and no request is made under the mutex; the write set is unchanged.
+var movesOfflinePrecopyWriteCensusOverlay = []censusEntry{
+ {"internal/console/authcli.go", "Server.registerCLIRoutes", "key-site", "lit:instances,lit:sidecars", "a0aff01b7e2f4bb6fd23e8b54d41d90f1cd4171822f9defb9bd16cb2a08e2e87"},
+ {"internal/lease/mirror.go", "MirrorPrefixKeep", "versioned-transport", "sel:CopyVersioned", "dff4d03fa4e56219c5a3e08862f24ad2a6bce7a7e0df806571fa52e8fb29fb19"},
+ {"internal/host/movesoffline_seal.go", "movesWorker.offlineTakeSeal", "nonrecord", "sel:Put", "f606873fb09e8644d449a36ae64cecbb84b2f83791d4ebec4cb491d3d3f7cdaa"},
+}
+
+// movesOfflinePrecopyWriteCensusInventory is design-plans-and-caps' inventory
+// (on scan-demand-revival's) with the moves-offline-precopy overlay, loaded
+// last: a changed site replaces its entry, a new one is added.
+func movesOfflinePrecopyWriteCensusInventory() []censusEntry {
+ entries := append([]censusEntry(nil), plansCapsWriteCensusInventory()...)
+ for _, replacement := range movesOfflinePrecopyWriteCensusOverlay {
+ found := false
+ for i, entry := range entries {
+ if entry.File == replacement.File && entry.Unit == replacement.Unit {
+ entries[i], found = replacement, true
+ break
+ }
+ }
+ if !found {
+ entries = append(entries, replacement)
+ }
+ }
+ return entries
+}
diff --git a/internal/provision/offline_move.go b/internal/provision/offline_move.go
index 2af9aa355..8d87fb943 100644
--- a/internal/provision/offline_move.go
+++ b/internal/provision/offline_move.go
@@ -31,11 +31,19 @@ const MoveKindOffline = "offline"
// The offline move's states, in the order a move takes them:
//
-// offline-stopping -> offline-sealing -> offline-sealed -> offline-landing
-// -> offline-landed -> offline-open -> offline-finishing -> (cleared)
+// [offline-staging ->] offline-stopping -> offline-sealing -> offline-sealed
+// -> offline-landing -> offline-landed -> offline-open -> offline-finishing
+// -> (cleared)
//
-// offline-reverting is abort's state, reached from stopping through landed.
+// offline-staging is optional: OfflineStage creates the move there instead
+// of in stopping, and the one source box that stages (moves.OfflineStager)
+// pre-copies the replica into the target's staging prefix while the
+// instance keeps serving. Nothing holds, fences or tombstones the instance
+// in staging. OfflineStartStaged then takes it to stopping, and the seal's
+// own round finds the pre-copied objects and skips them by their receipts.
+// offline-reverting is abort's state, reached from staging through landed.
const (
+ OfflineStaging = "offline-staging"
OfflineStopping = "offline-stopping"
OfflineSealing = "offline-sealing"
OfflineSealed = "offline-sealed"
@@ -118,6 +126,33 @@ func offlineIs(rec *Record, id string) error {
// malformed id. It stamps kind, id, from, to, state, requested_by and
// requested_at; it does not touch the region.
func OfflineStart(id, from, to, by string, now time.Time) func(*Record) error {
+ return offlineBegin(id, from, to, by, now, OfflineStopping)
+}
+
+// StageTarget is what a stage fixes for the whole move: the one box that
+// copies (Stager) and where the copy goes (Bucket, the target's data
+// bucket, in Region).
+type StageTarget struct{ Stager, Bucket, Region string }
+
+// OfflineStage is OfflineStart into offline-staging: the same refusals and
+// the same stamps, plus staged_at, the stager and the staging destination
+// (a stage that names any of them empty is malformed). The instance keeps
+// serving.
+func OfflineStage(id, from, to, by string, st StageTarget, now time.Time) func(*Record) error {
+ begin := offlineBegin(id, from, to, by, now, OfflineStaging)
+ return func(rec *Record) error {
+ if st.Stager == "" || st.Bucket == "" {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
+ }
+ if err := begin(rec); err != nil {
+ return err
+ }
+ rec.Move.Stager, rec.Move.StageBucket, rec.Move.StageRegion = st.Stager, st.Bucket, st.Region
+ return nil
+ }
+}
+
+func offlineBegin(id, from, to, by string, now time.Time, state string) func(*Record) error {
return func(rec *Record) error {
if err := seedMoveRefusal(rec); err != nil {
return err
@@ -145,14 +180,74 @@ func OfflineStart(id, from, to, by string, now time.Time) func(*Record) error {
ID: id,
From: from,
To: to,
- State: OfflineStopping,
+ State: state,
RequestedBy: by,
RequestedAt: now.UTC(),
}
+ if state == OfflineStaging {
+ t := now.UTC()
+ rec.Move.StagedAt = &t
+ }
return nil
}
}
+// OfflineStartStaged is staging to stopping: the operator takes a staged
+// instance offline. The record's region must still be the move's source.
+// It stamps requested_by and requested_at anew, since the instance is
+// offline from now, and requested_at is always after staged_at, which is
+// how OfflineFinish tells a move that started from one that never did.
+func OfflineStartStaged(id, by string, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ if err := offlineAt(rec, id, OfflineStaging); err != nil {
+ return err
+ }
+ if rec.Region != rec.Move.From {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
+ }
+ if now.IsZero() {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
+ }
+ t := now.UTC()
+ if s := rec.Move.StagedAt; s != nil && !t.After(*s) {
+ t = s.Add(time.Nanosecond)
+ }
+ rec.Move.State, rec.Move.RequestedBy, rec.Move.RequestedAt = OfflineStopping, by, t
+ return nil
+ }
+}
+
+// OfflineRestage is staging to staging: the operator asked for another
+// pre-copy round. It moves staged_at to now (always forward), which is the
+// request the stager answers with exactly one round (the stager report's
+// stage_request names the staged_at it served). requested_at stays, so the
+// move still reads as never started.
+func OfflineRestage(id string, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ if err := offlineAt(rec, id, OfflineStaging); err != nil {
+ return err
+ }
+ if rec.Region != rec.Move.From {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
+ }
+ if now.IsZero() {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
+ }
+ t := now.UTC()
+ if s := rec.Move.StagedAt; s != nil && !t.After(*s) {
+ t = s.Add(time.Nanosecond)
+ }
+ rec.Move.StagedAt = &t
+ return nil
+ }
+}
+
+// StartedOffline reports whether the move ever took its instance offline:
+// a move created in stopping did at once, a staged one at OfflineStartStaged.
+func (m *MoveState) StartedOffline() bool {
+ return m.StagedAt == nil || m.RequestedAt.After(*m.StagedAt)
+}
+
// regionWord is a region as a sentence names it: the home placement's empty
// code reads "home".
func regionWord(r string) string {
@@ -269,8 +364,8 @@ func OfflineFinishingTransition(id string) func(*Record) error {
}
}
-// OfflineAbort puts the move on its way back: any of stopping, sealing,
-// sealed, landing or landed becomes reverting. From landing and landed the
+// OfflineAbort puts the move on its way back: any of staging, stopping,
+// sealing, sealed, landing or landed becomes reverting. From landing and landed the
// region goes back to the source in the same write (the flip is undone with
// the state, never separately). From open and finishing it refuses, because
// the target already serves. From reverting it changes nothing and writes
@@ -285,7 +380,7 @@ func OfflineAbort(id string) func(*Record) error {
switch rec.Move.State {
case OfflineReverting:
return errRecordUnchanged
- case OfflineStopping, OfflineSealing, OfflineSealed:
+ case OfflineStaging, OfflineStopping, OfflineSealing, OfflineSealed:
case OfflineLanding, OfflineLanded:
rec.Region = rec.Move.From
default:
@@ -300,7 +395,8 @@ func OfflineAbort(id string) func(*Record) error {
// legal from finishing (outcome done, region must be the target's) and from
// reverting (outcome aborted, region must be the source's); anything else is
// refused. The outcome's offline time runs from the request to the open for a
-// done move and to now for an aborted one.
+// done move and to now for an aborted one; a move aborted while it was still
+// staging was never offline, and records none.
func OfflineFinish(id string, now time.Time) func(*Record) error {
return func(rec *Record) error {
if err := offlineIs(rec, id); err != nil {
@@ -326,7 +422,7 @@ func OfflineFinish(id string, now time.Time) func(*Record) error {
default:
return fmt.Errorf("%w: %s", ErrMoveConflict, offlineFinishState(m.State))
}
- if s := end.Sub(m.RequestedAt).Seconds(); s > 0 {
+ if s := end.Sub(m.RequestedAt).Seconds(); s > 0 && m.StartedOffline() {
out.OfflineS = s
}
rec.LastMove, rec.Move = out, nil
diff --git a/internal/provision/offline_stage_test.go b/internal/provision/offline_stage_test.go
new file mode 100644
index 000000000..9da62fd55
--- /dev/null
+++ b/internal/provision/offline_stage_test.go
@@ -0,0 +1,234 @@
+package provision
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The pre-copy (offline-staging): OfflineStage creates the move with the
+// instance still serving, OfflineStartStaged takes it to stopping, and abort
+// from staging ends a move that was never offline.
+
+func TestOfflineStageThenStartThenTheWholeMove(t *testing.T) {
+ ctx := context.Background()
+ now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+
+ got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now))
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := got.Move
+ if !m.Offline() || m.State != OfflineStaging || m.ID != offID || m.To != "uk" || got.Region != "" ||
+ m.StagedAt == nil || !m.StagedAt.Equal(now) || !m.RequestedAt.Equal(now) || m.Stager != "box-a" || m.StageBucket != "data-uk" || m.StageRegion != "eu-west-2" {
+ t.Fatalf("staged record: %+v region %q", m, got.Region)
+ }
+ // Staging is a move in flight: no second move starts beside it.
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStart(otherOff, "", "uk", "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a start beside a staged move: %v", err)
+ }
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(otherOff, "", "us", "ops@example.com", stageAt, now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a stage beside a staged move: %v", err)
+ }
+
+ start := now.Add(10 * time.Minute)
+ got, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "other@example.com", start))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.Move.State != OfflineStopping || got.Move.RequestedBy != "other@example.com" || !got.Move.RequestedAt.Equal(start) || !got.Move.StagedAt.Equal(now) || got.Move.Stager != "box-a" {
+ t.Fatalf("started: %+v", got.Move)
+ }
+ // The rest of the move is the ordinary path; offline time counts from
+ // the start, not the stage: start at +10m, open at +30m.
+ got = offlineWalk(t, s, rec, offlineSteps(now)[1:])
+ if got.Move != nil || got.LastMove.Outcome != OfflineOutcomeDone || got.LastMove.OfflineS != 1200 {
+ t.Fatalf("finished: %+v", got.LastMove)
+ }
+}
+
+func TestOfflineStartStagedRefusals(t *testing.T) {
+ ctx := context.Background()
+ now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+
+ // From every state but staging, and with another id.
+ steps := offlineSteps(now)
+ for walked := 1; walked <= 7; walked++ {
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+ offlineWalk(t, s, rec, steps[:walked])
+ before := offlineGet(t, s, rec)
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("start-staged from %s: %v", before.Move.State, err)
+ }
+ if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) {
+ t.Fatalf("a refused start-staged from %s changed the record", before.Move.State)
+ }
+ }
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(otherOff, "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("start-staged with another id: %v", err)
+ }
+ // A region written under the staged move.
+ if _, err := updateRecord(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region = "us"; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("start-staged over a moved region: %v", err)
+ }
+
+ // No other step leaves staging: seal through finish all refuse.
+ s = release.NewDirStore(t.TempDir())
+ rec = offlineRecord(t, s)
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil {
+ t.Fatal(err)
+ }
+ before := offlineGet(t, s, rec)
+ for _, st := range steps[1:] {
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, st.mut()); !errors.Is(err, ErrMoveConflict) {
+ t.Errorf("%s from staging: %v", st.name, err)
+ }
+ }
+ if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) {
+ t.Fatal("a refused step changed the staged record")
+ }
+}
+
+// A start in the same instant as the stage still lands after it, so the
+// finish can tell the two apart.
+func TestOfflineStartStagedIsAlwaysAfterTheStage(t *testing.T) {
+ ctx := context.Background()
+ now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil {
+ t.Fatal(err)
+ }
+ got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now.Add(-time.Second)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !got.Move.RequestedAt.After(*got.Move.StagedAt) || !got.Move.StartedOffline() {
+ t.Fatalf("start %v is not after the stage %v", got.Move.RequestedAt, got.Move.StagedAt)
+ }
+}
+
+// Abort from staging: reverting with the region untouched, then a finish
+// that records the move aborted and never offline. Aborted after its start,
+// a staged move counts its offline time from the start.
+func TestOfflineAbortFromStaging(t *testing.T) {
+ ctx := context.Background()
+ now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil {
+ t.Fatal(err)
+ }
+ got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineAbort(offID))
+ if err != nil || got.Move.State != OfflineReverting || got.Region != "" {
+ t.Fatalf("abort from staging: %v %+v", err, got)
+ }
+ done, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(time.Hour)))
+ if err != nil || done.Move != nil || done.LastMove.Outcome != OfflineOutcomeAborted || done.LastMove.OfflineS != 0 {
+ t.Fatalf("finish after abort from staging: %v %+v", err, done.LastMove)
+ }
+
+ s = release.NewDirStore(t.TempDir())
+ rec = offlineRecord(t, s)
+ for _, mut := range []func(*Record) error{
+ OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now),
+ OfflineStartStaged(offID, "ops@example.com", now.Add(20*time.Minute)),
+ OfflineAbort(offID),
+ } {
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, mut); err != nil {
+ t.Fatal(err)
+ }
+ }
+ done, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineFinish(offID, now.Add(30*time.Minute)))
+ if err != nil || done.LastMove.Outcome != OfflineOutcomeAborted || done.LastMove.OfflineS != 600 {
+ t.Fatalf("finish after a started staged move: %v %+v", err, done.LastMove)
+ }
+}
+
+// staged_at is additive like the other offline fields: a staged journal
+// round-trips, and every earlier shape still does (TestOfflineFieldsAreAdditiveOnTheWire).
+func TestOfflineStagedAtIsAdditiveOnTheWire(t *testing.T) {
+ const staged = `{"schema":1,"host":"a.go.example.com","slug":"s","app":"app","account":"bab","region":"ie","channel":"stable","created_at":"2026-09-01T08:00:00Z","move":{"id":"mv_abcdefghijklmnopqrstuvwxyz","from":"ie","to":"uk","state":"offline-staging","requested_by":"ops@example.com","requested_at":"2026-10-21T09:00:00Z","updated_at":"2026-10-21T09:00:00Z","kind":"offline","staged_at":"2026-10-21T09:00:00Z","stager":"box-a","stage_bucket":"data-uk","stage_region":"eu-west-2"}}`
+ rec, err := DecodeRecord([]byte(staged))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rec.Move.State != OfflineStaging || rec.Move.StagedAt == nil || rec.Move.StartedOffline() || rec.Move.Stager != "box-a" {
+ t.Fatalf("staged journal: %+v", rec.Move)
+ }
+ if out, _ := json.Marshal(rec); string(out) != staged {
+ t.Errorf("staged journal did not round-trip:\n got %s\nwant %s", out, staged)
+ }
+}
+
+// A second stage is OfflineRestage: staged_at moves forward, never back,
+// requested_at stays, and the move still reads as never started. Nothing
+// but staging admits it.
+func TestOfflineRestage(t *testing.T) {
+ ctx := context.Background()
+ now := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("restage at rest: %v", err)
+ }
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", stageAt, now)); err != nil {
+ t.Fatal(err)
+ }
+ got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now.Add(5*time.Minute)))
+ if err != nil || !got.Move.StagedAt.Equal(now.Add(5*time.Minute)) || !got.Move.RequestedAt.Equal(now) || got.Move.StartedOffline() || got.Move.State != OfflineStaging || got.Move.Stager != "box-a" {
+ t.Fatalf("restage: %v %+v", err, got.Move)
+ }
+ // A clock behind the last request still moves it forward.
+ got, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now))
+ if err != nil || !got.Move.StagedAt.After(now.Add(5*time.Minute)) {
+ t.Fatalf("restage with an earlier clock: %v %+v", err, got.Move)
+ }
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(otherOff, now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("restage with another id: %v", err)
+ }
+ // The start after a restage is still after the last request.
+ got, err = SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineStartStaged(offID, "ops@example.com", now.Add(time.Minute)))
+ if err != nil || !got.Move.StartedOffline() || got.Move.State != OfflineStopping {
+ t.Fatalf("start after restage: %v %+v", err, got.Move)
+ }
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, OfflineRestage(offID, now)); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("restage after the start: %v", err)
+ }
+}
+
+// stageAt is the stager and staging destination the tests' stage door names.
+var stageAt = StageTarget{Stager: "box-a", Bucket: "data-uk", Region: "eu-west-2"}
+
+// The stager is pinned at the stage: a stage that names none is malformed,
+// and so is one with no staging bucket.
+func TestOfflineStageNamesItsStager(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ rec := offlineRecord(t, s)
+ before := offlineGet(t, s, rec)
+ if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", StageTarget{Bucket: "data-uk"}, time.Now())); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a stage with no stager: %v", err)
+ }
+ if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, OfflineStage(offID, "", "uk", "ops@example.com", StageTarget{Stager: "box-a"}, time.Now())); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a stage with no staging bucket: %v", err)
+ }
+ if after := offlineGet(t, s, rec); !reflect.DeepEqual(after, before) {
+ t.Fatal("a refused stage changed the record")
+ }
+}
diff --git a/internal/provision/plan14_root_reads_test.go b/internal/provision/plan14_root_reads_test.go
index 2fff1e9c1..4fbe08f89 100644
--- a/internal/provision/plan14_root_reads_test.go
+++ b/internal/provision/plan14_root_reads_test.go
@@ -1062,6 +1062,24 @@ func TestPlan14RootReadCensus(t *testing.T) {
t.Fatal(err)
}
seal = "4f5e0ed6adf7d6bcefdfc9ce50599a21adb06ae3e6d49cab71835e116968df17"
+ // Census overlay moves-offline-precopy (parallel copies in
+ // lease.MirrorPrefix and the carlos move offline stage door), loaded
+ // last, after design-plans-and-caps, under
+ // census-ruling-moves-offline-precopy (the coordinator, as census owner,
+ // 2026-10-21); the ruling names each unit's executing identity
+ // (docs/superpowers/notes/census-proposal-moves-offline-precopy.md).
+ // offlineSourceRevert is re-frozen at its disposition, offlineBegin
+ // (OfflineStart's body) is new accepted legacy, and OfflineStart is
+ // retired by an H3 census-ruling removal; offlineTakeSeal is re-frozen
+ // for Astra's round 2. Three frozen declarations are added, so the seal
+ // moves; design-plans-and-caps' is checked first.
+ if errs := checkRootReadLegacy(inv, seal); len(errs) != 0 {
+ t.Fatal(errs)
+ }
+ if err = loadRootReadInventory(filepath.Join(root, "internal/provision/testdata/plan14-root-reads-moves-offline-precopy.tsv"), &inv, true); err != nil {
+ t.Fatal(err)
+ }
+ seal = "940ed1bba8d911e0f57af83ecac253c77ba5e14dad590258e0ee03ec98ccb818"
errs := append(compareRootReads(got, inv), checkRootReadLegacy(inv, seal)...)
errs = append(errs, checkRootReadDeclarations(got, inv)...)
if len(errs) > 0 {
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index 49dfe13cd..3a7bd7aee 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -366,7 +366,7 @@ func TestPlan9B8RecordWriteCensus(t *testing.T) {
if len(got) < 100 {
t.Fatalf("census found only %d units under %s; the walk is broken", len(got), root)
}
- inventory := plansCapsWriteCensusInventory()
+ inventory := movesOfflinePrecopyWriteCensusInventory()
problems := append(censusDiff(inventory, got), censusPolicy(inventory)...)
if len(problems) > 0 {
t.Fatalf("record-write census rejected %d site(s); update the explicit inventory only after reviewing the source:\n %s", len(problems), strings.Join(problems, "\n "))
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index 30a4bf1b3..a5657b0cb 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -1065,6 +1065,27 @@ type MoveState struct {
// Boxes are the source-region boxes that had to confirm the stop, named
// at seal.
Boxes []string `json:"boxes,omitempty"`
+ // StagedAt is when an offline move was staged (OfflineStage): created
+ // in offline-staging, with the instance still serving. RequestedAt is
+ // then moved to the start, when the instance went offline, so a staged
+ // move's offline time counts from the start like any other's; and a
+ // move aborted before its start was never offline (OfflineFinish).
+ // Additive and omitempty like the three above.
+ StagedAt *time.Time `json:"staged_at,omitempty"`
+ // Stager is the one source box that pre-copies a staged move, named
+ // by the stage door and never changed (Astra round 1, finding 2): no
+ // other box ever writes the staging prefix for this move, and an abort
+ // of a staged move finishes only once this box has removed the staged
+ // copy. Additive and omitempty.
+ Stager string `json:"stager,omitempty"`
+ // StageBucket and StageRegion are the staged copy's destination, the
+ // target's data bucket and its region as the stage door read them,
+ // fixed before any copy (Astra round 2, finding 2). Every stage round,
+ // the seal's hand-over and an abort's cleanup use them; a target that
+ // later publishes other coordinates is refused, never followed.
+ // Additive and omitempty.
+ StageBucket string `json:"stage_bucket,omitempty"`
+ StageRegion string `json:"stage_region,omitempty"`
}
// MoveInputs (what the target was checked against, §7.3 T3) is NOT
diff --git a/internal/provision/testdata/plan14-root-reads-moves-offline-precopy.tsv b/internal/provision/testdata/plan14-root-reads-moves-offline-precopy.tsv
new file mode 100644
index 000000000..4688b7ec8
--- /dev/null
+++ b/internal/provision/testdata/plan14-root-reads-moves-offline-precopy.tsv
@@ -0,0 +1,22 @@
+# Census overlay moves-offline-precopy (parallel copies in lease.MirrorPrefix and the carlos move offline stage door, an offline-staging record state), loaded LAST, after design-plans-and-caps. RULED as census-ruling-moves-offline-precopy (the coordinator, as census owner for this branch, 2026-10-21), accepted as proposed, re-ruled for Astra's rounds 1 and 2 and the merge of main f550afa2: every row is rev3=census-ruling-moves-offline-precopy. The ruling, with each unit's executing identity: docs/superpowers/notes/census-proposal-moves-offline-precopy.md.
+# Every unit here runs in the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos) or the console (user carlos-console) and makes no root decision, so each stays accepted legacy at the disposition census-ruling-moves-offline gave it or its sibling (J1 root-route for registry and route reads, J4 root-input for file reads).
+# movesWorker.offlineSourceRevert re-frozen with its new digest: it calls offlineUnstage, which reads only the object store; its read sites are unchanged and one offset moved. offlineBegin is new: OfflineStart's body, moved unchanged with the state as a parameter; its one site is rec.ActiveRun.Open(), a method, not a file read. OfflineStart is retired by an H3 census-ruling removal (the carlos-run getJSON form); a removal is not in the seal.
+# Round 1 (Astra, 692d41cf): the stage door pins the stager on the record (MoveState.Stager, written through SetRecordMove like every other move field); offlineUnstage cleans whenever staged_at is set, on the pinned stager only; the console's finish of a staged move waits for that stager's reverted report; MirrorPrefixKeep reads the receipts book before its workers start.
+# Round 2 (Astra, 0a6bb375): the stage door also fixes the staging destination on the record (MoveState.StageBucket, StageRegion, from the one target box's edges file, refusing a source box's own bucket, MV-O191); the stage round, the seal's hand-over (offlineTakeSeal, re-frozen here) and an abort's cleanup use only those; the stager re-reads the record (provision.GetRecord, the object store) before a round and before its delete phase (stageDeleteGate) and acknowledges it has stopped (offlineStagerStopped, its own stager report through writeOfflineReport); the console's start and seal doors wait for that acknowledgement and refuse a missing stager (MV-O190) or changed coordinates (MV-O192). No new root read or record write. Three frozen declarations in all, so the seal moves.
+# Envelopes re-pinned on the merged tree (main f550afa2 with this branch): . (move.go: the stage subcommand, its status and abort lines; no read), internal/console (the stage door, offlineBeginChecks, the stager pin and destination, the seal and start barriers, the finish gate, the status view; no read site but offlineClaimEvidence's, unchanged), internal/edgesfile (PlacementView reads data_bucket_region; no read site), internal/host (movesoffline_stage.go reads only the object store; edgesfile.go publishes offline-stage-1; the box-side stager election is gone, only the pinned box stages or cleans; offlineRound's staging branch), internal/lease (mirror.go: the round's copies and deletes run side by side; forEachParallel; the book read up front), internal/moves (the stager role, staged phase, report fields, OfflineStager) and internal/provision (offline_move.go: OfflineStage, which needs a StageTarget, OfflineRestage, OfflineStartStaged; provision.go: MoveState.StagedAt, Stager, StageBucket, StageRegion).
+@legacy e7c1242684b2da24edf4c9f56545d1208bd8433562aa494a911dda42e55d3452 internal/host/movesoffline.go movesWorker.offlineSourceRevert legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-precopy;basis=root-route (J1, census-ruling-moves-offline's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), on abort: census-ruling-moves-offline's body (reads the row's hold and releases only this move's hold, under reconcileMu), now calling offlineUnstage before it reports reverted. offlineUnstage acts only for a move carrying staged_at and only on the stager the stage door pinned on the record (move.stager); it reads only the object store (its own stager report, this box's receipts object) and deletes moves/<host>/ in the staging bucket and region the stage fixed on the record (move.stage_bucket, move.stage_region; it refuses a missing one or this box's own bucket and then acknowledges nothing) and this box's move-receipts/<host>.json, then writes its stager report reverted; it reads no local file and makes no root decision. Read sites unchanged; one offset moved (whole:Route@701 to @843) registry:HoldOp@562,whole:Route@549,whole:Route@843 b917ac2d1976eec14839ed65e3e562327d54af956e86f5218ed011731182aa1b
+internal/host/movesoffline.go movesWorker.offlineSourceRevert legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-precopy;basis=root-route (J1, census-ruling-moves-offline's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos), on abort: census-ruling-moves-offline's body (reads the row's hold and releases only this move's hold, under reconcileMu), now calling offlineUnstage before it reports reverted. offlineUnstage acts only for a move carrying staged_at and only on the stager the stage door pinned on the record (move.stager); it reads only the object store (its own stager report, this box's receipts object) and deletes moves/<host>/ in the staging bucket and region the stage fixed on the record (move.stage_bucket, move.stage_region; it refuses a missing one or this box's own bucket and then acknowledges nothing) and this box's move-receipts/<host>.json, then writes its stager report reverted; it reads no local file and makes no root decision. Read sites unchanged; one offset moved (whole:Route@701 to @843) registry:HoldOp@562,whole:Route@549,whole:Route@843 b917ac2d1976eec14839ed65e3e562327d54af956e86f5218ed011731182aa1b
+@legacy 06eb20ca59fc8172dfa9ef5bc6872987974b44f08202417705566816ecaeaf9f internal/host/movesoffline_seal.go movesWorker.offlineTakeSeal legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-precopy;basis=root-route (J1, census-ruling-moves-offline's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): census-ruling-moves-offline's body (seal steps 1 to 9 from the row: database path, next due, replication rung and timer rows, read for the seal) with one refusal before the mirror: a staged move (staged_at set) is sealed only into the staging bucket and region the stage fixed on the record (move.stage_bucket, move.stage_region), a comparison of record and edges-file fields that reads and writes nothing new. Read sites unchanged; every offset after the check moved by 487 bytes registry:DBPath@2621,registry:NextDue@12050,registry:NextDue@12962,registry:Replication@11955,registry:Replication@12937,whole:Route@10690,whole:Route@1713,whole:Route@2682,whole:Route@3463,whole:Route@5452,whole:Route@8226,whole:Timer@9377,whole:Timer@9442,whole:Timer@9448,whole:Timer@9707,whole:Timer@9806 f606873fb09e8644d449a36ae64cecbb84b2f83791d4ebec4cb491d3d3f7cdaa
+internal/host/movesoffline_seal.go movesWorker.offlineTakeSeal legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-precopy;basis=root-route (J1, census-ruling-moves-offline's disposition); the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): census-ruling-moves-offline's body (seal steps 1 to 9 from the row: database path, next due, replication rung and timer rows, read for the seal) with one refusal before the mirror: a staged move (staged_at set) is sealed only into the staging bucket and region the stage fixed on the record (move.stage_bucket, move.stage_region), a comparison of record and edges-file fields that reads and writes nothing new. Read sites unchanged; every offset after the check moved by 487 bytes registry:DBPath@2621,registry:NextDue@12050,registry:NextDue@12962,registry:Replication@11955,registry:Replication@12937,whole:Route@10690,whole:Route@1713,whole:Route@2682,whole:Route@3463,whole:Route@5452,whole:Route@8226,whole:Timer@9377,whole:Timer@9442,whole:Timer@9448,whole:Timer@9707,whole:Timer@9806 f606873fb09e8644d449a36ae64cecbb84b2f83791d4ebec4cb491d3d3f7cdaa
+@legacy 5e6a7aa9d23e5559c9b93f3a5c038e019a9a5d3bbcfbcf9229725ce7682b936a internal/provision/offline_move.go offlineBegin legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-precopy;basis=root-input (J4, as OfflineStart); the console (cmd/console serve under console.service, user carlos-console): new, OfflineStart's body moved unchanged with the state as a parameter (offline-stopping for OfflineStart, behaviour-identical; offline-staging plus staged_at for OfflineStage), the record transition the start and stage doors apply through SetRecordMove; its one site is rec.ActiveRun.Open(), a method on the record's run state (RunState.Open), not a file read file:Open@198 66e0216314538687d102a4cee1089a19896c7693260f748a046a5ff16e76dc1e
+internal/provision/offline_move.go offlineBegin legacy accepted-legacy accepted-legacy;rev3=census-ruling-moves-offline-precopy;basis=root-input (J4, as OfflineStart); the console (cmd/console serve under console.service, user carlos-console): new, OfflineStart's body moved unchanged with the state as a parameter (offline-stopping for OfflineStart, behaviour-identical; offline-staging plus staged_at for OfflineStage), the record transition the start and stage doors apply through SetRecordMove; its one site is rec.ActiveRun.Open(), a method on the record's run state (RunState.Open), not a file read file:Open@198 66e0216314538687d102a4cee1089a19896c7693260f748a046a5ff16e76dc1e
+# OfflineStart retired (H3, the carlos-run form). A removal is not in the seal.
+@remove internal/provision/offline_move.go OfflineStart
+@removed internal/provision/offline_move.go OfflineStart H3 census-ruling-moves-offline-precopy: still declared, and reads nothing now: its body and its one site (rec.ActiveRun.Open(), a method, not a file read) moved unchanged into offlineBegin, an accepted-legacy row in this overlay; OfflineStart calls offlineBegin with offline-stopping and is behaviour-identical. It runs in the console as carlos-console.
+@coverage . 649bb6a5f474312600edb213fb76a1a6444c6ff37a39412d1bbb04eeed49eabd
+@coverage internal/console 7bdd3caafb3a474a1bb3797c26c5a38e76b11ca7f0a46cd9c3a6af8ca62781c9
+@coverage internal/edgesfile 716924380656f85e428541c170e8c5bab9f0ad9c8abe931488f7a44e7953811e
+@coverage internal/host d58c8e606d61557dce896f82fc41c06969d99fd1e097b863aa986e0fa416e219
+@coverage internal/lease 980f38c4804ae1485c0c0df77bade8990b1d0248c0f44e41e0194c7731562a14
+@coverage internal/moves 33c7b507f13b4f4081c3bdb0a84e31d875e2cf1be61b480a7ca856193125e071
+@coverage internal/provision 210b174030fa650ef7345999643ef7e98add1a24378802cc134f9dea64d234dc
diff --git a/move.go b/move.go
index 3ebb08307..579078a79 100644
--- a/move.go
+++ b/move.go
@@ -83,8 +83,9 @@ const (
moveNoFlipTime = "The move's flip time could not be read, so its target cannot be declared lost from here."
moveOtherLoss = "The console asks to confirm a loss carlos move does not know. Use carlos ops host resolve."
moveLossChanged = "The move changed since it was read. Nothing was declared. Run the command again."
- // The offline subcommand named no verb, or one it does not have (MV-O56).
- moveOfflineUsage = "carlos move offline needs start, seal, flip, open, finish, abort or status."
+ // The offline subcommand named no verb, or one it does not have (MV-O181,
+ // which replaced MV-O56 when stage became a verb).
+ moveOfflineUsage = "carlos move offline needs stage, start, seal, flip, open, finish, abort or status."
)
func moveSay(out io.Writer, line string) {
@@ -122,7 +123,7 @@ type moveDeps struct {
type moveArgs struct {
verb string // request | status | cut | abort | resolve | offline
- offline string // the offline verb: start | seal | flip | open | finish | abort | status
+ offline string // the offline verb: stage | start | seal | flip | open | finish | abort | status
host, to, app string
account, accountID string
console string
@@ -172,7 +173,7 @@ func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {
case "cut", "abort":
fs.BoolVar(&a.noWait, "no-wait", false, "return once the console has accepted it")
case "offline":
- if a.offline == "start" {
+ if a.offline == "start" || a.offline == "stage" {
fs.StringVar(&a.to, "to", "", "the region to move the instance to")
}
case "resolve":
@@ -192,7 +193,7 @@ func parseMoveArgs(args []string, out io.Writer) (moveArgs, error) {
case a.host == "":
moveSay(out, moveNeedsHost)
return moveArgs{}, errors.New("no --host")
- case (a.verb == "request" || a.offline == "start") && a.to == "":
+ case (a.verb == "request" || a.offline == "start" || a.offline == "stage") && a.to == "":
moveSay(out, moveNeedsTo)
return moveArgs{}, errors.New("no --to")
case a.verb == "resolve" && a.retry && a.lostBox != "":
@@ -982,15 +983,17 @@ func (m *mover) refusal(r moveReply) int {
//
// An operator moves one instance with planned downtime, one step at a time:
// start, seal, flip, open, finish, abort, and status (offline-move plan,
-// Task 7). Each verb posts to its door, then polls the status door every
+// Task 7). stage comes before start when the operator wants the offline
+// window short: it copies the instance's data to the target while it keeps
+// serving, and can be run again. Each verb posts to its door, then polls the status door every
// interval until the step it started is done or has failed, and says which
// verb comes next. It never runs the next verb by itself. The exit codes are
// the family's: 0 the step is done, 3 the move was aborted, 4 the step failed
// or the move waits for the operator, 5 the console refused, 1 an error.
// Refusals are the console's own sentences (serverText); the CLI invents none.
-// offlineVerbs are the seven verbs after `carlos move offline`.
-var offlineVerbs = []string{"start", "seal", "flip", "open", "finish", "abort", "status"}
+// offlineVerbs are the eight verbs after `carlos move offline`.
+var offlineVerbs = []string{"stage", "start", "seal", "flip", "open", "finish", "abort", "status"}
// offlineView is the offline status view (GET) and the 202 every mutating
// door answers with.
@@ -1004,6 +1007,12 @@ type offlineView struct {
Boxes []string `json:"boxes"`
Next string `json:"next"`
CanAbort bool `json:"can_abort"`
+ // NeverOffline: staged and never started, so an abort only removes
+ // the staged copy.
+ NeverOffline bool `json:"never_offline"`
+ // StagedAt is the latest stage request (Unix nanoseconds), which
+ // Copying.Answered equals once the stager has run its round.
+ StagedAt int64 `json:"staged_at"`
// Failed: a box's report says the step cannot finish (the status view's
// own flag; the CLI says MV-O6 or MV-O9 for it).
Failed bool `json:"failed"`
@@ -1011,6 +1020,50 @@ type offlineView struct {
Reports []offlineReportView `json:"reports"`
Items []offlineItemView `json:"items"`
Last *moveOutcome `json:"last"`
+ // Copying is the pre-copy's progress (the console's stage), nil when no
+ // box has staged this move.
+ Copying *offlineCopying `json:"stage"`
+}
+
+// offlineCopying is the status view's stage: the stager's last round. The
+// fields are named apart from the tree's for the copy gate; Trouble is the
+// box's own diagnostic, not copy.
+type offlineCopying struct {
+ Stager string `json:"box"`
+ Phase string `json:"phase"`
+ RoundAt int64 `json:"at"`
+ Total int `json:"replica_objects"`
+ TotalBytes int64 `json:"replica_bytes"`
+ Done int `json:"staged_objects"`
+ DoneBytes int64 `json:"staged_bytes"`
+ Trouble string `json:"error"`
+ Answered int64 `json:"request"`
+}
+
+// megabytes is a byte count as a progress line says it.
+func megabytes(n int64) string { return fmt.Sprintf("%.1f", float64(n)/1e6) }
+
+// sayStaged says the stage's last round: how much of the replica the
+// target holds and when (MV-O183), that the round failed (MV-O185), or
+// that none has run yet (MV-O186).
+func (w *offlineRun) sayStaged(v offlineView) {
+ host, c := w.m.host, v.Copying
+ if c == nil {
+ moveSay(w.m.out, fmt.Sprintf("Waiting for the first staging round of %s.", host))
+ return
+ }
+ if c.Phase == moves.OfflinePhaseFailed {
+ moveSay(w.m.out, fmt.Sprintf("The last staging round failed: %s. Run carlos move offline stage --host %s --to %s to try again.", c.Trouble, host, v.To))
+ return
+ }
+ at := time.Unix(c.RoundAt, 0).UTC().Format("2006-01-02 15:04 UTC")
+ moveSay(w.m.out, fmt.Sprintf("Staged %d of %d objects for %s in %s (%s of %s MB) at %s.", c.Done, c.Total, host, v.To, megabytes(c.DoneBytes), megabytes(c.TotalBytes), at))
+}
+
+// sayStartWhenReady is MV-O184: what comes after a stage.
+func (w *offlineRun) sayStartWhenReady(v offlineView) {
+ host := w.m.host
+ moveSay(w.m.out, fmt.Sprintf("%s keeps serving. Run carlos move offline start --host %s --to %s when you are ready to take it offline.", host, host, v.To))
}
type offlineReportView struct {
@@ -1364,6 +1417,16 @@ func (w *offlineRun) sayServed(v offlineView) {
// headline says where a move in flight stands, in the verbs' own sentences.
func (w *offlineRun) headline(r offlineReply) {
v, host := r.v, w.m.host
+ if v.State == provision.OfflineStaging {
+ moveSay(w.m.out, fmt.Sprintf("Staging %s in %s. %s keeps serving while the data is copied.", host, v.To, host))
+ w.sayStaged(v)
+ if v.Next == "start" {
+ w.sayStartWhenReady(v)
+ return
+ }
+ w.sayWaiting(r)
+ return
+ }
moveSay(w.m.out, moves.MoveStartingLine(host, v.From, v.To))
if w.sayFailure(v) {
return
@@ -1413,6 +1476,8 @@ func (m *mover) offline(ctx context.Context, a moveArgs) int {
switch a.offline {
case "status":
return w.runStatus(ctx)
+ case "stage":
+ return w.runStage(ctx, a.to)
case "start":
return w.runStart(ctx, a.to)
case "seal":
@@ -1454,6 +1519,14 @@ func (w *offlineRun) runStatus(ctx context.Context) int {
return moveExitDone
}
w.headline(r)
+ if v.State == provision.OfflineStaging {
+ // Nothing of the manifest is checked before the start, so the
+ // items would all say waiting. The stage's own line said it.
+ if v.Copying != nil && v.Copying.Phase == moves.OfflinePhaseFailed {
+ return moveExitWaiting
+ }
+ return moveExitDone
+ }
failed := false
for _, it := range v.Items {
if it.Diagnostic == "" {
@@ -1479,6 +1552,37 @@ func (w *offlineRun) runStatus(ctx context.Context) int {
return moveExitDone
}
+// runStage posts the stage door, then waits for the round that answers the
+// stage request the post made (or a later one's, if another terminal ran
+// stage meanwhile).
+func (w *offlineRun) runStage(ctx context.Context, to string) int {
+ r, code := w.post(ctx, "/stage", map[string]string{"region": to})
+ if code >= 0 {
+ return code
+ }
+ host := w.m.host
+ if r.v.State != provision.OfflineStaging {
+ return w.stray(r)
+ }
+ moveSay(w.m.out, fmt.Sprintf("Staging %s in %s. %s keeps serving while the data is copied.", host, r.v.To, host))
+ asked := r.v.StagedAt
+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {
+ if r.v.State != provision.OfflineStaging {
+ return w.stray(r), true
+ }
+ c := r.v.Copying
+ if c == nil || c.Answered < asked || c.Answered != r.v.StagedAt {
+ return 0, false
+ }
+ w.sayStaged(r.v)
+ if c.Phase == moves.OfflinePhaseFailed {
+ return moveExitWaiting, true
+ }
+ w.sayStartWhenReady(r.v)
+ return moveExitDone, true
+ })
+}
+
func (w *offlineRun) runStart(ctx context.Context, to string) int {
r, code := w.post(ctx, "", map[string]string{"region": to})
if code >= 0 {
@@ -1640,6 +1744,11 @@ func (w *offlineRun) runAbort(ctx context.Context) int {
if code >= 0 {
return code
}
+ // A move aborted while staging was never offline: there is nothing to
+ // move back, only the staged copy to remove.
+ if r.v.NeverOffline {
+ return w.abortStaged(ctx, r)
+ }
host := w.m.host
moveSay(w.m.out, fmt.Sprintf("Moving %s back to %s.", host, r.v.From))
return w.follow(ctx, r, func(r offlineReply) (int, bool) {
@@ -1654,3 +1763,20 @@ func (w *offlineRun) runAbort(ctx context.Context) int {
return w.stray(r), true
})
}
+
+// abortStaged follows an abort from staging: the stager removes the staged
+// copy, every source box reports, and the operator runs finish.
+func (w *offlineRun) abortStaged(ctx context.Context, r offlineReply) int {
+ host := w.m.host
+ moveSay(w.m.out, fmt.Sprintf("Removing the staged copy of %s from %s.", host, r.v.To))
+ return w.follow(ctx, r, func(r offlineReply) (int, bool) {
+ if r.v.State == provision.OfflineReverting {
+ if r.v.Next == "finish" {
+ moveSay(w.m.out, fmt.Sprintf("The staged copy of %s is removed. Run carlos move offline finish --host %s to end the move.", host, host))
+ return moveExitRolled, true
+ }
+ return 0, false
+ }
+ return w.stray(r), true
+ })
+}
diff --git a/move_test.go b/move_test.go
index e91be1486..bff16ba4c 100644
--- a/move_test.go
+++ b/move_test.go
@@ -1238,6 +1238,8 @@ func TestParseMoveArgsOffline(t *testing.T) {
sub string
}{
{[]string{"offline", "start", "--host", oHost, "--to", "uk"}, true, "start"},
+ {[]string{"offline", "stage", "--host", oHost, "--to", "uk"}, true, "stage"},
+ {[]string{"offline", "stage", "--host", oHost}, false, ""},
{[]string{"offline", "seal", "--host", oHost}, true, "seal"},
{[]string{"offline", "flip", "--host", oHost}, true, "flip"},
{[]string{"offline", "open", "--host", oHost}, true, "open"},
@@ -1262,7 +1264,7 @@ func TestParseMoveArgsOffline(t *testing.T) {
if _, err := parseMoveArgs([]string{"offline", "cut", "--host", oHost}, &out); err == nil {
t.Fatal("an unknown offline verb parsed")
}
- wantLines(t, out.String(), "carlos move offline needs start, seal, flip, open, finish, abort or status.")
+ wantLines(t, out.String(), "carlos move offline needs stage, start, seal, flip, open, finish, abort or status.")
out.Reset()
_, _ = parseMoveArgs([]string{"offline", "start", "--host", oHost}, &out)
wantLines(t, out.String(), "carlos move needs --to <region>.")
@@ -1810,3 +1812,103 @@ func TestMoveOfflineFailedReportAfterTheOpen(t *testing.T) {
t.Errorf("a reverting move with an old failed target: exit %d:\n%s", code, out)
}
}
+
+// ---- carlos move offline stage ----
+
+const (
+ oStaging = "Staging acme.shop.example.com in uk. acme.shop.example.com keeps serving while the data is copied."
+ oReady = "acme.shop.example.com keeps serving. Run carlos move offline start --host acme.shop.example.com --to uk when you are ready to take it offline."
+ oUnstage = "Removing the staged copy of acme.shop.example.com from uk."
+ oUnstaged = "The staged copy of acme.shop.example.com is removed. Run carlos move offline finish --host acme.shop.example.com to end the move."
+)
+
+// stageView is a staging move's view with the stager's last round, which
+// answered the request at answered (0: no round yet).
+func stageView(stagedAt, answered int64, phase string, extra frame) frame {
+ f := frame{"staged_at": stagedAt, "never_offline": true, "sealer": "", "boxes": nil, "can_abort": true}
+ if answered != 0 {
+ f["stage"] = frame{"box": "ie-box", "phase": phase, "at": int64(1792573200), "replica_objects": 305, "replica_bytes": 1_580_000_000,
+ "staged_objects": 305, "staged_bytes": 1_580_000_000, "round_objects": 305, "round_bytes": 1_580_000_000, "request": answered,
+ "error": "mirror: copy instances/x: boom"}
+ }
+ for k, v := range extra {
+ f[k] = v
+ }
+ return offView("offline-staging", f)
+}
+
+// stage posts and then waits for the round that answers its own request:
+// not the earlier round the answer already showed.
+func TestMoveOfflineStage(t *testing.T) {
+ c := &fakeOfflineConsole{
+ posts: map[string][]offPost{"/stage": {{202, stageView(200, 100, "staged", nil)}}},
+ gets: []frame{
+ stageView(200, 100, "staged", nil),
+ stageView(200, 200, "staged", frame{"next": "start"}),
+ },
+ }
+ code, out := runOfflineAgainst(t, c, "stage", "--to", "uk")
+ want := oStaging + "\nStaged 305 of 305 objects for acme.shop.example.com in uk (1580.0 of 1580.0 MB) at 2026-10-21 09:00 UTC.\n" + oReady + "\n"
+ if code != 0 || out != want {
+ t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want)
+ }
+ if got := c.requests(); len(got) == 0 || got[0] != `POST /offline/stage {"region":"uk"}` {
+ t.Fatalf("requests %q", got)
+ }
+ if c.posted("") != 0 {
+ t.Error("stage started the move by itself")
+ }
+ // A failed round: the box's reason, the way on, exit 4.
+ c = &fakeOfflineConsole{
+ posts: map[string][]offPost{"/stage": {{202, stageView(300, 0, "", nil)}}},
+ gets: []frame{stageView(300, 300, "failed", nil)},
+ }
+ code, out = runOfflineAgainst(t, c, "stage", "--to", "uk")
+ want = oStaging + "\nThe last staging round failed: mirror: copy instances/x: boom. Run carlos move offline stage --host acme.shop.example.com --to uk to try again.\n"
+ if code != 4 || out != want {
+ t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want)
+ }
+ // A refusal is the console's sentence, as sent.
+ refusal := "acme.shop.example.com is staged for uk. abort the move to move it to another region"
+ c = &fakeOfflineConsole{posts: map[string][]offPost{"/stage": {{409, frame{"error": refusal}}}}}
+ if code, out := runOfflineAgainst(t, c, "stage", "--to", "us"); code != 5 || out != refusal+"\n" {
+ t.Fatalf("exit %d:\n%s", code, out)
+ }
+}
+
+// status while staging: the stage's line instead of the manifest's items.
+func TestMoveOfflineStatusWhileStaging(t *testing.T) {
+ c := &fakeOfflineConsole{gets: []frame{stageView(200, 0, "", nil)}}
+ code, out := runOfflineAgainst(t, c, "status")
+ if want := oStaging + "\nWaiting for the first staging round of acme.shop.example.com.\n"; code != 0 || out != want {
+ t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want)
+ }
+ c = &fakeOfflineConsole{gets: []frame{stageView(200, 200, "staged", frame{"next": "start"})}}
+ code, out = runOfflineAgainst(t, c, "status")
+ want := oStaging + "\nStaged 305 of 305 objects for acme.shop.example.com in uk (1580.0 of 1580.0 MB) at 2026-10-21 09:00 UTC.\n" + oReady + "\n"
+ if code != 0 || out != want {
+ t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want)
+ }
+ c = &fakeOfflineConsole{gets: []frame{stageView(200, 200, "failed", nil)}}
+ if code, _ := runOfflineAgainst(t, c, "status"); code != 4 {
+ t.Fatalf("status of a failed round exit %d, want 4", code)
+ }
+}
+
+// abort of a move still staging: nothing to move back, only the staged copy.
+func TestMoveOfflineAbortStaged(t *testing.T) {
+ c := &fakeOfflineConsole{
+ posts: map[string][]offPost{"/abort": {{202, offView("offline-reverting", frame{"never_offline": true, "sealer": ""})}}},
+ gets: []frame{
+ offView("offline-reverting", frame{"never_offline": true, "sealer": ""}),
+ offView("offline-reverting", frame{"never_offline": true, "sealer": "", "next": "finish"}),
+ },
+ }
+ code, out := runOfflineAgainst(t, c, "abort")
+ if want := oUnstage + "\n" + oUnstaged + "\n"; code != 3 || out != want {
+ t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want)
+ }
+ if c.posted("/finish") != 0 {
+ t.Error("abort ran finish by itself")
+ }
+}
diff --git a/offline_move_e2e_test.go b/offline_move_e2e_test.go
index fe8e07eb2..5299341ab 100644
--- a/offline_move_e2e_test.go
+++ b/offline_move_e2e_test.go
@@ -21,12 +21,18 @@ import (
"github.com/carlosframework/platform/internal/status"
)
-// TestMoveOfflineAgainstTheRealDoors drives `carlos move offline start` and
-// `status` through moveMain against a real console.Server: the stub tests
-// type the status view by hand, and this one ties the CLI's view structs to
-// the JSON the console writes (next, waiting, failed, items, reports, left).
-// The boxes are played by reports written where the real boxes write them.
-func TestMoveOfflineAgainstTheRealDoors(t *testing.T) {
+// offlineE2E is a real console.Server with one hibernating instance of the
+// app shop in ie, an operator's CLI login, and the ie and uk boxes
+// published with caps. run is moveMain for `carlos move offline <args>`
+// against it; put writes a box's report where the real box writes it.
+type offlineE2E struct {
+ host string
+ rel release.Store
+ run func(c context.Context, args ...string) (int, string)
+ put func(id, box, role, phase string, edit func(*moves.OfflineReport))
+}
+
+func newOfflineE2E(t *testing.T, caps []string) *offlineE2E {
const (
host = "acme.shop.example.com"
addr = "paul@keymail.dev"
@@ -109,7 +115,7 @@ func TestMoveOfflineAgainstTheRealDoors(t *testing.T) {
"v": 1, "box": b.box, "box_id": b.box, "eip": "1.2.3.4", "region": b.aws, "placement": b.placement,
"data_bucket": b.bucket, "hibernation_configured": true, "serves_shared_pool": true,
"accepts_inbound": true, "secrets_key_fp": fp,
- "capabilities": []string{"region-ha-1", "offline-move-1"}, "hosts": map[string]any{},
+ "capabilities": caps, "hosts": map[string]any{},
})
if _, err := rel.Put(ctx, "control/edges/"+b.box+".json", body, ""); err != nil {
t.Fatal(err)
@@ -150,6 +156,19 @@ func TestMoveOfflineAgainstTheRealDoors(t *testing.T) {
}
}
+ return &offlineE2E{host: host, rel: rel, run: run, put: put}
+}
+
+// TestMoveOfflineAgainstTheRealDoors drives `carlos move offline start` and
+// `status` through moveMain against a real console.Server: the stub tests
+// type the status view by hand, and this one ties the CLI's view structs to
+// the JSON the console writes (next, waiting, failed, items, reports, left).
+// The boxes are played by reports written where the real boxes write them.
+func TestMoveOfflineAgainstTheRealDoors(t *testing.T) {
+ e := newOfflineE2E(t, []string{"region-ha-1", "offline-move-1"})
+ host, rel, run, put := e.host, e.rel, e.run, e.put
+ ctx := context.Background()
+
// start waits for the source box; it runs on while the test plays it.
ctxStart, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
@@ -216,3 +235,113 @@ func TestMoveOfflineAgainstTheRealDoors(t *testing.T) {
_, out = run(ctx, "status")
wantLines(t, out, "ie-box left /opt/carlos/live/"+host+" in place: a root-owned directory.")
}
+
+// TestMoveOfflineStageAgainstTheRealDoors drives stage, status, start and
+// abort of a staged move through moveMain against a real console.Server,
+// tying the CLI's stage view to the JSON the console writes. The stager is
+// played by its report, written for the stage request the record carries.
+func TestMoveOfflineStageAgainstTheRealDoors(t *testing.T) {
+ e := newOfflineE2E(t, []string{"region-ha-1", "offline-move-1", "offline-stage-1"})
+ host, ctx := e.host, context.Background()
+ record := func() *provision.Record {
+ t.Helper()
+ rec, err := provision.GetRecord(ctx, e.rel, release.OpsAccount, "shop", host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return rec
+ }
+ // round plays the stager: one round for the record's current request.
+ round := func(id string) {
+ t.Helper()
+ e.put(id, "ie-box", moves.OfflineRoleStager, moves.OfflinePhaseStaged, func(r *moves.OfflineReport) {
+ r.Held, r.Fenced, r.Quiesced = false, false, false
+ r.StageRequest = *record().Move.StagedAt
+ r.ReplicaObjects, r.ReplicaBytes, r.StagedObjects, r.StagedBytes = 288, 72_000_000, 288, 72_000_000
+ })
+ }
+ type result struct {
+ code int
+ out string
+ }
+ // stage waits for its round; the test plays the stager once the record
+ // carries the request.
+ stage := func(wantRequestAfter time.Time) (string, result) {
+ t.Helper()
+ sctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+ done := make(chan result, 1)
+ go func() {
+ code, out := e.run(sctx, "stage", "--to", "uk")
+ done <- result{code, out}
+ }()
+ var id string
+ for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
+ if rec, err := provision.GetRecord(ctx, e.rel, release.OpsAccount, "shop", host); err == nil && rec.Move != nil && rec.Move.StagedAt != nil && rec.Move.StagedAt.After(wantRequestAfter) {
+ id = rec.Move.ID
+ break
+ }
+ }
+ if id == "" {
+ t.Fatal("stage never wrote its request")
+ }
+ round(id)
+ select {
+ case r := <-done:
+ return id, r
+ case <-time.After(20 * time.Second):
+ t.Fatal("stage never finished")
+ }
+ return "", result{}
+ }
+ staging := "Staging " + host + " in uk. " + host + " keeps serving while the data is copied."
+ ready := host + " keeps serving. Run carlos move offline start --host " + host + " --to uk when you are ready to take it offline."
+ id, r := stage(time.Time{})
+ if r.code != 0 {
+ t.Fatalf("stage exit %d:\n%s", r.code, r.out)
+ }
+ wantLines(t, r.out, staging, ready)
+ if !strings.Contains(r.out, "Staged 288 of 288 objects for "+host+" in uk (72.0 of 72.0 MB) at ") {
+ t.Fatalf("stage output lacks the progress line:\n%s", r.out)
+ }
+ // Again: a new request, answered by a new round.
+ first := *record().Move.StagedAt
+ if id2, r := stage(first); r.code != 0 || id2 != id {
+ t.Fatalf("second stage exit %d (move %s, want %s):\n%s", r.code, id2, id, r.out)
+ }
+ code, out := e.run(ctx, "status")
+ if code != 0 {
+ t.Fatalf("status exit %d:\n%s", code, out)
+ }
+ wantLines(t, out, staging, ready)
+
+ // abort from staging: the staged copy goes, then finish.
+ actx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+ aborted := make(chan result, 1)
+ go func() {
+ code, out := e.run(actx, "abort")
+ aborted <- result{code, out}
+ }()
+ for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && record().Move.State != provision.OfflineReverting; time.Sleep(10 * time.Millisecond) {
+ }
+ e.put(id, "ie-box", moves.OfflineRoleSource, moves.OfflinePhaseReverted, func(r *moves.OfflineReport) { r.Held = false })
+ // The pinned stager says the staged copy is gone: the finish waits for it.
+ e.put(id, "ie-box", moves.OfflineRoleStager, moves.OfflinePhaseReverted, func(r *moves.OfflineReport) { r.Held, r.Fenced, r.Quiesced = false, false, false })
+ select {
+ case r := <-aborted:
+ if r.code != 3 {
+ t.Fatalf("abort exit %d:\n%s", r.code, r.out)
+ }
+ wantLines(t, r.out, "Removing the staged copy of "+host+" from uk.",
+ "The staged copy of "+host+" is removed. Run carlos move offline finish --host "+host+" to end the move.")
+ case <-time.After(20 * time.Second):
+ t.Fatal("abort never finished")
+ }
+ if code, out := e.run(ctx, "finish"); code != 3 {
+ t.Fatalf("finish exit %d:\n%s", code, out)
+ }
+ if rec := record(); rec.Move != nil || rec.LastMove == nil || rec.LastMove.Outcome != provision.OfflineOutcomeAborted || rec.LastMove.OfflineS != 0 {
+ t.Fatalf("after finish: %+v %+v", rec.Move, rec.LastMove)
+ }
+}